def get_repo(remote): if not changegroup or experiment('wire'): if not changegroup: logging.warning('Mercurial libraries not found. Falling back to ' 'native access.') logging.warning( 'Native access to mercurial repositories is experimental!') stream = HgRepoHelper.connect(remote.url) if stream: return bundlerepo(remote.url, stream) return HelperRepo(remote.url) if remote.parsed_url.scheme == 'file': path = remote.parsed_url.path if sys.platform == 'win32': # TODO: This probably needs more thought. path = path.lstrip('/') if not os.path.isdir(path): return bundlerepo(path) ui = get_ui() if changegroup and remote.parsed_url.scheme == 'file': repo = localpeer(ui, path) else: try: repo = hg.peer(ui, {}, remote.url) except (error.RepoError, urllib2.HTTPError, IOError): return bundlerepo(remote.url, url.open(ui, remote.url)) assert repo.capable('getbundle') return repo
def _get_repo(remote): if not changegroup or experiment('wire'): if not changegroup and not check_enabled('no-mercurial'): logging.warning('Mercurial libraries not found. Falling back to ' 'experimental native access.') stream = HgRepoHelper.connect(remote.url) if stream: return bundlerepo(remote.url, stream) return HelperRepo(remote.url) if remote.parsed_url.scheme == b'file': # Make file://c:/... paths work by taking the netloc path = remote.parsed_url.netloc + remote.parsed_url.path if sys.platform == 'win32': # TODO: This probably needs more thought. path = path.lstrip(b'/') if not os.path.isdir(path): return bundlerepo(path) ui = get_ui() if changegroup and remote.parsed_url.scheme == b'file': repo = localpeer(ui, path) else: try: repo = hg.peer(ui, {}, remote.url) except (error.RepoError, HTTPError, IOError): if remote.parsed_url.scheme in ('http', 'https'): return bundlerepo(remote.url, HTTPReader(remote.url)) raise assert repo.capable(b'getbundle') return repo
def __call__(self, store): changeset_chunks = ChunksCollection( progress_iter('Reading %d changesets', next(self._bundle, None))) if experiment('store-manifest'): for rev_chunk in progress_iter( 'Reading and importing %d manifests', next(self._bundle, None)): GitHgHelper.store('manifest', rev_chunk) store.check_manifest(rev_chunk) else: for mn in progress_iter( 'Reading and importing %d manifests', iter_initialized( store.manifest, iter_chunks(next(self._bundle, None), ManifestInfo))): store.store_manifest(mn) for rev_chunk in progress_iter('Reading and importing %d files', next(self._bundle, None)): GitHgHelper.store('file', rev_chunk) if next(self._bundle, None) is not None: assert False del self._bundle for cs in progress_iter( 'Importing %d changesets', changeset_chunks.iter_initialized(lambda x: x, store.changeset, Changeset.from_chunk)): try: store.store_changeset(cs) except NothingToGraftException: logging.warn('Cannot graft %s, not importing.', cs.node)
def getbundle(repo, store, heads, branch_names): if isinstance(repo, bundlerepo): bundle = repo._unbundler else: common = findcommon(repo, store, store.heads(branch_names)) logging.info('common: %s', common) bundle = None got_partial = False if not common: if not store._has_metadata: manifest = Git.config('cinnabar.clone') if not manifest and experiment('git-clone') and \ repo.capable('cinnabarclone'): manifest = repo._call('cinnabarclone') if manifest: got_partial = do_cinnabarclone(repo, manifest, store) if not got_partial: if check_enabled('cinnabarclone'): raise Exception('cinnabarclone failed.') logging.warn('Falling back to normal clone.') if not got_partial and repo.capable('clonebundles'): bundle = get_clonebundle(repo) got_partial = bool(bundle) if not got_partial and check_enabled('clonebundles'): raise Exception('clonebundles failed.') if bundle: bundle = unbundler(bundle) # Manual move semantics apply_bundle = BundleApplier(bundle) del bundle apply_bundle(store) if got_partial: # Eliminate the heads that we got from the clonebundle or # cinnabarclone. heads = [h for h in heads if not store.changeset_ref(h)] if not heads: return common = findcommon(repo, store, store.heads(branch_names)) logging.info('common: %s', common) kwargs = {} if unbundle20 and repo.capable('bundle2'): bundle2caps = { 'HG20': (), 'changegroup': ('01', '02'), } kwargs['bundlecaps'] = set( ('HG20', 'bundle2=%s' % urllib.quote(encodecaps(bundle2caps)))) bundle = repo.getbundle('bundle', heads=[unhexlify(h) for h in heads], common=[unhexlify(h) for h in common], **kwargs) bundle = unbundler(bundle) # Manual move semantics apply_bundle = BundleApplier(bundle) del bundle apply_bundle(store)
def __init__(self, bundle): self._bundle = bundle self._use_store_changegroup = False if GitHgHelper.supports(GitHgHelper.STORE_CHANGEGROUP) and \ experiment('store-changegroup'): self._use_store_changegroup = True self._bundle = store_changegroup(bundle)
def get_repo(remote): if not changegroup or experiment('wire'): if not changegroup and not check_enabled('no-mercurial'): logging.warning('Mercurial libraries not found. Falling back to ' 'native access.') logging.warning( 'Native access to mercurial repositories is experimental!') stream = HgRepoHelper.connect(remote.url) if stream: return bundlerepo(remote.url, stream) return HelperRepo(remote.url) if remote.parsed_url.scheme == 'file': # Make file://c:/... paths work by taking the netloc path = remote.parsed_url.netloc + remote.parsed_url.path if sys.platform == 'win32': # TODO: This probably needs more thought. path = path.lstrip('/') if not os.path.isdir(path): return bundlerepo(path) ui = get_ui() if changegroup and remote.parsed_url.scheme == 'file': repo = localpeer(ui, path) else: try: repo = hg.peer(ui, {}, remote.url) except (error.RepoError, urllib2.HTTPError, IOError): return bundlerepo(remote.url, HTTPReader(remote.url)) assert repo.capable('getbundle') return repo
def __call__(self, store): changeset_chunks = ChunksCollection( progress_iter('Reading {} changesets', next(self._bundle, None))) if experiment('store-manifest'): for rev_chunk in progress_iter( 'Reading and importing {} manifests', next(self._bundle, None)): GitHgHelper.store('manifest', rev_chunk) store.check_manifest(rev_chunk) else: for mn in progress_iter( 'Reading and importing {} manifests', iter_initialized( store.manifest, iter_chunks(next(self._bundle, None), ManifestInfo))): store.store_manifest(mn) def enumerate_files(iter): last_name = None count_names = 0 for count_chunks, (name, chunk) in enumerate(iter): if name != last_name: count_names += 1 last_name = name yield (count_chunks, count_names), chunk for rev_chunk in progress_enum( 'Reading and importing {} revisions of {} files', enumerate_files(next(self._bundle, None))): GitHgHelper.store('file', rev_chunk) if next(self._bundle, None) is not None: assert False del self._bundle for cs in progress_iter( 'Importing {} changesets', changeset_chunks.iter_initialized(lambda x: x, store.changeset, Changeset.from_chunk)): try: store.store_changeset(cs) except NothingToGraftException: logging.warn('Cannot graft %s, not importing.', cs.node)
def get_repo(remote): if remote.parsed_url.scheme == 'file': path = remote.parsed_url.path if sys.platform == 'win32': # TODO: This probably needs more thought. path = path.lstrip('/') if not os.path.isdir(path): return bundlerepo(path) if not changegroup or experiment('wire'): if not changegroup: logging.warning('Mercurial libraries not found. Falling back to ' 'native access.') logging.warning( 'Native access to mercurial repositories is experimental!') return HelperRepo(remote.url) if changegroup and remote.parsed_url.scheme == 'file': repo = localpeer(get_ui(), path) else: repo = hg.peer(get_ui(), {}, remote.url) assert repo.capable('getbundle') return repo
def create_hg_manifest(self, commit, parents): manifest = GeneratedManifestInfo(NULL_NODE_ID) changeset_files = [] if parents: parent_changeset = self.changeset(self.hg_changeset(parents[0])) parent_manifest = self.manifest(parent_changeset.manifest) parent_node = parent_manifest.node if len(parents) == 2: parent2_changeset = self.changeset(self.hg_changeset(parents[1])) parent2_manifest = self.manifest(parent2_changeset.manifest) parent2_node = parent2_manifest.node if parent_node == parent2_node: parents = parents[:1] if not parents: for line in Git.ls_tree(commit, recursive=True): mode, typ, sha1, path = line node = self.create_file(sha1, git_manifest_parents=(), path=path) manifest.add(path, node, self.ATTR[mode], modified=True) changeset_files.append(path) manifest.parents = [] manifest.delta_node = NULL_NODE_ID return manifest, changeset_files elif len(parents) == 2: if not experiment('merge'): raise Exception('Pushing merges is not supported yet') if not self._merge_warn: logging.warning('Pushing merges is experimental.') logging.warning('This may irremediably push bad state to the ' 'mercurial server!') self._merge_warn = 1 git_manifests = (self.manifest_ref(parent_node), self.manifest_ref(parent2_node)) # TODO: this would benefit from less git queries changes = list(get_changes(commit, parents)) files = [ (path, mode, sha1) for mode, _, sha1, path in Git.ls_tree(commit, recursive=True) ] manifests = sorted_merge(parent_manifest, parent2_manifest, key=lambda i: i.path, non_key=lambda i: i) for line in sorted_merge(files, sorted_merge(changes, manifests)): path, f, (change, (manifest_line_p1, manifest_line_p2)) = line if not f: # File was removed if manifest_line_p1: manifest.removed.add(path) changeset_files.append(path) continue mode, sha1 = f attr = self.ATTR[mode] if manifest_line_p1 and not manifest_line_p2: file_parents = (manifest_line_p1.sha1, ) elif manifest_line_p2 and not manifest_line_p1: file_parents = (manifest_line_p2.sha1, ) elif not manifest_line_p1 and not manifest_line_p2: file_parents = () elif manifest_line_p1.sha1 == manifest_line_p2.sha1: file_parents = (manifest_line_p1.sha1, ) else: if self._merge_warn == 1: logging.warning('This may take a while...') self._merge_warn = 2 file_parents = (manifest_line_p1.sha1, manifest_line_p2.sha1) assert file_parents is not None f = self._create_file_internal( sha1, *file_parents, git_manifest_parents=git_manifests, path=path) file_parents = tuple(p for p in (f.parent1, f.parent2) if p != NULL_NODE_ID) merged = len(file_parents) == 2 if not merged and file_parents: if self.git_file_ref(file_parents[0]) == sha1: node = file_parents[0] else: merged = True if merged: node = self._store_file_internal(f) else: node = file_parents[0] attr_change = (manifest_line_p1 and manifest_line_p1.attr != attr) manifest.add(path, node, attr, modified=merged or attr_change) if merged or attr_change: changeset_files.append(path) if manifest.raw_data == parent_manifest.raw_data: return parent_manifest, [] manifest.parents = (parent_node, parent2_node) return manifest, changeset_files def process_diff(diff): for (mode_before, mode_after, sha1_before, sha1_after, status, path) in diff: if status[:1] == b'R': yield status[1:], (b'000000', sha1_before, NULL_NODE_ID, b'D') yield path, (mode_after, sha1_before, sha1_after, status) git_diff = sorted(l for l in process_diff( GitHgHelper.diff_tree(parents[0], commit, detect_copy=True))) if not git_diff: return parent_manifest, [] parent_lines = OrderedDict((l.path, l) for l in parent_manifest) items = manifest.items for line in sorted_merge(iteritems(parent_lines), git_diff, non_key=lambda i: i[1]): path, manifest_line, change = line if not change: items.append(manifest_line) continue mode_after, sha1_before, sha1_after, status = change path2 = status[1:] status = status[:1] attr = self.ATTR.get(mode_after) if status == b'D': manifest.removed.add(path) changeset_files.append(path) continue if status in b'MT': if sha1_before == sha1_after: node = manifest_line.sha1 else: node = self.create_file( sha1_after, manifest_line.sha1, git_manifest_parents=( self.manifest_ref(parent_node), ), path=path) elif status in b'RC': if sha1_after != EMPTY_BLOB: node = self.create_copy( (path2, parent_lines[path2].sha1), sha1_after, git_manifest_parents=( self.manifest_ref(parent_node), ), path=path) else: node = self.create_file( sha1_after, git_manifest_parents=( self.manifest_ref(parent_node), ), path=path) else: assert status == b'A' node = self.create_file( sha1_after, git_manifest_parents=(self.manifest_ref(parent_node), ), path=path) manifest.add(path, node, attr, modified=True) changeset_files.append(path) manifest.parents = (parent_node, ) manifest.delta_node = parent_node return manifest, changeset_files
def create_hg_manifest(self, commit, parents): manifest = GeneratedManifestInfo(NULL_NODE_ID) changeset_files = [] if parents: parent_changeset = self.changeset(self.hg_changeset(parents[0])) parent_manifest = self.manifest(parent_changeset.manifest) parent_node = parent_manifest.node if len(parents) == 2: parent2_changeset = self.changeset(self.hg_changeset(parents[1])) parent2_manifest = self.manifest(parent2_changeset.manifest) parent2_node = parent2_manifest.node if parent_node == parent2_node: parents = parents[:1] if not parents: for line in Git.ls_tree(commit, recursive=True): mode, typ, sha1, path = line node = self.create_file(sha1, git_manifest_parents=(), path=path) manifest.append_line(ManifestLine(path, node, self.ATTR[mode]), modified=True) changeset_files.append(path) manifest.set_parents(NULL_NODE_ID) manifest.delta_node = NULL_NODE_ID return manifest, changeset_files elif len(parents) == 2: if not experiment('merge'): raise Exception('Pushing merges is not supported yet') if not self._merge_warn: logging.warning('Pushing merges is experimental.') logging.warning('This may irremediably push bad state to the ' 'mercurial server!') self._merge_warn = 1 git_manifests = (self.manifest_ref(parent_node), self.manifest_ref(parent2_node)) # TODO: this would benefit from less git queries changes = list(get_changes(commit, parents)) files = [(path, mode, sha1) for mode, _, sha1, path in Git.ls_tree(commit, recursive=True)] manifests = sorted_merge(parent_manifest._lines, parent2_manifest._lines, key=lambda i: i.name, non_key=lambda i: i) for line in sorted_merge(files, sorted_merge(changes, manifests)): path, f, (change, (manifest_line_p1, manifest_line_p2)) = line if not f: # File was removed if manifest_line_p1: manifest.removed.add(path) changeset_files.append(path) continue mode, sha1 = f attr = self.ATTR[mode] if manifest_line_p1 and not manifest_line_p2: file_parents = (manifest_line_p1.node,) elif manifest_line_p2 and not manifest_line_p1: file_parents = (manifest_line_p2.node,) elif not manifest_line_p1 and not manifest_line_p2: file_parents = () elif manifest_line_p1.node == manifest_line_p2.node: file_parents = (manifest_line_p1.node,) else: if self._merge_warn == 1: logging.warning('This may take a while...') self._merge_warn = 2 file_parents = (manifest_line_p1.node, manifest_line_p2.node) assert file_parents is not None f = self._create_file_internal( sha1, *file_parents, git_manifest_parents=git_manifests, path=path ) file_parents = tuple(p for p in (f.parent1, f.parent2) if p != NULL_NODE_ID) merged = len(file_parents) == 2 if not merged and file_parents: if self.git_file_ref(file_parents[0]) == sha1: node = file_parents[0] else: merged = True if merged: node = self._store_file_internal(f) else: node = file_parents[0] attr_change = (manifest_line_p1 and manifest_line_p1.attr != attr) manifest.append_line(ManifestLine(path, node, attr), modified=merged or attr_change) if merged or attr_change: changeset_files.append(path) if manifest.data == parent_manifest.data: return parent_manifest, [] manifest.set_parents(parent_node, parent2_node) return manifest, changeset_files def process_diff(diff): for (mode_before, mode_after, sha1_before, sha1_after, status, path) in diff: if status[0] == 'R': yield status[1:], ( '000000', sha1_before, NULL_NODE_ID, 'D') yield path, (mode_after, sha1_before, sha1_after, status) git_diff = sorted( l for l in process_diff(GitHgHelper.diff_tree( parents[0], commit, detect_copy=True)) ) if not git_diff: return parent_manifest, [] parent_lines = OrderedDict((l.name, l) for l in parent_manifest._lines) for line in sorted_merge(parent_lines.iteritems(), git_diff, non_key=lambda i: i[1]): path, manifest_line, change = line if not change: manifest.append_line(manifest_line) continue mode_after, sha1_before, sha1_after, status = change path2 = status[1:] status = status[0] attr = self.ATTR.get(mode_after) if status == 'D': manifest.removed.add(path) changeset_files.append(path) continue if status in 'MT': if sha1_before == sha1_after: node = manifest_line.node else: node = self.create_file( sha1_after, str(manifest_line.node), git_manifest_parents=( self.manifest_ref(parent_node),), path=path) elif status in 'RC': if sha1_after != EMPTY_BLOB: node = self.create_copy( (path2, parent_lines[path2].node), sha1_after, git_manifest_parents=( self.manifest_ref(parent_node),), path=path) else: node = self.create_file( sha1_after, git_manifest_parents=( self.manifest_ref(parent_node),), path=path) else: assert status == 'A' node = self.create_file( sha1_after, git_manifest_parents=( self.manifest_ref(parent_node),), path=path) manifest.append_line(ManifestLine(path, node, attr), modified=True) changeset_files.append(path) manifest.set_parents(parent_node) manifest.delta_node = parent_node return manifest, changeset_files
def create_hg_manifest(self, commit, parents): manifest = GeneratedManifestInfo(NULL_NODE_ID) if parents: parent_changeset_data = self.read_changeset_data(parents[0]) parent_manifest = self.manifest(parent_changeset_data['manifest']) parent_node = parent_manifest.node if len(parents) == 2: parent2_changeset_data = self.read_changeset_data(parents[1]) parent2_manifest = self.manifest( parent2_changeset_data['manifest']) parent2_node = parent2_manifest.node if parent_node == parent2_node: parents = parents[:1] if not parents: for line in Git.ls_tree(commit, recursive=True): mode, typ, sha1, path = line node = self.create_file(sha1, git_manifest_parents=(), path=path) manifest.append_line(ManifestLine(path, node, self.ATTR[mode]), modified=True) manifest.set_parents(NULL_NODE_ID) manifest.delta_node = NULL_NODE_ID return manifest elif len(parents) == 2: if not experiment('merge'): raise Exception('Pushing merges is not supported yet') logging.warning('Pushing merges is experimental.') logging.warning('This may irremediably push bad state to the ' 'mercurial server!') warned = False git_manifests = (self.manifest_ref(parent_node), self.manifest_ref(parent2_node)) # TODO: this would benefit from less git queries changes = list(get_changes(commit, parents)) files = [ (path, mode, sha1) for mode, _, sha1, path in Git.ls_tree(commit, recursive=True) ] manifests = sorted_merge(parent_manifest._lines, parent2_manifest._lines, key=lambda i: i.name, non_key=lambda i: i) for line in sorted_merge(files, sorted_merge(changes, manifests)): path, f, (change, (manifest_line_p1, manifest_line_p2)) = line if not f: # File was removed if manifest_line_p1: manifest.removed.add(path) continue mode, sha1 = f attr = self.ATTR[mode] if manifest_line_p1 and not manifest_line_p2: file_parents = (manifest_line_p1.node, ) elif manifest_line_p2 and not manifest_line_p1: file_parents = (manifest_line_p2.node, ) elif not manifest_line_p1 and not manifest_line_p2: file_parents = () elif manifest_line_p1.node == manifest_line_p2.node: file_parents = (manifest_line_p1.node, ) else: if (any(isinstance(p, Mark) for p in git_manifests)): raise Exception( 'Cannot push %s. Please first push %s separately' % (commit, ' and '.join( p for i, p in enumerate(parents) if isinstance(git_manifests[i], Mark)))) if not warned: logging.warning('This may take a while...') warned = True file_parents = (manifest_line_p1.node, manifest_line_p2.node) assert file_parents is not None f = self._create_file_internal( sha1, *file_parents, git_manifest_parents=git_manifests, path=path) file_parents = tuple(p for p in (f.parent1, f.parent2) if p != NULL_NODE_ID) merged = len(file_parents) == 2 if not merged and file_parents: if self.git_file_ref(file_parents[0]) == sha1: node = file_parents[0] else: merged = True if merged: node = self._store_file_internal(f) else: node = PseudoString(file_parents[0]) attr_change = (manifest_line_p1 and manifest_line_p1.attr != attr) manifest.append_line(ManifestLine(path, node, attr), modified=merged or attr_change) if manifest.data == parent_manifest.data: return parent_manifest manifest.set_parents(parent_node, parent2_node) return manifest def process_diff(diff): for (mode_before, mode_after, sha1_before, sha1_after, status, path) in diff: if status[0] == 'R': yield status[1:], ('000000', sha1_before, NULL_NODE_ID, 'D') yield path, (mode_after, sha1_before, sha1_after, status) git_diff = sorted(l for l in process_diff( Git.diff_tree(parents[0], commit, detect_copy=True))) if not git_diff: return parent_manifest parent_lines = OrderedDict((l.name, l) for l in parent_manifest._lines) for line in sorted_merge(parent_lines.iteritems(), git_diff, non_key=lambda i: i[1]): path, manifest_line, change = line if not change: manifest.append_line(manifest_line) continue mode_after, sha1_before, sha1_after, status = change path2 = status[1:] status = status[0] attr = self.ATTR.get(mode_after) if status == 'D': manifest.removed.add(path) continue if status in 'MT': if sha1_before == sha1_after: node = PseudoString(manifest_line.node) else: node = self.create_file( sha1_after, str(manifest_line.node), git_manifest_parents=( self.manifest_ref(parent_node), ), path=path) elif status in 'RC': if sha1_after != EMPTY_BLOB: node = self.create_copy( (path2, parent_lines[path2].node), sha1_after, git_manifest_parents=( self.manifest_ref(parent_node), ), path=path) else: node = self.create_file( sha1_after, git_manifest_parents=( self.manifest_ref(parent_node), ), path=path) else: assert status == 'A' node = self.create_file( sha1_after, git_manifest_parents=(self.manifest_ref(parent_node), ), path=path) manifest.append_line(ManifestLine(path, node, attr), modified=True) manifest.set_parents(parent_node) manifest.delta_node = parent_node return manifest
def getbundle(repo, store, heads, branch_names): if isinstance(repo, bundlerepo): bundle = repo._unbundler else: common = findcommon(repo, store, store.heads(branch_names)) logging.info('common: %s', common) bundle = None got_partial = False if not common: if not store._has_metadata and not store._graft: manifest = Git.config('cinnabar.clone') if not manifest and experiment('git-clone') and \ repo.capable('cinnabarclone'): manifest = repo._call('cinnabarclone') if manifest: got_partial = do_cinnabarclone(repo, manifest, store) if not got_partial: if check_enabled('cinnabarclone'): raise Exception('cinnabarclone failed.') logging.warn('Falling back to normal clone.') if not got_partial and repo.capable('clonebundles'): bundle = get_clonebundle(repo) got_partial = bool(bundle) if not got_partial and check_enabled('clonebundles'): raise Exception('clonebundles failed.') if bundle: bundle = unbundler(bundle) # Manual move semantics apply_bundle = BundleApplier(bundle) del bundle apply_bundle(store) if not changegroup: BundleHelper.close() if got_partial: # Eliminate the heads that we got from the clonebundle or # cinnabarclone. heads = [h for h in heads if not store.changeset_ref(h)] if not heads: return common = findcommon(repo, store, store.heads(branch_names)) logging.info('common: %s', common) kwargs = {} if unbundle20 and repo.capable('bundle2'): bundle2caps = { 'HG20': (), 'changegroup': ('01', '02'), } kwargs['bundlecaps'] = set(( 'HG20', 'bundle2=%s' % urllib.quote(encodecaps(bundle2caps)))) bundle = repo.getbundle('bundle', heads=[unhexlify(h) for h in heads], common=[unhexlify(h) for h in common], **kwargs) bundle = unbundler(bundle) # Manual move semantics apply_bundle = BundleApplier(bundle) del bundle apply_bundle(store)