def test_get_transport_and_path_tcp(self): client, path = get_transport_and_path('git://foo.com/bar/baz') self.assertTrue(isinstance(client, TCPGitClient)) self.assertEquals('foo.com', client._host) self.assertEquals(TCP_GIT_PORT, client._port) self.assertEqual('/bar/baz', path) client, path = get_transport_and_path('git://foo.com:1234/bar/baz') self.assertTrue(isinstance(client, TCPGitClient)) self.assertEquals('foo.com', client._host) self.assertEquals(1234, client._port) self.assertEqual('/bar/baz', path)
def test_get_transport_and_path_ssh_explicit(self): client, path = get_transport_and_path('git+ssh://foo.com/bar/baz') self.assertTrue(isinstance(client, SSHGitClient)) self.assertEquals('foo.com', client.host) self.assertEquals(None, client.port) self.assertEquals(None, client.username) self.assertEqual('/bar/baz', path) client, path = get_transport_and_path('git+ssh://foo.com:1234/bar/baz') self.assertTrue(isinstance(client, SSHGitClient)) self.assertEquals('foo.com', client.host) self.assertEquals(1234, client.port) self.assertEqual('/bar/baz', path)
def push(repo, remote_location, refs_path, config=None, opener=None, outstream=sys.stdout, errstream=sys.stderr): """Remote push with dulwich via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refs_path: relative path to the refs to push to remote :param config: Optional config object :param opener: Custom urllib2 opener for http(s) transport; primarily used for authentication :param outstream: A stream file to write output :param errstream: A stream file to write errors """ # Open the repo r = open_repo(repo) # Get the client and path client, path = get_transport_and_path(remote_location, **{'config': config} if config else {}) if type(client) is HttpGitClient and opener: client.opener = opener def update_refs(refs): new_refs = r.get_refs() refs[refs_path] = new_refs['HEAD'] del new_refs['HEAD'] return refs try: client.send_pack(path, update_refs, r.object_store.generate_pack_contents, progress=errstream.write) outstream.write("Push to %s successful.\n" % remote_location) except (UpdateRefsError, SendPackError) as e: outstream.write("Push to %s failed.\n" % remote_location) errstream.write("Push to %s failed -> '%s'\n" % (remote_location, e.message))
def clone(source, target=None, bare=False, outstream=sys.stdout): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param outstream: Optional stream to write progress to :return: The new repository """ client, host_path = get_transport_and_path(source) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) remote_refs = client.fetch(host_path, r, determine_wants=r.object_store.determine_wants_all, progress=outstream.write) r["HEAD"] = remote_refs["HEAD"] return r
def get_dulwich_client_and_remote_path_from_course(course): ssh_kwargs = {} if course.ssh_private_key: from six import StringIO key_file = StringIO(course.ssh_private_key) ssh_kwargs["pkey"] = paramiko.RSAKey.from_private_key(key_file) def get_dulwich_ssh_vendor(): vendor = DulwichParamikoSSHVendor(ssh_kwargs) return vendor # writing to another module's global variable: gross! import dulwich.client dulwich.client.get_ssh_vendor = get_dulwich_ssh_vendor from dulwich.client import get_transport_and_path client, remote_path = get_transport_and_path( course.git_source) try: # Work around # https://bugs.launchpad.net/dulwich/+bug/1025886 client._fetch_capabilities.remove('thin-pack') except KeyError: pass except AttributeError: pass from dulwich.client import LocalGitClient if not isinstance(client, LocalGitClient): # LocalGitClient uses Py3 Unicode path names to refer to # paths, so it doesn't want an encoded path. remote_path = remote_path.encode("utf-8") return client, remote_path
def push(repo, remote_location, refs_path, outstream=sys.stdout, errstream=sys.stderr): """Remote push with dulwich via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refs_path: relative path to the refs to push to remote :param outstream: A stream file to write output :param errstream: A stream file to write errors """ # Open the repo r = open_repo(repo) # Get the client and path client, path = get_transport_and_path(remote_location) def update_refs(refs): new_refs = r.get_refs() refs[refs_path] = new_refs[b'HEAD'] del new_refs[b'HEAD'] return refs err_encoding = getattr(errstream, 'encoding', 'utf-8') if not isinstance(remote_location, bytes): remote_location_bytes = remote_location.encode(err_encoding) else: remote_location_bytes = remote_location try: client.send_pack(path, update_refs, r.object_store.generate_pack_contents, progress=errstream.write) errstream.write(b"Push to " + remote_location_bytes + b" successful.\n") except (UpdateRefsError, SendPackError) as e: errstream.write(b"Push to " + remote_location_bytes + b" failed -> " + e.message.encode(err_encoding) + b"\n")
def TestRepo(): checkout = Repo.init(tempfile.mkdtemp()) with open(os.path.join(checkout.path, 'foo'), 'w') as fp: fp.write('monty') with open(os.path.join(checkout.path, 'bar'), 'w') as fp: fp.write('python') sub_path = os.path.join(checkout.path, 'sub') os.mkdir(sub_path) with open(os.path.join(sub_path, 'foo'), 'w') as fp: fp.write('sub_monty') with open(os.path.join(sub_path, 'bar'), 'w') as fp: fp.write('sub_python') checkout.stage(['foo', 'bar', os.path.join('sub', 'foo'), os.path.join('sub', 'bar')]) checkout.do_commit( 'The first commit', committer='John Doe <*****@*****.**>' ) bare = Repo.init_bare(tempfile.mkdtemp()) client, host_path = get_transport_and_path(checkout.path) refs = client.fetch( host_path, bare, determine_wants=bare.object_store.determine_wants_all, ) bare["HEAD"] = refs["HEAD"] bare["refs/heads/master"] = refs["refs/heads/master"] return bare, checkout
def test_ssh_explicit(self): c, path = get_transport_and_path('git+ssh://foo.com/bar/baz') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('foo.com', c.host) self.assertEqual(None, c.port) self.assertEqual(None, c.username) self.assertEqual('bar/baz', path)
def test_ssh_port_abspath_explicit(self): c, path = get_transport_and_path( 'git+ssh://foo.com:1234//bar/baz') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('foo.com', c.host) self.assertEqual(1234, c.port) self.assertEqual('/bar/baz', path)
def test_ssh_implicit(self): c, path = get_transport_and_path('foo:/bar/baz') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('foo', c.host) self.assertEqual(None, c.port) self.assertEqual(None, c.username) self.assertEqual('/bar/baz', path)
def test_ssh_user_host_relpath(self): c, path = get_transport_and_path('[email protected]:bar/baz') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('foo.com', c.host) self.assertEqual(None, c.port) self.assertEqual('user', c.username) self.assertEqual('bar/baz', path)
def pull(repo, remote_location, refs_path, config=None, opener=None, outstream=sys.stdout, errstream=sys.stderr): """Pull from remote via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refs_path: relative path to the fetched refs :param config: Optional config object :param opener: Custom urllib2 opener for http(s) transport; primarily used for authentication :param outstream: A stream file to write to output :param errstream: A stream file to write to errors """ # Open the repo r = open_repo(repo) client, path = get_transport_and_path(remote_location, **{'config': config} if config else {}) if type(client) is HttpGitClient and opener: client.opener = opener remote_refs = client.fetch(path, r, progress=errstream.write) r['HEAD'] = remote_refs[refs_path] # Perform 'git checkout .' - syncs staged changes indexfile = r.index_path() tree = r["HEAD"].tree index.build_index_from_tree(r.path, indexfile, r.object_store, tree)
def test_get_transport_and_path_ssh_host_relpath(self): client, path = get_transport_and_path('foo.com:bar/baz') self.assertTrue(isinstance(client, SSHGitClient)) self.assertEqual('foo.com', client.host) self.assertEqual(None, client.port) self.assertEqual(None, client.username) self.assertEqual('bar/baz', path)
def pull(repo, remote_location, refspecs=None, outstream=OUTPUT_STREAM, errstream=ERROR_STREAM): """Pull from remote via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refspec: refspecs to fetch :param outstream: A stream file to write to output :param errstream: A stream file to write to errors """ # Open the repo with open_repo_closing(repo) as r: selected_refs = [] def determine_wants(remote_refs): selected_refs.extend(parse_reftuples(remote_refs, r.refs, refspecs)) return [remote_refs[lh] for (lh, rh, force) in selected_refs] client, path = get_transport_and_path(remote_location) remote_refs = client.fetch(path, r, progress=errstream.write, determine_wants=determine_wants) for (lh, rh, force) in selected_refs: r.refs[rh] = remote_refs[lh] if selected_refs: r[b'HEAD'] = remote_refs[selected_refs[0][1]] # Perform 'git checkout .' - syncs staged changes tree = r[b"HEAD"].tree r.reset_index()
def pull(repo, remote_location=None, refspecs=None, outstream=default_bytes_out_stream, errstream=default_bytes_err_stream): """Pull from remote via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refspec: refspecs to fetch :param outstream: A stream file to write to output :param errstream: A stream file to write to errors """ # Open the repo with open_repo_closing(repo) as r: if remote_location is None: # TODO(jelmer): Lookup 'remote' for current branch in config raise NotImplementedError( "looking up remote from branch config not supported yet") if refspecs is None: refspecs = [b"HEAD"] selected_refs = [] def determine_wants(remote_refs): selected_refs.extend(parse_reftuples(remote_refs, r.refs, refspecs)) return [remote_refs[lh] for (lh, rh, force) in selected_refs] client, path = get_transport_and_path(remote_location) remote_refs = client.fetch(path, r, progress=errstream.write, determine_wants=determine_wants) for (lh, rh, force) in selected_refs: r.refs[rh] = remote_refs[lh] if selected_refs: r[b'HEAD'] = remote_refs[selected_refs[0][1]] # Perform 'git checkout .' - syncs staged changes tree = r[b"HEAD"].tree r.reset_index()
def clone(repo_url, ref=None, folder=None): is_commit = False if ref is None: ref = "refs/heads/master" elif not ref.startswith("refs/"): is_commit = True logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref)) if folder is None: folder = tempfile.mkdtemp() logger.debug("folder = {0}".format(folder)) rep = Repo.init(folder) client, relative_path = get_transport_and_path(repo_url) logger.debug("client={0}".format(client)) remote_refs = client.fetch(relative_path, rep) for k, v in remote_refs.iteritems(): try: rep.refs.add_if_new(k, v) except: pass if is_commit: rep["HEAD"] = rep.commit(ref) else: rep["HEAD"] = remote_refs[ref] indexfile = rep.index_path() tree = rep["HEAD"].tree index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree) logger.debug("done") return rep, folder
def clone(source, target=None, bare=False, checkout=None, outstream=sys.stdout): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param outstream: Optional stream to write progress to :return: The new repository """ if checkout is None: checkout = (not bare) if checkout and bare: raise ValueError("checkout and bare are incompatible") client, host_path = get_transport_and_path(source) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) remote_refs = client.fetch(host_path, r, determine_wants=r.object_store.determine_wants_all, progress=outstream.write) r["HEAD"] = remote_refs["HEAD"] if checkout: outstream.write('Checking out HEAD') index.build_index_from_tree(r.path, r.index_path(), r.object_store, r["HEAD"].tree) return r
def push(repo, remote_location, refs_path, outstream=sys.stdout, errstream=sys.stderr): """Remote push with dulwich via dulwich.client :param repo: Path to repository :param remote_location: Location of the remote :param refs_path: relative path to the refs to push to remote :param outstream: A stream file to write output :param errstream: A stream file to write errors """ # Open the repo r = open_repo(repo) # Get the client and path client, path = get_transport_and_path(remote_location) def update_refs(refs): new_refs = r.get_refs() refs[refs_path] = new_refs['HEAD'] del new_refs['HEAD'] return refs try: client.send_pack(path, update_refs, r.object_store.generate_pack_contents, progress=errstream.write) outstream.write("Push to %s successful.\n" % remote_location) except (UpdateRefsError, SendPackError) as e: outstream.write("Push to %s failed.\n" % remote_location) errstream.write("Push to %s failed -> '%s'\n" % e.message)
def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout, errstream=default_bytes_err_stream, message=None, **kwargs): """Fetch objects from a remote server. :param repo: Path to the repository :param remote_location: String identifying a remote server :param remote_name: Name for remote server :param outstream: Output stream (defaults to stdout) :param errstream: Error stream (defaults to stderr) :param message: Reflog message (defaults to b"fetch: from <remote_name>") :return: Dictionary with refs on the remote """ if message is None: message = b'fetch: from ' + remote_location.encode("utf-8") with open_repo_closing(repo) as r: client, path = get_transport_and_path( remote_location, config=r.get_config_stack(), **kwargs) fetch_result = client.fetch(path, r, progress=errstream.write) stripped_refs = strip_peeled_refs(fetch_result.refs) branches = { n[len(b'refs/heads/'):]: v for (n, v) in stripped_refs.items() if n.startswith(b'refs/heads/')} r.refs.import_refs( b'refs/remotes/' + remote_name, branches, message=message) tags = { n[len(b'refs/tags/'):]: v for (n, v) in stripped_refs.items() if n.startswith(b'refs/tags/') and not n.endswith(ANNOTATED_TAG_SUFFIX)} r.refs.import_refs(b'refs/tags', tags, message=message) return fetch_result.refs
def clone(repo_url, ref=None, folder=None, rep=None): if ref is None: ref = 'refs/heads/master' logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref)) if not rep: if folder is None: folder = tempfile.mkdtemp() else: os.mkdir(folder) logger.debug("folder = {0}".format(folder)) rep = Repo.init(folder) client, relative_path = get_transport_and_path(repo_url) logger.debug("client={0}".format(client)) remote_refs = client.fetch(relative_path, rep) for k, v in remote_refs.iteritems(): try: rep.refs.add_if_new(k, v) except: pass if ref.startswith('refs/tags'): ref = rep.ref(ref) if isinstance(rep[ref], Tag): rep['HEAD'] = rep[ref].object[1] else: rep['HEAD'] = rep[ref] indexfile = rep.index_path() tree = rep["HEAD"].tree index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree) logger.debug("done") return rep, folder
def ls_remote(remote): """List the refs in a remote. :param remote: Remote repository location :return: Dictionary with remote refs """ client, host_path = get_transport_and_path(remote) return client.get_refs(host_path)
def test_username_and_port_explicit(self): c, path = get_transport_and_path( 'ssh://git@server:7999/dply/stuff.git') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('git', c.username) self.assertEqual('server', c.host) self.assertEqual(7999, c.port) self.assertEqual('/dply/stuff.git', path)
def validate_git_url(self, url): error = _("Invalid Git URL: '%s'") % url try: client, path = get_transport_and_path(url) except ValueError: raise forms.ValidationError(error) if isinstance(client, SubprocessGitClient): raise forms.ValidationError(error)
def update_git(repourl, repodir): client, host_path = get_transport_and_path(repourl) # todo: make fetch work if os.path.exists(repodir): shutil.rmtree(repodir) repo = Repo.init(repodir, mkdir=True) remote_refs = client.fetch(host_path, repo, progress=sys.stdout.write) repo["HEAD"] = remote_refs["HEAD"]
def test_http_no_auth(self): url = 'https://github.com/jelmer/dulwich' c, path = get_transport_and_path(url) self.assertTrue(isinstance(c, HttpGitClient)) self.assertEqual('/jelmer/dulwich', path) self.assertIs(None, c._username) self.assertIs(None, c._password)
def test_http_auth_with_username_and_in_url(self): url = 'https://*****:*****@github.com/jelmer/dulwich' c, path = get_transport_and_path( url, username='******', password='******') self.assertTrue(isinstance(c, HttpGitClient)) self.assertEqual('/jelmer/dulwich', path) self.assertEqual('user', c._username) self.assertEqual('passwd', c._password)
def fetch(self, retry=False): def on_error(e): self.source.log.exception('failed fetching repository') with transaction: self.source.last_error = 'failed fetching repository: {}'.format(e) return False old_version = self.source.version try: repo = self.source._open_repo() if repo is None: p = self.source.basepath if not os.path.exists(p): os.makedirs(p) repo = Repo.init_bare(p) client, host_path = get_transport_and_path(self.source.url) remote_refs = client.fetch(host_path, repo) repo["HEAD"] = remote_refs["HEAD"] except (KeyboardInterrupt, SystemExit, gevent.GreenletExit): self.source.unlink() # it is possible that the clone process is broken when the operation was interrupted raise except BaseException as e: if retry: return on_error(e) self.source.log.exception('failed fetching repository; deleting repo') del repo try: really_clean_repo(self.source.basepath) except: m = re.match('^(.+)-tmp(\d+)$', self.source.basepath) if m: basepath = m.group(1) tmp = int(m.group(2)) + 1 else: basepath = self.source.basepath tmp = 1 while True: p = '{}-tmp{}'.format(basepath, tmp) if not os.path.exists(p): break tmp += 1 self.source.log.error('failed deleting broken repo, trying alternative base path {}'.format(p)) self.source.basepath = p return self.fetch(True) #except BaseException as e: # return on_error(e) else: self.source.log.debug('fetch complete; fetched ({})'.format(', '.join(remote_refs))) new_version = self.source.version if old_version == new_version: return False self.source.log.info('updated branch {} from {} to {}'.format(self.source.get_branch(), old_version, new_version)) return True
def ls_remote(remote, config=None, **kwargs): """List the refs in a remote. :param remote: Remote repository location :param config: Configuration to use :return: Dictionary with remote refs """ if config is None: config = StackedConfig.default() client, host_path = get_transport_and_path(remote, config=config, **kwargs) return client.get_refs(host_path)
def cmd_fetch(self, r, url_path): c, path = get_transport_and_path(url_path) c._fetch_capabilities.remove("thin-pack") ### for debugging ### c = client.SubprocessGitClient(thin_packs=False) path = url_path determine_wants = r.object_store.determine_wants_all refs = c.fetch(path, r, progress=self.progress) for k in refs.keys(): if k[-3:] == "^{}": # Annotated tag ref k = k[:-3] r[k] = refs[k]
def copy(self, path, overwrite=False): from dulwich.repo import Repo from dulwich.client import get_transport_and_path r = Repo.init(path) client, host_path = get_transport_and_path(self.url) remote_refs = client.fetch(host_path, r, determine_wants=r.object_store.determine_wants_all, progress=sys.stdout.write) if self.branch: r["HEAD"] = remote_refs[branch] else: r["HEAD"] = remote_refs["HEAD"]
def push_refspec( self, url: str, src: Optional[str], dest: str, force: bool = False, on_diverged: Optional[Callable[[str, str], bool]] = None, ): from dulwich.client import get_transport_and_path from dulwich.errors import NotGitRepository, SendPackError from dulwich.porcelain import ( DivergedBranches, check_diverged, get_remote_repo, ) dest_refs, values = self._push_dest_refs(src, dest) try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location) except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL" ) from exc def update_refs(refs): new_refs = {} for ref, value in zip(dest_refs, values): if ref in refs: local_sha = self.repo.refs[ref] remote_sha = refs[ref] try: check_diverged(self.repo, remote_sha, local_sha) except DivergedBranches: if not force: overwrite = False if on_diverged: overwrite = on_diverged( os.fsdecode(ref), os.fsdecode(remote_sha), ) if not overwrite: continue new_refs[ref] = value return new_refs try: with Tqdm( desc="Pushing git refs", bar_format=self.BAR_FMT_NOTOTAL ) as pbar: def progress(msg_b): msg = msg_b.decode("ascii").strip() pbar.update_msg(msg) pbar.refresh() logger.trace(msg) client.send_pack( path, update_refs, self.repo.object_store.generate_pack_data, progress=progress, ) except (NotGitRepository, SendPackError) as exc: raise SCMError("Git failed to push '{src}' to '{url}'") from exc
def fetch_refspecs( self, url: str, refspecs: Iterable[str], force: Optional[bool] = False, on_diverged: Optional[Callable[[str, str], bool]] = None, ): from dulwich.client import get_transport_and_path from dulwich.objectspec import parse_reftuples from dulwich.porcelain import ( DivergedBranches, check_diverged, get_remote_repo, ) fetch_refs = [] def determine_wants(remote_refs): fetch_refs.extend( parse_reftuples( remote_refs, self.repo.refs, [os.fsencode(refspec) for refspec in refspecs], force=force, ) ) return [ remote_refs[lh] for (lh, _, _) in fetch_refs if remote_refs[lh] not in self.repo.object_store ] try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location) except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL" ) from exc with Tqdm( desc="Fetching git refs", bar_format=self.BAR_FMT_NOTOTAL ) as pbar: def progress(msg_b): msg = msg_b.decode("ascii").strip() pbar.update_msg(msg) pbar.refresh() logger.trace(msg) fetch_result = client.fetch( path, self.repo, progress=progress, determine_wants=determine_wants, ) for (lh, rh, _) in fetch_refs: try: if rh in self.repo.refs: check_diverged( self.repo, self.repo.refs[rh], fetch_result.refs[lh] ) except DivergedBranches: if not force: overwrite = False if on_diverged: overwrite = on_diverged( os.fsdecode(rh), os.fsdecode(fetch_result.refs[lh]) ) if not overwrite: continue self.repo.refs[rh] = fetch_result.refs[lh]
def test_get_transport_and_path_tcp(self): client, path = get_transport_and_path('git://foo.com/bar/baz') self.assertTrue(isinstance(client, TCPGitClient)) self.assertEqual('foo.com', client._host) self.assertEqual(TCP_GIT_PORT, client._port) self.assertEqual('/bar/baz', path)
def ls_remote(remote): client, host_path = get_transport_and_path(remote) return client.get_refs(encode_path(host_path))
def test_ssh_port_abspath_explicit(self): c, path = get_transport_and_path('git+ssh://foo.com:1234//bar/baz') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('foo.com', c.host) self.assertEqual(1234, c.port) self.assertEqual('/bar/baz', path)
def test_local(self): c, path = get_transport_and_path('foo.bar/baz') self.assertTrue(isinstance(c, LocalGitClient)) self.assertEqual('foo.bar/baz', path)
def test_local_abs_windows_path(self): c, path = get_transport_and_path('C:\\foo.bar\\baz') self.assertTrue(isinstance(c, LocalGitClient)) self.assertEqual('C:\\foo.bar\\baz', path)
def test_error(self): # Need to use a known urlparse.uses_netloc URL scheme to get the # expected parsing of the URL on Python versions less than 2.6.5 c, path = get_transport_and_path('prospero://bar/baz') self.assertTrue(isinstance(c, SSHGitClient))
def clone(source, target=None, bare=False, checkout=None, errstream=default_bytes_err_stream, outstream=None, origin=b"origin"): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param checkout: Whether or not to check-out HEAD after cloning :param errstream: Optional stream to write progress to :param outstream: Optional stream to write progress to (deprecated) :return: The new repository """ if outstream is not None: import warnings warnings.warn( "outstream= has been deprecated in favour of errstream=.", DeprecationWarning, stacklevel=3) errstream = outstream if checkout is None: checkout = (not bare) if checkout and bare: raise ValueError("checkout and bare are incompatible") client, host_path = get_transport_and_path(source) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) try: remote_refs = client.fetch( host_path, r, determine_wants=r.object_store.determine_wants_all, progress=errstream.write) r.refs.import_refs( b'refs/remotes/' + origin, { n[len(b'refs/heads/'):]: v for (n, v) in remote_refs.items() if n.startswith(b'refs/heads/') }) r.refs.import_refs( b'refs/tags', { n[len(b'refs/tags/'):]: v for (n, v) in remote_refs.items() if n.startswith(b'refs/tags/') and not n.endswith(ANNOTATED_TAG_SUFFIX) }) if b"HEAD" in remote_refs and not bare: # TODO(jelmer): Support symref capability, # https://github.com/jelmer/dulwich/issues/485 r[b"HEAD"] = remote_refs[b"HEAD"] target_config = r.get_config() if not isinstance(source, bytes): source = source.encode(DEFAULT_ENCODING) target_config.set((b'remote', b'origin'), b'url', source) target_config.set((b'remote', b'origin'), b'fetch', b'+refs/heads/*:refs/remotes/origin/*') target_config.write_to_path() if checkout and b"HEAD" in r.refs: errstream.write(b'Checking out HEAD\n') r.reset_index() except: r.close() raise return r
def test_http(self): url = 'https://github.com/jelmer/dulwich' c, path = get_transport_and_path(url) self.assertTrue(isinstance(c, HttpGitClient)) self.assertEqual('/jelmer/dulwich', path)
def test_username_and_port_explicit_unknown_scheme(self): c, path = get_transport_and_path( 'unknown://git@server:7999/dply/stuff.git') self.assertTrue(isinstance(c, SSHGitClient)) self.assertEqual('unknown', c.host) self.assertEqual('//git@server:7999/dply/stuff.git', path)
def fetch_refspecs( self, url: str, refspecs: Iterable[str], force: Optional[bool] = False, on_diverged: Optional[Callable[[str, str], bool]] = None, progress: Callable[["GitProgressEvent"], None] = None, **kwargs, ): from dulwich.client import get_transport_and_path from dulwich.objectspec import parse_reftuples from dulwich.porcelain import ( DivergedBranches, check_diverged, get_remote_repo, ) fetch_refs = [] def determine_wants(remote_refs): fetch_refs.extend( parse_reftuples( remote_refs, self.repo.refs, [os.fsencode(refspec) for refspec in refspecs], force=force, )) return [ remote_refs[lh] for (lh, _, _) in fetch_refs if remote_refs[lh] not in self.repo.object_store ] try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL") from exc from dvc.scm.progress import GitProgressReporter fetch_result = client.fetch( path, self.repo, progress=GitProgressReporter(progress) if progress else None, determine_wants=determine_wants, ) for (lh, rh, _) in fetch_refs: try: if rh in self.repo.refs: check_diverged(self.repo, self.repo.refs[rh], fetch_result.refs[lh]) except DivergedBranches: if not force: overwrite = False if on_diverged: overwrite = on_diverged( os.fsdecode(rh), os.fsdecode(fetch_result.refs[lh])) if not overwrite: continue self.repo.refs[rh] = fetch_result.refs[lh]
def test_get_transport_and_path_subprocess(self): client, path = get_transport_and_path('foo.bar/baz') self.assertTrue(isinstance(client, SubprocessGitClient)) self.assertEqual('foo.bar/baz', path)
def test_tcp_port(self): c, path = get_transport_and_path('git://foo.com:1234/bar/baz') self.assertTrue(isinstance(c, TCPGitClient)) self.assertEqual('foo.com', c._host) self.assertEqual(1234, c._port) self.assertEqual('/bar/baz', path)
def clone(source, target=None, bare=False, checkout=None, errstream=default_bytes_err_stream, outstream=None, origin=b"origin", **kwargs): """Clone a local or remote git repository. :param source: Path or URL for source repository :param target: Path to target repository (optional) :param bare: Whether or not to create a bare repository :param checkout: Whether or not to check-out HEAD after cloning :param errstream: Optional stream to write progress to :param outstream: Optional stream to write progress to (deprecated) :param origin: Name of remote from the repository used to clone :return: The new repository """ # TODO(jelmer): This code overlaps quite a bit with Repo.clone if outstream is not None: import warnings warnings.warn( "outstream= has been deprecated in favour of errstream=.", DeprecationWarning, stacklevel=3) errstream = outstream if checkout is None: checkout = (not bare) if checkout and bare: raise ValueError("checkout and bare are incompatible") config = StackedConfig.default() client, host_path = get_transport_and_path(source, config=config, **kwargs) if target is None: target = host_path.split("/")[-1] if not os.path.exists(target): os.mkdir(target) if bare: r = Repo.init_bare(target) else: r = Repo.init(target) reflog_message = b'clone: from ' + source.encode('utf-8') try: fetch_result = fetch(r, host_path, origin, message=reflog_message) target_config = r.get_config() if not isinstance(source, bytes): source = source.encode(DEFAULT_ENCODING) target_config.set((b'remote', origin), b'url', source) target_config.set((b'remote', origin), b'fetch', b'+refs/heads/*:refs/remotes/' + origin + b'/*') target_config.write_to_path() # TODO(jelmer): Support symref capability, # https://github.com/jelmer/dulwich/issues/485 try: head = r[fetch_result[b'HEAD']] except KeyError: head = None else: r[b'HEAD'] = head.id if checkout and not bare and head is not None: errstream.write(b'Checking out ' + head.id + b'\n') r.reset_index(head.tree) except BaseException: r.close() raise return r
def push_refspec( self, url: str, src: Optional[str], dest: str, force: bool = False, on_diverged: Optional[Callable[[str, str], bool]] = None, progress: Callable[["GitProgressEvent"], None] = None, **kwargs, ): from dulwich.client import HTTPUnauthorized, get_transport_and_path from dulwich.errors import NotGitRepository, SendPackError from dulwich.porcelain import ( DivergedBranches, check_diverged, get_remote_repo, ) dest_refs, values = self._push_dest_refs(src, dest) try: _remote, location = get_remote_repo(self.repo, url) client, path = get_transport_and_path(location, **kwargs) except Exception as exc: raise SCMError( f"'{url}' is not a valid Git remote or URL") from exc def update_refs(refs): from dulwich.objects import ZERO_SHA new_refs = {} for ref, value in zip(dest_refs, values): if ref in refs and value != ZERO_SHA: local_sha = self.repo.refs[ref] remote_sha = refs[ref] try: check_diverged(self.repo, remote_sha, local_sha) except DivergedBranches: if not force: overwrite = False if on_diverged: overwrite = on_diverged( os.fsdecode(ref), os.fsdecode(remote_sha)) if not overwrite: continue new_refs[ref] = value return new_refs try: from dvc.scm.progress import GitProgressReporter client.send_pack( path, update_refs, self.repo.object_store.generate_pack_data, progress=GitProgressReporter(progress) if progress else None, ) except (NotGitRepository, SendPackError) as exc: raise SCMError("Git failed to push '{src}' to '{url}'") from exc except HTTPUnauthorized: raise AuthError(url)