def test_get_transport_and_path_tcp(self):
client, path = get_transport_and_path('git://foo.com/bar/baz')
self.assertTrue(isinstance(client, TCPGitClient))
self.assertEquals('foo.com', client._host)
self.assertEquals(TCP_GIT_PORT, client._port)
self.assertEqual('/bar/baz', path)
client, path = get_transport_and_path('git://foo.com:1234/bar/baz')
self.assertTrue(isinstance(client, TCPGitClient))
self.assertEquals('foo.com', client._host)
self.assertEquals(1234, client._port)
self.assertEqual('/bar/baz', path)
def test_get_transport_and_path_ssh_explicit(self):
client, path = get_transport_and_path('git+ssh://foo.com/bar/baz')
self.assertTrue(isinstance(client, SSHGitClient))
self.assertEquals('foo.com', client.host)
self.assertEquals(None, client.port)
self.assertEquals(None, client.username)
self.assertEqual('/bar/baz', path)
client, path = get_transport_and_path('git+ssh://foo.com:1234/bar/baz')
self.assertTrue(isinstance(client, SSHGitClient))
self.assertEquals('foo.com', client.host)
self.assertEquals(1234, client.port)
self.assertEqual('/bar/baz', path)
def push(repo, remote_location, refs_path, config=None, opener=None,
outstream=sys.stdout, errstream=sys.stderr):
"""Remote push with dulwich via dulwich.client
:param repo: Path to repository
:param remote_location: Location of the remote
:param refs_path: relative path to the refs to push to remote
:param config: Optional config object
:param opener: Custom urllib2 opener for http(s) transport; primarily used for authentication
:param outstream: A stream file to write output
:param errstream: A stream file to write errors
"""
# Open the repo
r = open_repo(repo)
# Get the client and path
client, path = get_transport_and_path(remote_location, **{'config': config} if config else {})
if type(client) is HttpGitClient and opener:
client.opener = opener
def update_refs(refs):
new_refs = r.get_refs()
refs[refs_path] = new_refs['HEAD']
del new_refs['HEAD']
return refs
try:
client.send_pack(path, update_refs,
r.object_store.generate_pack_contents, progress=errstream.write)
outstream.write("Push to %s successful.\n" % remote_location)
except (UpdateRefsError, SendPackError) as e:
outstream.write("Push to %s failed.\n" % remote_location)
errstream.write("Push to %s failed -> '%s'\n" % (remote_location, e.message))
def clone(source, target=None, bare=False, outstream=sys.stdout):
"""Clone a local or remote git repository.
:param source: Path or URL for source repository
:param target: Path to target repository (optional)
:param bare: Whether or not to create a bare repository
:param outstream: Optional stream to write progress to
:return: The new repository
"""
client, host_path = get_transport_and_path(source)
if target is None:
target = host_path.split("/")[-1]
if not os.path.exists(target):
os.mkdir(target)
if bare:
r = Repo.init_bare(target)
else:
r = Repo.init(target)
remote_refs = client.fetch(host_path, r,
determine_wants=r.object_store.determine_wants_all,
progress=outstream.write)
r["HEAD"] = remote_refs["HEAD"]
return r
def get_dulwich_client_and_remote_path_from_course(course):
ssh_kwargs = {}
if course.ssh_private_key:
from six import StringIO
key_file = StringIO(course.ssh_private_key)
ssh_kwargs["pkey"] = paramiko.RSAKey.from_private_key(key_file)
def get_dulwich_ssh_vendor():
vendor = DulwichParamikoSSHVendor(ssh_kwargs)
return vendor
# writing to another module's global variable: gross!
import dulwich.client
dulwich.client.get_ssh_vendor = get_dulwich_ssh_vendor
from dulwich.client import get_transport_and_path
client, remote_path = get_transport_and_path(
course.git_source)
try:
# Work around
# https://bugs.launchpad.net/dulwich/+bug/1025886
client._fetch_capabilities.remove('thin-pack')
except KeyError:
pass
except AttributeError:
pass
from dulwich.client import LocalGitClient
if not isinstance(client, LocalGitClient):
# LocalGitClient uses Py3 Unicode path names to refer to
# paths, so it doesn't want an encoded path.
remote_path = remote_path.encode("utf-8")
return client, remote_path
def push(repo, remote_location, refs_path,
outstream=sys.stdout, errstream=sys.stderr):
"""Remote push with dulwich via dulwich.client
:param repo: Path to repository
:param remote_location: Location of the remote
:param refs_path: relative path to the refs to push to remote
:param outstream: A stream file to write output
:param errstream: A stream file to write errors
"""
# Open the repo
r = open_repo(repo)
# Get the client and path
client, path = get_transport_and_path(remote_location)
def update_refs(refs):
new_refs = r.get_refs()
refs[refs_path] = new_refs[b'HEAD']
del new_refs[b'HEAD']
return refs
err_encoding = getattr(errstream, 'encoding', 'utf-8')
if not isinstance(remote_location, bytes):
remote_location_bytes = remote_location.encode(err_encoding)
else:
remote_location_bytes = remote_location
try:
client.send_pack(path, update_refs,
r.object_store.generate_pack_contents, progress=errstream.write)
errstream.write(b"Push to " + remote_location_bytes + b" successful.\n")
except (UpdateRefsError, SendPackError) as e:
errstream.write(b"Push to " + remote_location_bytes + b" failed -> " + e.message.encode(err_encoding) + b"\n")
def TestRepo():
checkout = Repo.init(tempfile.mkdtemp())
with open(os.path.join(checkout.path, 'foo'), 'w') as fp:
fp.write('monty')
with open(os.path.join(checkout.path, 'bar'), 'w') as fp:
fp.write('python')
sub_path = os.path.join(checkout.path, 'sub')
os.mkdir(sub_path)
with open(os.path.join(sub_path, 'foo'), 'w') as fp:
fp.write('sub_monty')
with open(os.path.join(sub_path, 'bar'), 'w') as fp:
fp.write('sub_python')
checkout.stage(['foo', 'bar',
os.path.join('sub', 'foo'),
os.path.join('sub', 'bar')])
checkout.do_commit(
'The first commit',
committer='John Doe <*****@*****.**>'
)
bare = Repo.init_bare(tempfile.mkdtemp())
client, host_path = get_transport_and_path(checkout.path)
refs = client.fetch(
host_path, bare,
determine_wants=bare.object_store.determine_wants_all,
)
bare["HEAD"] = refs["HEAD"]
bare["refs/heads/master"] = refs["refs/heads/master"]
return bare, checkout
def test_ssh_explicit(self):
c, path = get_transport_and_path('git+ssh://foo.com/bar/baz')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('foo.com', c.host)
self.assertEqual(None, c.port)
self.assertEqual(None, c.username)
self.assertEqual('bar/baz', path)
def test_ssh_port_abspath_explicit(self):
c, path = get_transport_and_path(
'git+ssh://foo.com:1234//bar/baz')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('foo.com', c.host)
self.assertEqual(1234, c.port)
self.assertEqual('/bar/baz', path)
def test_ssh_implicit(self):
c, path = get_transport_and_path('foo:/bar/baz')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('foo', c.host)
self.assertEqual(None, c.port)
self.assertEqual(None, c.username)
self.assertEqual('/bar/baz', path)
def test_ssh_user_host_relpath(self):
c, path = get_transport_and_path('[email protected]:bar/baz')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('foo.com', c.host)
self.assertEqual(None, c.port)
self.assertEqual('user', c.username)
self.assertEqual('bar/baz', path)
def pull(repo, remote_location, refs_path, config=None, opener=None,
outstream=sys.stdout, errstream=sys.stderr):
"""Pull from remote via dulwich.client
:param repo: Path to repository
:param remote_location: Location of the remote
:param refs_path: relative path to the fetched refs
:param config: Optional config object
:param opener: Custom urllib2 opener for http(s) transport; primarily used for authentication
:param outstream: A stream file to write to output
:param errstream: A stream file to write to errors
"""
# Open the repo
r = open_repo(repo)
client, path = get_transport_and_path(remote_location, **{'config': config} if config else {})
if type(client) is HttpGitClient and opener:
client.opener = opener
remote_refs = client.fetch(path, r, progress=errstream.write)
r['HEAD'] = remote_refs[refs_path]
# Perform 'git checkout .' - syncs staged changes
indexfile = r.index_path()
tree = r["HEAD"].tree
index.build_index_from_tree(r.path, indexfile, r.object_store, tree)
def test_get_transport_and_path_ssh_host_relpath(self):
client, path = get_transport_and_path('foo.com:bar/baz')
self.assertTrue(isinstance(client, SSHGitClient))
self.assertEqual('foo.com', client.host)
self.assertEqual(None, client.port)
self.assertEqual(None, client.username)
self.assertEqual('bar/baz', path)
def pull(repo, remote_location, refspecs=None,
outstream=OUTPUT_STREAM, errstream=ERROR_STREAM):
"""Pull from remote via dulwich.client
:param repo: Path to repository
:param remote_location: Location of the remote
:param refspec: refspecs to fetch
:param outstream: A stream file to write to output
:param errstream: A stream file to write to errors
"""
# Open the repo
with open_repo_closing(repo) as r:
selected_refs = []
def determine_wants(remote_refs):
selected_refs.extend(parse_reftuples(remote_refs, r.refs, refspecs))
return [remote_refs[lh] for (lh, rh, force) in selected_refs]
client, path = get_transport_and_path(remote_location)
remote_refs = client.fetch(path, r, progress=errstream.write,
determine_wants=determine_wants)
for (lh, rh, force) in selected_refs:
r.refs[rh] = remote_refs[lh]
if selected_refs:
r[b'HEAD'] = remote_refs[selected_refs[0][1]]
# Perform 'git checkout .' - syncs staged changes
tree = r[b"HEAD"].tree
r.reset_index()
def pull(repo, remote_location=None, refspecs=None,
outstream=default_bytes_out_stream, errstream=default_bytes_err_stream):
"""Pull from remote via dulwich.client
:param repo: Path to repository
:param remote_location: Location of the remote
:param refspec: refspecs to fetch
:param outstream: A stream file to write to output
:param errstream: A stream file to write to errors
"""
# Open the repo
with open_repo_closing(repo) as r:
if remote_location is None:
# TODO(jelmer): Lookup 'remote' for current branch in config
raise NotImplementedError(
"looking up remote from branch config not supported yet")
if refspecs is None:
refspecs = [b"HEAD"]
selected_refs = []
def determine_wants(remote_refs):
selected_refs.extend(parse_reftuples(remote_refs, r.refs, refspecs))
return [remote_refs[lh] for (lh, rh, force) in selected_refs]
client, path = get_transport_and_path(remote_location)
remote_refs = client.fetch(path, r, progress=errstream.write,
determine_wants=determine_wants)
for (lh, rh, force) in selected_refs:
r.refs[rh] = remote_refs[lh]
if selected_refs:
r[b'HEAD'] = remote_refs[selected_refs[0][1]]
# Perform 'git checkout .' - syncs staged changes
tree = r[b"HEAD"].tree
r.reset_index()
def clone(repo_url, ref=None, folder=None):
is_commit = False
if ref is None:
ref = "refs/heads/master"
elif not ref.startswith("refs/"):
is_commit = True
logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref))
if folder is None:
folder = tempfile.mkdtemp()
logger.debug("folder = {0}".format(folder))
rep = Repo.init(folder)
client, relative_path = get_transport_and_path(repo_url)
logger.debug("client={0}".format(client))
remote_refs = client.fetch(relative_path, rep)
for k, v in remote_refs.iteritems():
try:
rep.refs.add_if_new(k, v)
except:
pass
if is_commit:
rep["HEAD"] = rep.commit(ref)
else:
rep["HEAD"] = remote_refs[ref]
indexfile = rep.index_path()
tree = rep["HEAD"].tree
index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
logger.debug("done")
return rep, folder
def clone(source, target=None, bare=False, checkout=None, outstream=sys.stdout):
"""Clone a local or remote git repository.
:param source: Path or URL for source repository
:param target: Path to target repository (optional)
:param bare: Whether or not to create a bare repository
:param outstream: Optional stream to write progress to
:return: The new repository
"""
if checkout is None:
checkout = (not bare)
if checkout and bare:
raise ValueError("checkout and bare are incompatible")
client, host_path = get_transport_and_path(source)
if target is None:
target = host_path.split("/")[-1]
if not os.path.exists(target):
os.mkdir(target)
if bare:
r = Repo.init_bare(target)
else:
r = Repo.init(target)
remote_refs = client.fetch(host_path, r,
determine_wants=r.object_store.determine_wants_all,
progress=outstream.write)
r["HEAD"] = remote_refs["HEAD"]
if checkout:
outstream.write('Checking out HEAD')
index.build_index_from_tree(r.path, r.index_path(),
r.object_store, r["HEAD"].tree)
return r
def push(repo, remote_location, refs_path,
outstream=sys.stdout, errstream=sys.stderr):
"""Remote push with dulwich via dulwich.client
:param repo: Path to repository
:param remote_location: Location of the remote
:param refs_path: relative path to the refs to push to remote
:param outstream: A stream file to write output
:param errstream: A stream file to write errors
"""
# Open the repo
r = open_repo(repo)
# Get the client and path
client, path = get_transport_and_path(remote_location)
def update_refs(refs):
new_refs = r.get_refs()
refs[refs_path] = new_refs['HEAD']
del new_refs['HEAD']
return refs
try:
client.send_pack(path, update_refs,
r.object_store.generate_pack_contents, progress=errstream.write)
outstream.write("Push to %s successful.\n" % remote_location)
except (UpdateRefsError, SendPackError) as e:
outstream.write("Push to %s failed.\n" % remote_location)
errstream.write("Push to %s failed -> '%s'\n" % e.message)
def fetch(repo, remote_location, remote_name=b'origin', outstream=sys.stdout,
errstream=default_bytes_err_stream, message=None, **kwargs):
"""Fetch objects from a remote server.
:param repo: Path to the repository
:param remote_location: String identifying a remote server
:param remote_name: Name for remote server
:param outstream: Output stream (defaults to stdout)
:param errstream: Error stream (defaults to stderr)
:param message: Reflog message (defaults to b"fetch: from <remote_name>")
:return: Dictionary with refs on the remote
"""
if message is None:
message = b'fetch: from ' + remote_location.encode("utf-8")
with open_repo_closing(repo) as r:
client, path = get_transport_and_path(
remote_location, config=r.get_config_stack(), **kwargs)
fetch_result = client.fetch(path, r, progress=errstream.write)
stripped_refs = strip_peeled_refs(fetch_result.refs)
branches = {
n[len(b'refs/heads/'):]: v for (n, v) in stripped_refs.items()
if n.startswith(b'refs/heads/')}
r.refs.import_refs(
b'refs/remotes/' + remote_name, branches, message=message)
tags = {
n[len(b'refs/tags/'):]: v for (n, v) in stripped_refs.items()
if n.startswith(b'refs/tags/') and
not n.endswith(ANNOTATED_TAG_SUFFIX)}
r.refs.import_refs(b'refs/tags', tags, message=message)
return fetch_result.refs
def clone(repo_url, ref=None, folder=None, rep=None):
if ref is None:
ref = 'refs/heads/master'
logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref))
if not rep:
if folder is None:
folder = tempfile.mkdtemp()
else:
os.mkdir(folder)
logger.debug("folder = {0}".format(folder))
rep = Repo.init(folder)
client, relative_path = get_transport_and_path(repo_url)
logger.debug("client={0}".format(client))
remote_refs = client.fetch(relative_path, rep)
for k, v in remote_refs.iteritems():
try:
rep.refs.add_if_new(k, v)
except:
pass
if ref.startswith('refs/tags'):
ref = rep.ref(ref)
if isinstance(rep[ref], Tag):
rep['HEAD'] = rep[ref].object[1]
else:
rep['HEAD'] = rep[ref]
indexfile = rep.index_path()
tree = rep["HEAD"].tree
index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree)
logger.debug("done")
return rep, folder
def ls_remote(remote):
"""List the refs in a remote.
:param remote: Remote repository location
:return: Dictionary with remote refs
"""
client, host_path = get_transport_and_path(remote)
return client.get_refs(host_path)
def test_username_and_port_explicit(self):
c, path = get_transport_and_path(
'ssh://git@server:7999/dply/stuff.git')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('git', c.username)
self.assertEqual('server', c.host)
self.assertEqual(7999, c.port)
self.assertEqual('/dply/stuff.git', path)
def validate_git_url(self, url):
error = _("Invalid Git URL: '%s'") % url
try:
client, path = get_transport_and_path(url)
except ValueError:
raise forms.ValidationError(error)
if isinstance(client, SubprocessGitClient):
raise forms.ValidationError(error)
def update_git(repourl, repodir):
client, host_path = get_transport_and_path(repourl)
# todo: make fetch work
if os.path.exists(repodir):
shutil.rmtree(repodir)
repo = Repo.init(repodir, mkdir=True)
remote_refs = client.fetch(host_path, repo,
progress=sys.stdout.write)
repo["HEAD"] = remote_refs["HEAD"]
def test_http_no_auth(self):
url = 'https://github.com/jelmer/dulwich'
c, path = get_transport_and_path(url)
self.assertTrue(isinstance(c, HttpGitClient))
self.assertEqual('/jelmer/dulwich', path)
self.assertIs(None, c._username)
self.assertIs(None, c._password)
def test_http_auth_with_username_and_in_url(self):
url = 'https://*****:*****@github.com/jelmer/dulwich'
c, path = get_transport_and_path(
url, username='******', password='******')
self.assertTrue(isinstance(c, HttpGitClient))
self.assertEqual('/jelmer/dulwich', path)
self.assertEqual('user', c._username)
self.assertEqual('passwd', c._password)
def fetch(self, retry=False):
def on_error(e):
self.source.log.exception('failed fetching repository')
with transaction:
self.source.last_error = 'failed fetching repository: {}'.format(e)
return False
old_version = self.source.version
try:
repo = self.source._open_repo()
if repo is None:
p = self.source.basepath
if not os.path.exists(p):
os.makedirs(p)
repo = Repo.init_bare(p)
client, host_path = get_transport_and_path(self.source.url)
remote_refs = client.fetch(host_path, repo)
repo["HEAD"] = remote_refs["HEAD"]
except (KeyboardInterrupt, SystemExit, gevent.GreenletExit):
self.source.unlink() # it is possible that the clone process is broken when the operation was interrupted
raise
except BaseException as e:
if retry:
return on_error(e)
self.source.log.exception('failed fetching repository; deleting repo')
del repo
try:
really_clean_repo(self.source.basepath)
except:
m = re.match('^(.+)-tmp(\d+)$', self.source.basepath)
if m:
basepath = m.group(1)
tmp = int(m.group(2)) + 1
else:
basepath = self.source.basepath
tmp = 1
while True:
p = '{}-tmp{}'.format(basepath, tmp)
if not os.path.exists(p):
break
tmp += 1
self.source.log.error('failed deleting broken repo, trying alternative base path {}'.format(p))
self.source.basepath = p
return self.fetch(True)
#except BaseException as e:
# return on_error(e)
else:
self.source.log.debug('fetch complete; fetched ({})'.format(', '.join(remote_refs)))
new_version = self.source.version
if old_version == new_version:
return False
self.source.log.info('updated branch {} from {} to {}'.format(self.source.get_branch(), old_version, new_version))
return True
def ls_remote(remote, config=None, **kwargs):
"""List the refs in a remote.
:param remote: Remote repository location
:param config: Configuration to use
:return: Dictionary with remote refs
"""
if config is None:
config = StackedConfig.default()
client, host_path = get_transport_and_path(remote, config=config, **kwargs)
return client.get_refs(host_path)
def cmd_fetch(self, r, url_path):
c, path = get_transport_and_path(url_path)
c._fetch_capabilities.remove("thin-pack")
### for debugging ### c = client.SubprocessGitClient(thin_packs=False)
path = url_path
determine_wants = r.object_store.determine_wants_all
refs = c.fetch(path, r, progress=self.progress)
for k in refs.keys():
if k[-3:] == "^{}": # Annotated tag ref
k = k[:-3]
r[k] = refs[k]
def copy(self, path, overwrite=False):
from dulwich.repo import Repo
from dulwich.client import get_transport_and_path
r = Repo.init(path)
client, host_path = get_transport_and_path(self.url)
remote_refs = client.fetch(host_path, r,
determine_wants=r.object_store.determine_wants_all,
progress=sys.stdout.write)
if self.branch:
r["HEAD"] = remote_refs[branch]
else:
r["HEAD"] = remote_refs["HEAD"]
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
):
from dulwich.client import get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL"
) from exc
def update_refs(refs):
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha),
)
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
with Tqdm(
desc="Pushing git refs", bar_format=self.BAR_FMT_NOTOTAL
) as pbar:
def progress(msg_b):
msg = msg_b.decode("ascii").strip()
pbar.update_msg(msg)
pbar.refresh()
logger.trace(msg)
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=progress,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
def fetch_refspecs(
self,
url: str,
refspecs: Iterable[str],
force: Optional[bool] = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
):
from dulwich.client import get_transport_and_path
from dulwich.objectspec import parse_reftuples
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
fetch_refs = []
def determine_wants(remote_refs):
fetch_refs.extend(
parse_reftuples(
remote_refs,
self.repo.refs,
[os.fsencode(refspec) for refspec in refspecs],
force=force,
)
)
return [
remote_refs[lh]
for (lh, _, _) in fetch_refs
if remote_refs[lh] not in self.repo.object_store
]
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL"
) from exc
with Tqdm(
desc="Fetching git refs", bar_format=self.BAR_FMT_NOTOTAL
) as pbar:
def progress(msg_b):
msg = msg_b.decode("ascii").strip()
pbar.update_msg(msg)
pbar.refresh()
logger.trace(msg)
fetch_result = client.fetch(
path,
self.repo,
progress=progress,
determine_wants=determine_wants,
)
for (lh, rh, _) in fetch_refs:
try:
if rh in self.repo.refs:
check_diverged(
self.repo, self.repo.refs[rh], fetch_result.refs[lh]
)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(rh), os.fsdecode(fetch_result.refs[lh])
)
if not overwrite:
continue
self.repo.refs[rh] = fetch_result.refs[lh]
def test_get_transport_and_path_tcp(self):
client, path = get_transport_and_path('git://foo.com/bar/baz')
self.assertTrue(isinstance(client, TCPGitClient))
self.assertEqual('foo.com', client._host)
self.assertEqual(TCP_GIT_PORT, client._port)
self.assertEqual('/bar/baz', path)
def ls_remote(remote):
client, host_path = get_transport_and_path(remote)
return client.get_refs(encode_path(host_path))
def test_ssh_port_abspath_explicit(self):
c, path = get_transport_and_path('git+ssh://foo.com:1234//bar/baz')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('foo.com', c.host)
self.assertEqual(1234, c.port)
self.assertEqual('/bar/baz', path)
def test_local(self):
c, path = get_transport_and_path('foo.bar/baz')
self.assertTrue(isinstance(c, LocalGitClient))
self.assertEqual('foo.bar/baz', path)
def test_local_abs_windows_path(self):
c, path = get_transport_and_path('C:\\foo.bar\\baz')
self.assertTrue(isinstance(c, LocalGitClient))
self.assertEqual('C:\\foo.bar\\baz', path)
def test_error(self):
# Need to use a known urlparse.uses_netloc URL scheme to get the
# expected parsing of the URL on Python versions less than 2.6.5
c, path = get_transport_and_path('prospero://bar/baz')
self.assertTrue(isinstance(c, SSHGitClient))
def clone(source,
target=None,
bare=False,
checkout=None,
errstream=default_bytes_err_stream,
outstream=None,
origin=b"origin"):
"""Clone a local or remote git repository.
:param source: Path or URL for source repository
:param target: Path to target repository (optional)
:param bare: Whether or not to create a bare repository
:param checkout: Whether or not to check-out HEAD after cloning
:param errstream: Optional stream to write progress to
:param outstream: Optional stream to write progress to (deprecated)
:return: The new repository
"""
if outstream is not None:
import warnings
warnings.warn(
"outstream= has been deprecated in favour of errstream=.",
DeprecationWarning,
stacklevel=3)
errstream = outstream
if checkout is None:
checkout = (not bare)
if checkout and bare:
raise ValueError("checkout and bare are incompatible")
client, host_path = get_transport_and_path(source)
if target is None:
target = host_path.split("/")[-1]
if not os.path.exists(target):
os.mkdir(target)
if bare:
r = Repo.init_bare(target)
else:
r = Repo.init(target)
try:
remote_refs = client.fetch(
host_path,
r,
determine_wants=r.object_store.determine_wants_all,
progress=errstream.write)
r.refs.import_refs(
b'refs/remotes/' + origin, {
n[len(b'refs/heads/'):]: v
for (n, v) in remote_refs.items()
if n.startswith(b'refs/heads/')
})
r.refs.import_refs(
b'refs/tags', {
n[len(b'refs/tags/'):]: v
for (n, v) in remote_refs.items()
if n.startswith(b'refs/tags/')
and not n.endswith(ANNOTATED_TAG_SUFFIX)
})
if b"HEAD" in remote_refs and not bare:
# TODO(jelmer): Support symref capability,
# https://github.com/jelmer/dulwich/issues/485
r[b"HEAD"] = remote_refs[b"HEAD"]
target_config = r.get_config()
if not isinstance(source, bytes):
source = source.encode(DEFAULT_ENCODING)
target_config.set((b'remote', b'origin'), b'url', source)
target_config.set((b'remote', b'origin'), b'fetch',
b'+refs/heads/*:refs/remotes/origin/*')
target_config.write_to_path()
if checkout and b"HEAD" in r.refs:
errstream.write(b'Checking out HEAD\n')
r.reset_index()
except:
r.close()
raise
return r
def test_http(self):
url = 'https://github.com/jelmer/dulwich'
c, path = get_transport_and_path(url)
self.assertTrue(isinstance(c, HttpGitClient))
self.assertEqual('/jelmer/dulwich', path)
def test_username_and_port_explicit_unknown_scheme(self):
c, path = get_transport_and_path(
'unknown://git@server:7999/dply/stuff.git')
self.assertTrue(isinstance(c, SSHGitClient))
self.assertEqual('unknown', c.host)
self.assertEqual('//git@server:7999/dply/stuff.git', path)
def fetch_refspecs(
self,
url: str,
refspecs: Iterable[str],
force: Optional[bool] = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
progress: Callable[["GitProgressEvent"], None] = None,
**kwargs,
):
from dulwich.client import get_transport_and_path
from dulwich.objectspec import parse_reftuples
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
fetch_refs = []
def determine_wants(remote_refs):
fetch_refs.extend(
parse_reftuples(
remote_refs,
self.repo.refs,
[os.fsencode(refspec) for refspec in refspecs],
force=force,
))
return [
remote_refs[lh] for (lh, _, _) in fetch_refs
if remote_refs[lh] not in self.repo.object_store
]
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
from dvc.scm.progress import GitProgressReporter
fetch_result = client.fetch(
path,
self.repo,
progress=GitProgressReporter(progress) if progress else None,
determine_wants=determine_wants,
)
for (lh, rh, _) in fetch_refs:
try:
if rh in self.repo.refs:
check_diverged(self.repo, self.repo.refs[rh],
fetch_result.refs[lh])
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(rh),
os.fsdecode(fetch_result.refs[lh]))
if not overwrite:
continue
self.repo.refs[rh] = fetch_result.refs[lh]
def test_get_transport_and_path_subprocess(self):
client, path = get_transport_and_path('foo.bar/baz')
self.assertTrue(isinstance(client, SubprocessGitClient))
self.assertEqual('foo.bar/baz', path)
def test_tcp_port(self):
c, path = get_transport_and_path('git://foo.com:1234/bar/baz')
self.assertTrue(isinstance(c, TCPGitClient))
self.assertEqual('foo.com', c._host)
self.assertEqual(1234, c._port)
self.assertEqual('/bar/baz', path)
def clone(source,
target=None,
bare=False,
checkout=None,
errstream=default_bytes_err_stream,
outstream=None,
origin=b"origin",
**kwargs):
"""Clone a local or remote git repository.
:param source: Path or URL for source repository
:param target: Path to target repository (optional)
:param bare: Whether or not to create a bare repository
:param checkout: Whether or not to check-out HEAD after cloning
:param errstream: Optional stream to write progress to
:param outstream: Optional stream to write progress to (deprecated)
:param origin: Name of remote from the repository used to clone
:return: The new repository
"""
# TODO(jelmer): This code overlaps quite a bit with Repo.clone
if outstream is not None:
import warnings
warnings.warn(
"outstream= has been deprecated in favour of errstream=.",
DeprecationWarning,
stacklevel=3)
errstream = outstream
if checkout is None:
checkout = (not bare)
if checkout and bare:
raise ValueError("checkout and bare are incompatible")
config = StackedConfig.default()
client, host_path = get_transport_and_path(source, config=config, **kwargs)
if target is None:
target = host_path.split("/")[-1]
if not os.path.exists(target):
os.mkdir(target)
if bare:
r = Repo.init_bare(target)
else:
r = Repo.init(target)
reflog_message = b'clone: from ' + source.encode('utf-8')
try:
fetch_result = fetch(r, host_path, origin, message=reflog_message)
target_config = r.get_config()
if not isinstance(source, bytes):
source = source.encode(DEFAULT_ENCODING)
target_config.set((b'remote', origin), b'url', source)
target_config.set((b'remote', origin), b'fetch',
b'+refs/heads/*:refs/remotes/' + origin + b'/*')
target_config.write_to_path()
# TODO(jelmer): Support symref capability,
# https://github.com/jelmer/dulwich/issues/485
try:
head = r[fetch_result[b'HEAD']]
except KeyError:
head = None
else:
r[b'HEAD'] = head.id
if checkout and not bare and head is not None:
errstream.write(b'Checking out ' + head.id + b'\n')
r.reset_index(head.tree)
except BaseException:
r.close()
raise
return r
def push_refspec(
self,
url: str,
src: Optional[str],
dest: str,
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
progress: Callable[["GitProgressEvent"], None] = None,
**kwargs,
):
from dulwich.client import HTTPUnauthorized, get_transport_and_path
from dulwich.errors import NotGitRepository, SendPackError
from dulwich.porcelain import (
DivergedBranches,
check_diverged,
get_remote_repo,
)
dest_refs, values = self._push_dest_refs(src, dest)
try:
_remote, location = get_remote_repo(self.repo, url)
client, path = get_transport_and_path(location, **kwargs)
except Exception as exc:
raise SCMError(
f"'{url}' is not a valid Git remote or URL") from exc
def update_refs(refs):
from dulwich.objects import ZERO_SHA
new_refs = {}
for ref, value in zip(dest_refs, values):
if ref in refs and value != ZERO_SHA:
local_sha = self.repo.refs[ref]
remote_sha = refs[ref]
try:
check_diverged(self.repo, remote_sha, local_sha)
except DivergedBranches:
if not force:
overwrite = False
if on_diverged:
overwrite = on_diverged(
os.fsdecode(ref), os.fsdecode(remote_sha))
if not overwrite:
continue
new_refs[ref] = value
return new_refs
try:
from dvc.scm.progress import GitProgressReporter
client.send_pack(
path,
update_refs,
self.repo.object_store.generate_pack_data,
progress=GitProgressReporter(progress) if progress else None,
)
except (NotGitRepository, SendPackError) as exc:
raise SCMError("Git failed to push '{src}' to '{url}'") from exc
except HTTPUnauthorized:
raise AuthError(url)
