def test_get_url_bytes_path(self):
base_url = 'https://github.com/jelmer/dulwich'
path_bytes = b'/jelmer/dulwich'
c = HttpGitClient(base_url)
url = c.get_url(path_bytes)
self.assertEqual('https://github.com/jelmer/dulwich', url)
def test_get_url_with_username_and_passwd(self):
base_url = 'https://github.com/jelmer/dulwich'
path = '/jelmer/dulwich'
c = HttpGitClient(base_url, username='******', password='******')
url = c.get_url(path)
self.assertEqual('https://github.com/jelmer/dulwich', url)
def create_repo(repository_url, dir_name, to_fetch, user):
"""Check on valid state repository url and try download it into."""
pth = path.join(REPOS_PATH, dir_name)
flag = 0
if not path.exists(pth):
rep = Repository()
try:
if not is_rep(repository_url):
local = repo.Repo.init(pth, mkdir=True)
client = HttpGitClient(repository_url)
remote_refs = client.fetch(
to_fetch, local,
determine_wants=local.object_store.determine_wants_all,
)
local["HEAD"] = remote_refs["HEAD"]
local._build_tree()
rep.repo_dir_name = pth
rep.dir_name = dir_name
rep.url = '/'.join((repository_url, dir_name))
rep.save()
flag = 1
UserRepository(repo=rep, user=user).save()
rep.last_check = get_head_commit(rep)
rep.save()
create_analysis(dir_name)
except Exception:
rmtree(pth)
if flag == 1:
rep.delete()
raise RuntimeError("Something went wrong.")
else:
rep = get_by_dir_name(dir_name)
if rep:
UserRepository(repo=rep, user=user).save()
def test_get_url_bytes_path(self):
base_url = 'https://github.com/jelmer/dulwich'
path_bytes = b'/jelmer/dulwich'
c = HttpGitClient(base_url)
url = c.get_url(path_bytes)
self.assertEqual('https://github.com/jelmer/dulwich', url)
def test_get_url_with_username_and_passwd(self):
base_url = 'https://github.com/jelmer/dulwich'
path = '/jelmer/dulwich'
c = HttpGitClient(base_url, username='******', password='******')
url = c.get_url(path)
self.assertEqual('https://github.com/jelmer/dulwich', url)
def create_github_release(
repository: Repository,
version: str,
) -> None:
"""
Create a tag and release on GitHub.
"""
changelog_url = 'https://dcos-e2e.readthedocs.io/en/latest/changelog.html'
release_name = 'Release ' + version
release_message = 'See ' + changelog_url
github_release = repository.create_git_tag_and_release(
tag=version,
tag_message='Release ' + version,
release_name=release_name,
release_message=release_message,
type='commit',
object=repository.get_commits()[0].sha,
draft=False,
)
# The artifacts we build must be built from the tag we just created.
# This tag is created remotely on GitHub using the GitHub HTTP API.
#
# We fetch all tags from GitHub and set our local HEAD to the latest master
# from GitHub.
#
# One symptom of this is that ``minidcos --version`` from the PyInstaller
# binary shows the correct version.
local_repository = Repo('.')
client = HttpGitClient(repository.owner.html_url)
remote_refs = client.fetch(repository.name + '.git', local_repository)
# Update the local tags and references with the remote ones.
for key, value in remote_refs.items():
local_repository.refs[key] = value
# Advance local HEAD to remote master HEAD.
local_repository[b'HEAD'] = remote_refs[b'refs/heads/master']
# We need to make the artifacts just after creating a tag so that the
# --version output is exactly the one of the tag.
# No tag exists when the GitHub release is a draft.
# This means that temporarily we have a release without binaries.
linux_artifacts = make_linux_binaries(repo_root=Path('.'))
for installer_path in linux_artifacts:
github_release.upload_asset(
path=str(installer_path),
label=installer_path.name,
)
def test_init_no_username_passwd(self):
url = 'https://github.com/jelmer/dulwich'
c = HttpGitClient(url, config=None)
self.assertIs(None, c._username)
self.assertIs(None, c._password)
self.assertNotIn('authorization', c.pool_manager.headers)
def test_init_no_username_passwd(self):
url = 'https://github.com/jelmer/dulwich'
c = HttpGitClient(url, config=None)
self.assertIs(None, c._username)
self.assertIs(None, c._password)
pw_handler = [
h for h in c.opener.handlers if getattr(h, 'passwd', None) is not None]
self.assertEqual(0, len(pw_handler))
def pull(self, wire, url, apply_refs=True, refs=None, update_after=False):
if url != 'default' and '://' not in url:
client = LocalGitClient(url)
else:
url_obj = url_parser(url)
o = self._build_opener(url)
url, _ = url_obj.authinfo()
client = HttpGitClient(base_url=url, opener=o)
repo = self._factory.repo(wire)
determine_wants = repo.object_store.determine_wants_all
if refs:
def determine_wants_requested(references):
return [references[r] for r in references if r in refs]
determine_wants = determine_wants_requested
try:
remote_refs = client.fetch(path=url,
target=repo,
determine_wants=determine_wants)
except NotGitRepository as e:
log.warning(
'Trying to fetch from "%s" failed, not a Git repository.', url)
# Exception can contain unicode which we convert
raise exceptions.AbortException(e)(repr(e))
# mikhail: client.fetch() returns all the remote refs, but fetches only
# refs filtered by `determine_wants` function. We need to filter result
# as well
if refs:
remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
if apply_refs:
# TODO: johbo: Needs proper test coverage with a git repository
# that contains a tag object, so that we would end up with
# a peeled ref at this point.
for k in remote_refs:
if k.endswith(PEELED_REF_MARKER):
log.debug("Skipping peeled reference %s", k)
continue
repo[k] = remote_refs[k]
if refs and not update_after:
# mikhail: explicitly set the head to the last ref.
repo['HEAD'] = remote_refs[refs[-1]]
if update_after:
# we want to checkout HEAD
repo["HEAD"] = remote_refs["HEAD"]
index.build_index_from_tree(repo.path, repo.index_path(),
repo.object_store, repo["HEAD"].tree)
return remote_refs
def test_init_username_passwd_set(self):
url = 'https://github.com/jelmer/dulwich'
c = HttpGitClient(url, config=None, username='******', password='******')
self.assertEqual('user', c._username)
self.assertEqual('passwd', c._password)
[pw_handler] = [
h for h in c.opener.handlers if getattr(h, 'passwd', None) is not None]
self.assertEqual(
('user', 'passwd'),
pw_handler.passwd.find_user_password(
None, 'https://github.com/jelmer/dulwich'))
def test_init_username_passwd_set(self):
url = 'https://github.com/jelmer/dulwich'
c = HttpGitClient(url, config=None, username='******', password='******')
self.assertEqual('user', c._username)
self.assertEqual('passwd', c._password)
basic_auth = c.pool_manager.headers['authorization']
auth_string = '%s:%s' % ('user', 'passwd')
b64_credentials = self.b64encode(auth_string)
expected_basic_auth = 'Basic %s' % b64_credentials
self.assertEqual(basic_auth, expected_basic_auth)
def fetch(self, wire, url, apply_refs=True, refs=None):
if url != 'default' and '://' not in url:
client = LocalGitClient(url)
else:
url_obj = hg_url(url)
o = self._build_opener(url)
url, _ = url_obj.authinfo()
client = HttpGitClient(base_url=url, opener=o)
repo = self._factory.repo(wire)
determine_wants = repo.object_store.determine_wants_all
if refs:
def determine_wants_requested(references):
return [references[r] for r in references if r in refs]
determine_wants = determine_wants_requested
try:
remote_refs = client.fetch(path=url,
target=repo,
determine_wants=determine_wants)
except NotGitRepository:
log.warning(
'Trying to fetch from "%s" failed, not a Git repository.', url)
raise exceptions.AbortException()
# mikhail: client.fetch() returns all the remote refs, but fetches only
# refs filtered by `determine_wants` function. We need to filter result
# as well
if refs:
remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
if apply_refs:
# TODO: johbo: Needs proper test coverage with a git repository
# that contains a tag object, so that we would end up with
# a peeled ref at this point.
PEELED_REF_MARKER = '^{}'
for k in remote_refs:
if k.endswith(PEELED_REF_MARKER):
log.info("Skipping peeled reference %s", k)
continue
repo[k] = remote_refs[k]
if refs:
# mikhail: explicitly set the head to the last ref.
repo['HEAD'] = remote_refs[refs[-1]]
# TODO: mikhail: should we return remote_refs here to be
# consistent?
else:
return remote_refs
def test_from_parsedurl_on_url_with_quoted_credentials(self):
original_username = '******'
quoted_username = urlquote(original_username)
original_password = '******'
quoted_password = urlquote(original_password)
url = 'https://{username}:{password}@github.com/jelmer/dulwich'.format(
username=quoted_username, password=quoted_password)
c = HttpGitClient.from_parsedurl(urlparse.urlparse(url))
self.assertEqual(original_username, c._username)
self.assertEqual(original_password, c._password)
basic_auth = c.pool_manager.headers['authorization']
auth_string = '%s:%s' % (original_username, original_password)
b64_credentials = self.b64encode(auth_string)
expected_basic_auth = 'Basic %s' % str(b64_credentials)
self.assertEqual(basic_auth, expected_basic_auth)
def test_from_parsedurl_on_url_with_quoted_credentials(self):
original_username = '******'
quoted_username = urlquote(original_username)
original_password = '******'
quoted_password = urlquote(original_password)
url = 'https://{username}:{password}@github.com/jelmer/dulwich'.format(
username=quoted_username, password=quoted_password)
c = HttpGitClient.from_parsedurl(urlparse.urlparse(url))
self.assertEqual(original_username, c._username)
self.assertEqual(original_password, c._password)
[pw_handler] = [
h for h in c.opener.handlers
if getattr(h, 'passwd', None) is not None
]
self.assertEqual((original_username, original_password),
pw_handler.passwd.find_user_password(
None, 'https://github.com/jelmer/dulwich'))
def test_from_parsedurl_on_url_with_quoted_credentials(self):
original_username = '******'
quoted_username = urlquote(original_username)
original_password = '******'
quoted_password = urlquote(original_password)
url = 'https://{username}:{password}@github.com/jelmer/dulwich'.format(
username=quoted_username,
password=quoted_password
)
c = HttpGitClient.from_parsedurl(urlparse.urlparse(url))
self.assertEqual(original_username, c._username)
self.assertEqual(original_password, c._password)
basic_auth = c.pool_manager.headers['authorization']
auth_string = '%s:%s' % (original_username, original_password)
b64_credentials = self.b64encode(auth_string)
expected_basic_auth = 'Basic %s' % str(b64_credentials)
self.assertEqual(basic_auth, expected_basic_auth)
def test_from_parsedurl_on_url_with_quoted_credentials(self):
original_username = '******'
quoted_username = urlquote(original_username)
original_password = '******'
quoted_password = urlquote(original_password)
url = 'https://{username}:{password}@github.com/jelmer/dulwich'.format(
username=quoted_username,
password=quoted_password
)
c = HttpGitClient.from_parsedurl(urlparse.urlparse(url))
self.assertEqual(original_username, c._username)
self.assertEqual(original_password, c._password)
[pw_handler] = [
h for h in c.opener.handlers if getattr(h, 'passwd', None) is not None]
self.assertEqual(
(original_username, original_password),
pw_handler.passwd.find_user_password(
None, 'https://github.com/jelmer/dulwich'))
def _fetch(self, local_path, remote_id):
"""
Fetches a remote repository identified by remote_id (usually a
url) into the local repo identified by local_path. Returns all
remote references fetched.
"""
# dulwich repo
local = Repo(local_path)
pr = urlparse(remote_id)
# Determine the fetch strategy based on protocol.
if pr.scheme == 'http':
# XXX determine if the following is actually correct.
root, frag = remote_id.rsplit('/', 1)
client = HttpGitClient(root)
try:
remote_refs = client.fetch(frag, local)
except Exception: # XXX blind
raise ValueError('error fetching from remote: %s' % remote_id)
elif pr.scheme == 'git':
netloc = pr.netloc.split(':')
if len(netloc) == 1:
host = netloc[0]
port = None
else:
host, port = netloc
# in case pr.path is empty, make it '/' instead.
path = pr.path or '/'
client = TCPGitClient(host, port)
try:
remote_refs = client.fetch(path, local)
except Exception: # XXX blind
raise ValueError('error fetching from remote: %s' % remote_id)
elif remote_id.startswith('/'):
client = Repo(remote_id)
remote_refs = client.fetch(local)
else:
raise ValueError('remote not supported: %s' % remote_id)
return remote_refs
def test_url_redirect_location(self):
from urllib3.response import HTTPResponse
test_data = {
'https://gitlab.com/inkscape/inkscape/': {
'redirect_url':
'https://gitlab.com/inkscape/inkscape.git/',
'refs_data': (b'001e# service=git-upload-pack\n00000032'
b'fb2bebf4919a011f0fd7cec085443d0031228e76 '
b'HEAD\n0000')
},
'https://github.com/jelmer/dulwich/': {
'redirect_url':
'https://github.com/jelmer/dulwich/',
'refs_data': (b'001e# service=git-upload-pack\n00000032'
b'3ff25e09724aa4d86ea5bca7d5dd0399a3c8bfcf '
b'HEAD\n0000')
}
}
tail = 'info/refs?service=git-upload-pack'
# we need to mock urllib3.PoolManager as this test will fail
# otherwise without an active internet connection
class PoolManagerMock():
def __init__(self):
self.headers = {}
def request(self,
method,
url,
fields=None,
headers=None,
redirect=True):
base_url = url[:-len(tail)]
redirect_base_url = test_data[base_url]['redirect_url']
redirect_url = redirect_base_url + tail
headers = {
'Content-Type':
'application/x-git-upload-pack-advertisement'
}
body = test_data[base_url]['refs_data']
# urllib3 handles automatic redirection by default
status = 200
request_url = redirect_url
# simulate urllib3 behavior when redirect parameter is False
if redirect is False:
request_url = url
if redirect_base_url != base_url:
body = ''
headers['location'] = redirect_url
status = 301
return HTTPResponse(body=body,
headers=headers,
request_method=method,
request_url=request_url,
status=status)
pool_manager = PoolManagerMock()
for base_url in test_data.keys():
# instantiate HttpGitClient with mocked pool manager
c = HttpGitClient(base_url, pool_manager=pool_manager, config=None)
# call method that detects url redirection
_, _, processed_url = c._discover_references(
b'git-upload-pack', base_url)
# send the same request as the method above without redirection
resp = c.pool_manager.request('GET',
base_url + tail,
redirect=False)
# check expected behavior of urllib3
redirect_location = resp.get_redirect_location()
if resp.status == 200:
self.assertFalse(redirect_location)
if redirect_location:
# check that url redirection has been correctly detected
self.assertEqual(processed_url, redirect_location[:-len(tail)])
else:
# check also the no redirection case
self.assertEqual(processed_url, base_url)
def test_url_redirect_location(self):
from urllib3.response import HTTPResponse
test_data = {
'https://gitlab.com/inkscape/inkscape/': {
'redirect_url': 'https://gitlab.com/inkscape/inkscape.git/',
'refs_data': (b'001e# service=git-upload-pack\n00000032'
b'fb2bebf4919a011f0fd7cec085443d0031228e76 '
b'HEAD\n0000')
},
'https://github.com/jelmer/dulwich/': {
'redirect_url': 'https://github.com/jelmer/dulwich/',
'refs_data': (b'001e# service=git-upload-pack\n00000032'
b'3ff25e09724aa4d86ea5bca7d5dd0399a3c8bfcf '
b'HEAD\n0000')
}
}
tail = 'info/refs?service=git-upload-pack'
# we need to mock urllib3.PoolManager as this test will fail
# otherwise without an active internet connection
class PoolManagerMock():
def __init__(self):
self.headers = {}
def request(self, method, url, fields=None, headers=None,
redirect=True):
base_url = url[:-len(tail)]
redirect_base_url = test_data[base_url]['redirect_url']
redirect_url = redirect_base_url + tail
headers = {
'Content-Type':
'application/x-git-upload-pack-advertisement'
}
body = test_data[base_url]['refs_data']
# urllib3 handles automatic redirection by default
status = 200
request_url = redirect_url
# simulate urllib3 behavior when redirect parameter is False
if redirect is False:
request_url = url
if redirect_base_url != base_url:
body = ''
headers['location'] = redirect_url
status = 301
return HTTPResponse(body=body,
headers=headers,
request_method=method,
request_url=request_url,
status=status)
pool_manager = PoolManagerMock()
for base_url in test_data.keys():
# instantiate HttpGitClient with mocked pool manager
c = HttpGitClient(base_url, pool_manager=pool_manager,
config=None)
# call method that detects url redirection
_, _, processed_url = c._discover_references(b'git-upload-pack',
base_url)
# send the same request as the method above without redirection
resp = c.pool_manager.request('GET', base_url + tail,
redirect=False)
# check expected behavior of urllib3
redirect_location = resp.get_redirect_location()
if resp.status == 200:
self.assertFalse(redirect_location)
if redirect_location:
# check that url redirection has been correctly detected
self.assertEqual(processed_url, redirect_location[:-len(tail)])
else:
# check also the no redirection case
self.assertEqual(processed_url, base_url)
