def __call__(self, repo):
assert repo.url
repo_cache = self._distribution_cache.source_repo_package_xmls.get(repo.name, None)
if not repo_cache:
# Use manifest providers to lazy load
for mp in self._source_manifest_providers or []:
try:
repo_cache = mp(repo)
except Exception as e:
# pass and try next manifest provider
logger.debug('Skipped "%s()": %s' % (mp.__name__, e))
continue
self._distribution_cache.source_repo_package_xmls[repo.name] = repo_cache
break
else:
logger.debug('Load package XMLs for repo "%s" from cache' % repo.name)
# De-duplicate with the release package XMLs. This will cause the YAML writer
# to use references, saving a lot of space in the cache file.
for key in repo_cache or {}:
if key[0] != '_':
repo_cache[key][1] = sanitize_xml(repo_cache[key][1])
if key in self._distribution_cache.release_package_xmls:
release_package_xml = self._distribution_cache.release_package_xmls[key]
if repo_cache[key][1] == release_package_xml:
repo_cache[key][1] = release_package_xml
return repo_cache
def __call__(self, repo):
assert repo.url
repo_cache = self._distribution_cache.source_repo_package_xmls.get(repo.name, None)
if not repo_cache:
# Use manifest providers to lazy load
for mp in self._source_manifest_providers or []:
try:
repo_cache = mp(repo)
except Exception as e:
# pass and try next manifest provider
logger.debug('Skipped "%s()": %s' % (mp.__name__, e))
continue
self._distribution_cache.source_repo_package_xmls[repo.name] = repo_cache
break
else:
logger.debug('Load package XMLs for repo "%s" from cache' % repo.name)
# De-duplicate with the release package XMLs. This will cause the YAML writer
# to use references, saving a lot of space in the cache file.
for key in repo_cache or {}:
if key[0] != '_':
repo_cache[key][1] = sanitize_xml(repo_cache[key][1])
if key in self._distribution_cache.release_package_xmls:
release_package_xml = self._distribution_cache.release_package_xmls[key]
if repo_cache[key][1] == release_package_xml:
repo_cache[key][1] = release_package_xml
return repo_cache
def __call__(self, dist_name, repo, pkg_name):
assert repo.version
package_xml = self._distribution_cache.release_package_xmls.get(
pkg_name, None)
if package_xml:
package_xml = sanitize_xml(package_xml)
self._distribution_cache.release_package_xmls[
pkg_name] = package_xml
logger.debug('Loading package.xml for package "%s" from cache' %
pkg_name)
else:
# use manifest providers to lazy load
for mp in self._manifest_providers or []:
try:
package_xml = sanitize_xml(mp(dist_name, repo, pkg_name))
break
except Exception as e:
# pass and try next manifest provider
logger.debug('Skipped "%s()": %s' % (mp.__name__, e))
if package_xml is None:
return None
# populate the cache
self._distribution_cache.release_package_xmls[
pkg_name] = package_xml
return package_xml
def bitbucket_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
server, path = repo.get_url_parts()
if not server.endswith('bitbucket.org'):
logger.debug('Skip non-bitbucket url "%s"' % repo.url)
raise RuntimeError('Cannot handle non bitbucket url.')
release_tag = repo.get_release_tag(pkg_name)
if not repo.has_remote_tag(release_tag):
raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
url = 'https://bitbucket.org/%s/raw/%s/package.xml' % (path, release_tag)
try:
logger.debug('Load package.xml file from url "%s"' % url)
req = Request(url)
if BITBUCKET_USER and BITBUCKET_PASSWORD:
logger.debug(
'- using http basic auth from supplied environment variables.')
authheader = 'Basic %s' % base64.b64encode(
'%s:%s' % (BITBUCKET_USER, BITBUCKET_PASSWORD))
req.add_header('Authorization', authheader)
package_xml = urlopen(req).read().decode('utf-8')
return package_xml
except URLError as e:
logger.debug('- failed (%s)' % e)
raise RuntimeError()
def bitbucket_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
server, path = repo.get_url_parts()
if not server.endswith('bitbucket.org'):
logger.debug('Skip non-bitbucket url "%s"' % repo.url)
raise RuntimeError('Cannot handle non bitbucket url.')
release_tag = repo.get_release_tag(pkg_name)
if not repo.has_remote_tag(release_tag):
raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
url = 'https://bitbucket.org/%s/raw/%s/package.xml' % (path, release_tag)
try:
logger.debug('Load package.xml file from url "%s"' % url)
req = Request(url)
if BITBUCKET_USER and BITBUCKET_PASSWORD:
logger.debug('- using http basic auth from supplied environment variables.')
authheader = 'Basic %s' % base64.b64encode('%s:%s' % (BITBUCKET_USER, BITBUCKET_PASSWORD))
req.add_header('Authorization', authheader)
package_xml = urlopen(req).read().decode('utf-8')
return package_xml
except URLError as e:
logger.debug('- failed (%s)' % e)
raise RuntimeError()
def check_remote_tag_exists(url, tag):
base = tempfile.mkdtemp('rosdistro')
try:
cmd = [_git_client_executable, 'ls-remote', '--tags', url]
result = _run_command(cmd, base)
if result['returncode'] != 0:
logger.debug('Could not list remote tags of repository "%s": %s' % (url, result['output']))
else:
suffix = '\trefs/tags/%s' % tag
for line in result['output'].splitlines():
if line.endswith(suffix):
return True
finally:
shutil.rmtree(base)
return False
def check_remote_tag_exists(url, tag):
base = tempfile.mkdtemp('rosdistro')
try:
assert _git_client_executable is not None, "'git' not found"
cmd = [_git_client_executable, 'ls-remote', '--tags', url]
result = _run_command(cmd, base)
if result['returncode'] != 0:
logger.debug('Could not list remote tags of repository "%s": %s' % (url, result['output']))
else:
suffix = '\trefs/tags/%s' % tag
for line in result['output'].splitlines():
if line.endswith(suffix):
return True
finally:
shutil.rmtree(base)
return False
def __call__(self, dist_name, repo, pkg_name):
assert repo.version
if pkg_name not in self._distribution_cache.package_xmls:
# use manifest providers to lazy load
package_xml = None
for mp in self._manifest_providers or []:
try:
package_xml = mp(dist_name, repo, pkg_name)
break
except Exception:
pass
if package_xml is None:
return None
self._distribution_cache.package_xmls[pkg_name] = package_xml
else:
logger.debug('Load package.xml file for package "%s" from cache' % pkg_name)
return self._distribution_cache.package_xmls[pkg_name]
def github_source_manifest_provider(repo):
server, path = repo.get_url_parts()
if not server.endswith('github.com'):
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
tree_url = 'https://api.github.com/repos/%s/git/trees/%s?recursive=1' % (
path, repo.version)
req = Request(tree_url)
if GITHUB_USER and GITHUB_PASSWORD:
logger.debug(
'- using http basic auth from supplied environment variables.')
credential_pair = '%s:%s' % (GITHUB_USER, GITHUB_PASSWORD)
authheader = 'Basic %s' % base64.b64encode(
credential_pair.encode()).decode()
req.add_header('Authorization', authheader)
try:
tree_json = json.loads(_get_url_contents(req))
logger.debug('- load repo tree from %s' % tree_url)
except URLError as e:
raise RuntimeError('Unable to fetch JSON tree from %s: %s' %
(tree_url, e))
if tree_json['truncated']:
raise RuntimeError('JSON tree is truncated, must perform full clone.')
package_xml_paths = set()
for obj in tree_json['tree']:
if obj['path'].split('/')[-1] == 'package.xml':
package_xml_paths.add(os.path.dirname(obj['path']))
# Filter out ones that are inside other packages (eg, part of tests)
def package_xml_in_parent(path):
if path == '':
return True
parent = path
while True:
parent = os.path.dirname(parent)
if parent in package_xml_paths:
return False
if parent == '':
return True
package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths))
cache = SourceRepositoryCache.from_ref(tree_json['sha'])
for package_xml_path in package_xml_paths:
url = 'https://raw.githubusercontent.com/%s/%s/%s' % \
(path, cache.ref(), package_xml_path + '/package.xml' if package_xml_path else 'package.xml')
logger.debug('- load package.xml from %s' % url)
package_xml = _get_url_contents(url)
name = parse_package_string(package_xml).name
cache.add(name, package_xml_path, package_xml)
return cache
def __call__(self, dist_name, repo, pkg_name):
assert repo.version
if pkg_name not in self._distribution_cache.release_package_xmls:
# use manifest providers to lazy load
package_xml = None
for mp in self._manifest_providers or []:
try:
package_xml = mp(dist_name, repo, pkg_name)
break
except Exception as e:
# pass and try next manifest provider
logger.debug('Skipped "%s()": %s' % (mp.__name__, e))
if package_xml is None:
return None
self._distribution_cache.release_package_xmls[pkg_name] = package_xml
else:
logger.debug('Load package.xml file for package "%s" from cache' % pkg_name)
return self._distribution_cache.release_package_xmls[pkg_name]
def __call__(self, dist_name, repo, pkg_name):
assert repo.version
if pkg_name not in self._distribution_cache.package_xmls:
# use manifest providers to lazy load
package_xml = None
for mp in self._manifest_providers or []:
try:
package_xml = mp(dist_name, repo, pkg_name)
break
except Exception:
pass
if package_xml is None:
return None
self._distribution_cache.package_xmls[pkg_name] = package_xml
else:
logger.debug('Load package.xml file for package "%s" from cache' %
pkg_name)
return self._distribution_cache.package_xmls[pkg_name]
def __call__(self, dist_name, repo, pkg_name):
assert repo.version
if pkg_name not in self._distribution_cache.release_package_xmls:
# use manifest providers to lazy load
package_xml = None
for mp in self._manifest_providers or []:
try:
package_xml = mp(dist_name, repo, pkg_name)
break
except Exception as e:
# pass and try next manifest provider
logger.debug('Skipped "%s()": %s' % (mp.__name__, e))
if package_xml is None:
return None
self._distribution_cache.release_package_xmls[
pkg_name] = package_xml
else:
logger.debug('Load package.xml file for package "%s" from cache' %
pkg_name)
return self._distribution_cache.release_package_xmls[pkg_name]
def __call__(self, dist_name, repo, pkg_name):
assert repo.version
package_xml = self._distribution_cache.release_package_xmls.get(pkg_name, None)
if package_xml:
package_xml = sanitize_xml(package_xml)
self._distribution_cache.release_package_xmls[pkg_name] = package_xml
logger.debug('Loading package.xml for package "%s" from cache' % pkg_name)
else:
# use manifest providers to lazy load
for mp in self._manifest_providers or []:
try:
package_xml = sanitize_xml(mp(dist_name, repo, pkg_name))
break
except Exception as e:
# pass and try next manifest provider
logger.debug('Skipped "%s()": %s' % (mp.__name__, e))
if package_xml is None:
return None
# populate the cache
self._distribution_cache.release_package_xmls[pkg_name] = package_xml
return package_xml
def github_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
if 'github.com' not in repo.url:
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
release_tag = get_release_tag(repo, pkg_name)
if not check_remote_tag_exists(repo.url, release_tag):
raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
url = repo.url
if url.endswith('.git'):
url = url[:-4]
url += '/%s/package.xml' % release_tag
if url.startswith('git://'):
url = 'https://' + url[6:]
if url.startswith('https://'):
url = 'https://raw.' + url[8:]
try:
logger.debug('Load package.xml file from url "%s"' % url)
package_xml = urlopen(url).read()
return package_xml
except URLError as e:
logger.debug('- failed (%s), trying "%s"' % (e, url))
raise RuntimeError()
def _get_package_xml(url, tag):
base = tempfile.mkdtemp('rosdistro')
package_xml = None
# git 1.7.9 does not support cloning a tag directly, so doing it in two steps
cmd = [_git_client_executable, 'clone', '--depth', '0', url, base]
result = _run_command(cmd, base)
if result['returncode'] != 0:
logger.debug('Could not shallow clone repository "%s"' % url)
else:
cmd = [_git_client_executable, 'checkout', tag]
result = _run_command(cmd, base)
if result['returncode'] != 0:
logger.debug('Could not checkout tag "%s" of repository "%s"' %
(tag, url))
else:
filename = os.path.join(base, 'package.xml')
if os.path.exists(filename):
with open(filename, 'r') as f:
package_xml = f.read()
else:
logger.debug('Could not find package.xml in repository "%s"' %
url)
shutil.rmtree(base)
if package_xml is None:
raise RuntimeError('unable to fetch package.xml')
return package_xml
def github_source_manifest_provider(repo):
server, path = repo.get_url_parts()
if not server.endswith('github.com'):
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
tree_url = 'https://api.github.com/repos/%s/git/trees/%s?recursive=1' % (path, repo.version)
req = Request(tree_url)
if GITHUB_USER and GITHUB_PASSWORD:
logger.debug('- using http basic auth from supplied environment variables.')
authheader = 'Basic %s' % base64.b64encode('%s:%s' % (GITHUB_USER, GITHUB_PASSWORD))
req.add_header('Authorization', authheader)
try:
tree_json = json.loads(_get_url_contents(req))
logger.debug('- load repo tree from %s' % tree_url)
except URLError as e:
raise RuntimeError('Unable to fetch JSON tree from %s: %s' % (tree_url, e))
if tree_json['truncated']:
raise RuntimeError('JSON tree is truncated, must perform full clone.')
package_xml_paths = set()
for obj in tree_json['tree']:
if obj['path'].split('/')[-1] == 'package.xml':
package_xml_paths.add(os.path.dirname(obj['path']))
# Filter out ones that are inside other packages (eg, part of tests)
def package_xml_in_parent(path):
if path == '':
return True
parent = path
while True:
parent = os.path.dirname(parent)
if parent in package_xml_paths:
return False
if parent == '':
return True
package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths))
cache = SourceRepositoryCache.from_ref(tree_json['sha'])
for package_xml_path in package_xml_paths:
url = 'https://raw.githubusercontent.com/%s/%s/%s' % \
(path, cache.ref(), package_xml_path + '/package.xml' if package_xml_path else 'package.xml')
logger.debug('- load package.xml from %s' % url)
package_xml = _get_url_contents(url)
name = parse_package_string(package_xml).name
cache.add(name, package_xml_path, package_xml)
return cache
def tar_source_manifest_provider(repo):
assert repo.type == 'tar'
try:
request = Request(repo.url)
if _TAR_USER and _TAR_PASSWORD:
logger.debug('- using http basic auth from supplied environment variables.')
credential_pair = '%s:%s' % (_TAR_USER, _TAR_PASSWORD)
authheader = 'Basic %s' % base64.b64encode(credential_pair.encode()).decode()
request.add_header('Authorization', authheader)
elif _TAR_PASSWORD:
logger.debug('- using private token auth from supplied environment variables.')
request.add_header('Private-Token', _TAR_PASSWORD)
response = urlopen(request)
with tarfile.open(fileobj=io.BytesIO(response.read())) as tar:
tmpdir = tempfile.mkdtemp()
try:
# Extract just the package.xmls
tar.extractall(path=tmpdir, members=_package_xml_members(tar))
cache = SourceRepositoryCache.from_ref(None)
for package_path in find_package_paths(tmpdir):
if package_path == '.':
package_path = ''
with open(os.path.join(tmpdir, package_path, 'package.xml'), 'r') as f:
package_xml = f.read()
try:
name = parse_package_string(package_xml).name
except InvalidPackage:
raise RuntimeError('Unable to parse package.xml file found in %s' % repo.url)
cache.add(name, package_path, package_xml)
return cache
finally:
rmtree(tmpdir)
except Exception as e:
raise RuntimeError('Unable to fetch source package.xml files: %s' % e)
def tar_manifest_provider(_dist_name, repo, pkg_name):
assert repo.type == 'tar'
subdir = repo.get_release_tag(pkg_name)
request = Request(repo.url)
if _TAR_USER and _TAR_PASSWORD:
logger.debug('- using http basic auth from supplied environment variables.')
authheader = 'Basic %s' % base64.b64encode('%s:%s' % (_TAR_USER, _TAR_PASSWORD))
request.add_header('Authorization', authheader)
elif _TAR_PASSWORD:
logger.debug('- using private token auth from supplied environment variables.')
request.add_header('Private-Token', _TAR_PASSWORD)
response = urlopen(request)
with tarfile.open(fileobj=io.BytesIO(response.read())) as tar:
package_xml = tar.extractfile(os.path.join(subdir, 'package.xml')).read()
# Python2 returns strings, Python3 returns bytes-- support both
try:
return package_xml.decode('utf-8')
except AttributeError:
return package_xml
def github_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
if 'github.com' not in repo.url:
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
release_tag = get_release_tag(repo, pkg_name)
url = repo.url.replace('.git', '/%s/package.xml' % release_tag)
url = url.replace('git://', 'https://')
url = url.replace('https://', 'https://raw.')
try:
logger.debug('Load package.xml file from url "%s"' % url)
package_xml = urllib2.urlopen(url).read()
return package_xml
except urllib2.URLError as e:
logger.debug('- failed (%s), trying "%s"' % (e, url))
raise RuntimeError()
def github_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
server, path = repo.get_url_parts()
if not server.endswith('github.com'):
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
release_tag = repo.get_release_tag(pkg_name)
if not repo.has_remote_tag(release_tag):
raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
url = 'https://raw.githubusercontent.com/%s/%s/package.xml' % (path, release_tag)
try:
logger.debug('Load package.xml file from url "%s"' % url)
return _get_url_contents(url)
except URLError as e:
logger.debug('- failed (%s), trying "%s"' % (e, url))
raise RuntimeError()
def github_manifest_provider(_dist_name, repo, pkg_name):
assert repo.version
server, path = repo.get_url_parts()
if not server.endswith('github.com'):
logger.debug('Skip non-github url "%s"' % repo.url)
raise RuntimeError('can not handle non github urls')
release_tag = repo.get_release_tag(pkg_name)
if not repo.has_remote_tag(release_tag):
raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
url = 'https://raw.githubusercontent.com/%s/%s/package.xml' % (path,
release_tag)
try:
logger.debug('Load package.xml file from url "%s"' % url)
return _get_url_contents(url)
except URLError as e:
logger.debug('- failed (%s), trying "%s"' % (e, url))
raise RuntimeError()
def _get_package_xml(url, tag):
base = tempfile.mkdtemp('rosdistro')
package_xml = None
# git 1.7.9 does not support cloning a tag directly, so doing it in two steps
cmd = [_git_client_executable, 'clone', '--depth', '0', url, base]
result = _run_command(cmd, base)
if result['returncode'] != 0:
logger.debug('Could not shallow clone repository "%s"' % url)
else:
cmd = [_git_client_executable, 'checkout', tag]
result = _run_command(cmd, base)
if result['returncode'] != 0:
logger.debug('Could not checkout tag "%s" of repository "%s"' % (tag, url))
else:
filename = os.path.join(base, 'package.xml')
if os.path.exists(filename):
with open(filename, 'r') as f:
package_xml = f.read()
else:
logger.debug('Could not find package.xml in repository "%s"' % url)
shutil.rmtree(base)
if package_xml is None:
raise RuntimeError('unable to fetch package.xml')
return package_xml