def test_get_brew_url_when_build_was_removed(mocker):
mocker.patch('subprocess.check_output', side_effect=brew_call_removed)
with pytest.raises(CekitError) as excinfo:
tools.get_brew_url('aa')
assert 'Artifact with checksum aa was found in Koji metadata but the build is in incorrect state (DELETED) making the artifact not available for downloading anymore' in str(
excinfo.value)
def test_get_brew_url_no_kerberos(mocker, caplog):
caplog.set_level(logging.DEBUG, logger="cekit")
kerberos_error = subprocess.CalledProcessError(1, 'CMD')
kerberos_error.output = "2019-05-06 14:58:44,502 [ERROR] koji: AuthError: unable to obtain a session"
mocker.patch('subprocess.check_output', side_effect=kerberos_error)
with pytest.raises(CekitError) as excinfo:
tools.get_brew_url('aa')
assert 'Could not fetch archives for checksum aa' in str(excinfo.value)
assert "Brew authentication failed, please make sure you have a valid Kerberos ticket" in caplog.text
def _copy_impl(self, target):
# First of all try to download the file using cacher if specified
if config.get('common', 'cache_url'):
try:
self._download_file(None, target)
return target
except Exception as e:
logger.debug(str(e))
logger.warning(
"Could not download '{}' artifact using cacher".format(
self.name))
md5 = self.get('md5')
# Next option is to download it from Brew directly but only if the md5 checkum
# is provided and we are running with the --redhat switch
if md5 and config.get('common', 'redhat'):
logger.debug(
"Trying to download artifact '{}' from Brew directly".format(
self.name))
try:
# Generate the URL
url = get_brew_url(md5)
# Use the URL to download the file
self._download_file(url, target, use_cache=False)
return target
except Exception as e:
logger.debug(str(e))
logger.warning(
"Could not download artifact '{}' from Brew".format(
self.name))
raise CekitError("Artifact {} could not be found".format(self.name))
def prepare_artifacts(self):
"""Goes through artifacts section of image descriptor
and fetches all of them
"""
if 'artifacts' not in self.image:
logger.debug("No artifacts to fetch")
return
logger.info("Handling artifacts...")
target_dir = os.path.join(self.target, 'image')
fetch_artifacts_url = []
for artifact in self.image['artifacts']:
if isinstance(artifact, _PlainResource) and \
config.get('common', 'redhat'):
fetch_artifacts_url.append({'md5': artifact['md5'],
'url': get_brew_url(artifact['md5']),
'target': os.path.join(artifact['target'])})
artifact['target'] = os.path.join('artifacts', artifact['target'])
else:
artifact.copy(target_dir)
if fetch_artifacts_url:
with open('target/image/fetch-artifacts-url.yaml', 'w') as _file:
yaml.safe_dump(fetch_artifacts_url, _file, default_flow_style=False)
logger.debug("Artifacts handled")
def _copy_impl(self, target):
if config.get('common', 'cache_url'):
logger.debug("Trying to download artifact %s from remote cache" %
self.name)
# If cacher URL is set, use it
try:
self._download_file(self.url, target)
return target
except:
logger.warning(
"Could not download artifact %s from the remote cache" %
self.name)
md5 = self.get('md5')
if md5:
logger.debug("Trying to download artifact %s from Brew directly" %
self.name)
try:
self.url = get_brew_url(md5)
self._download_file(self.url, target, use_cache=False)
return target
except:
logger.warning("Could not download artifact %s from Brew" %
self.name)
raise CekitError("Artifact %s could not be found" % self.name)
def prepare_artifacts(self):
"""Goes through artifacts section of image descriptor
and fetches all of them
"""
if 'artifacts' not in self.image:
logger.debug("No artifacts to fetch")
return
logger.info("Handling artifacts...")
target_dir = os.path.join(self.target, 'image')
for artifact in self.image['artifacts']:
artifact_cache = ArtifactCache()
if isinstance(artifact, _PlainResource):
if artifact_cache.is_cached(artifact):
pass
elif not artifact_cache.is_cached(artifact) and \
config.get('common', 'redhat'):
artifact.url = get_brew_url(artifact['md5'])
else:
if 'description' in artifact:
logger.error(
"Cannot fetch Artifact: '%s', %s" %
(artifact['name'], artifact['description']))
raise CekitError(
"Cannot fetch Artifact: '%s', please cache it via cekit-cache."
% artifact['name'])
artifact.copy(target_dir)
logger.debug("Artifacts handled")
def prepare_artifacts(self):
"""Goes through artifacts section of image descriptor
and fetches all of them
"""
if not self.image.all_artifacts:
logger.debug("No artifacts to fetch")
return
logger.info("Handling artifacts...")
target_dir = os.path.join(self.target, 'image')
fetch_artifacts_url = []
for artifact in self.image.all_artifacts:
logger.info("Preparing artifact {}".format(artifact['name']))
if isinstance(artifact, _PlainResource) and \
config.get('common', 'redhat'):
try:
fetch_artifacts_url.append({
'md5':
artifact['md5'],
'url':
get_brew_url(artifact['md5']),
'target':
os.path.join(artifact['target'])
})
artifact['target'] = os.path.join('artifacts',
artifact['target'])
logger.debug(
"Artifact '{}' added to fetch-artifacts-url.yaml".
format(artifact['name']))
except:
logger.warning(
"Plain artifact {} could not be found in Brew, trying to handle it using lookaside cache"
.format(artifact['name']))
artifact.copy(target_dir)
# TODO: This is ugly, rewrite this!
artifact['lookaside'] = True
else:
artifact.copy(target_dir)
fetch_artifacts_file = os.path.join(self.target, 'image',
'fetch-artifacts-url.yaml')
if fetch_artifacts_url:
with open(fetch_artifacts_file, 'w') as _file:
yaml.safe_dump(fetch_artifacts_url,
_file,
default_flow_style=False)
logger.debug("Artifacts handled")
def test_get_brew_url(mocker):
mocker.patch('subprocess.check_output', side_effect=brew_call_ok)
url = tools.get_brew_url('aa')
assert url == "http://download.devel.redhat.com/brewroot/packages/package_name/" + \
"version/release/maven/group_id/artifact_id/version/filename"
def prepare_artifacts(self):
"""Goes through artifacts section of image descriptor
and fetches all of them
"""
logger.info("Handling artifacts for OSBS...")
target_dir = os.path.join(self.target, 'image')
fetch_artifacts_url = []
url_description = {}
for image in self.images:
for artifact in image.all_artifacts:
logger.info("Preparing artifact '{}' (of type {})".format(artifact['name'], type(artifact)))
if isinstance(artifact, _UrlResource):
intersected_hash = [x for x in crypto.SUPPORTED_HASH_ALGORITHMS if x in artifact]
logger.debug("Found checksum markers of {}".format(intersected_hash))
if not intersected_hash:
logger.warning("No md5 supplied for {}, calculating from the remote artifact".format(artifact['url']))
intersected_hash = ["md5"]
tmpfile = tempfile.NamedTemporaryFile()
try:
artifact.download_file(artifact['url'], tmpfile.name)
artifact["md5"] = crypto.get_sum(tmpfile.name, "md5")
finally:
tmpfile.close()
fetch_artifacts_url.append({'url': artifact['url'],
'target': os.path.join(artifact['target'])})
for c in intersected_hash:
fetch_artifacts_url[0].update({c: artifact[c]})
if 'description' in artifact:
url_description[artifact['url']] = artifact['description']
logger.debug(
"Artifact '{}' (as URL) added to fetch-artifacts-url.yaml".format(artifact['target']))
# OSBS by default downloads all artifacts to artifacts/<target_path>
artifact['target'] = os.path.join('artifacts', artifact['target'])
elif isinstance(artifact, _PlainResource) and config.get('common', 'redhat'):
try:
fetch_artifacts_url.append({'md5': artifact['md5'],
'url': get_brew_url(artifact['md5']),
'target': os.path.join(artifact['target'])})
logger.debug(
"Artifact '{}' (as plain) added to fetch-artifacts-url.yaml".format(artifact['target']))
# OSBS by default downloads all artifacts to artifacts/<target_path>
artifact['target'] = os.path.join('artifacts', artifact['target'])
except:
logger.warning("Plain artifact {} could not be found in Brew, trying to handle it using lookaside cache".
format(artifact['name']))
artifact.copy(target_dir)
# TODO: This is ugly, rewrite this!
artifact['lookaside'] = True
else:
artifact.copy(target_dir)
fetch_artifacts_file = os.path.join(self.target, 'image', 'fetch-artifacts-url.yaml')
if fetch_artifacts_url:
with open(fetch_artifacts_file, 'w') as _file:
yaml.safe_dump(fetch_artifacts_url, _file, default_flow_style=False)
if config.get('common', 'redhat'):
for key,value in url_description.items():
logger.debug("Processing to add build references for {} -> {}".format(key, value))
for line in fileinput.input(fetch_artifacts_file, inplace=1):
line = line.replace(key, key + ' # ' + value)
sys.stdout.write(line)
logger.debug("Artifacts handled")
def test_get_brew_url_with_artifact_containing_dot(mocker):
mocker.patch('subprocess.check_output', side_effect=brew_call_ok_with_dot)
url = tools.get_brew_url('aa')
assert url == "http://download.devel.redhat.com/brewroot/packages/org.glassfish-javax.json/1.0.4/1/maven/org/glassfish/javax.json/1.0.4/javax.json-1.0.4.jar"
def test_get_brew_url(mocker):
mocker.patch('subprocess.check_output', side_effect=brew_call_ok)
url = tools.get_brew_url('aa')
assert url == "http://download.devel.redhat.com/brewroot/packages/net.oauth.core-oauth/20100527/1/maven/net/oauth/core/oauth/20100527/oauth-20100527.jar"
