def download_build(cls, client, build_id, destination):
logger.info('Downloading packages and logs for build %d', build_id)
rpms = []
logs = []
for chroot in client.build_chroot_proxy.get_list(build_id):
url = chroot.result_url
url = url if url.endswith('/') else url + '/'
d = pyquery.PyQuery(url)
d.make_links_absolute()
for a in d('a[href$=\'.rpm\'], a[href$=\'.log.gz\']'):
fn = os.path.basename(
urllib.parse.urlsplit(a.attrib['href']).path)
dest = os.path.join(destination, chroot.name)
os.makedirs(dest, exist_ok=True)
dest = os.path.join(dest, fn)
if fn.endswith('.src.rpm'):
# skip source RPM
continue
DownloadHelper.download_file(a.attrib['href'], dest)
if fn.endswith('.rpm'):
rpms.append(dest)
elif fn.endswith('.log.gz'):
local_path = dest.replace('.log.gz', '.log')
os.rename(dest, local_path)
logs.append(local_path)
return rpms, logs
def download_build(cls, client, build_id, destination):
logger.info('Downloading packages and logs for build %d', build_id)
try:
result = client.get_build_details(build_id)
except copr.client.exceptions.CoprRequestException as e:
raise RebaseHelperError(
'Failed to get copr build details for {}: {}'.format(
build_id, str(e)))
rpms = []
logs = []
for _, url in six.iteritems(result.data['results_by_chroot']):
url = url if url.endswith('/') else url + '/'
d = pyquery.PyQuery(url)
d.make_links_absolute()
for a in d('a[href$=\'.rpm\'], a[href$=\'.log.gz\']'):
fn = os.path.basename(
urllib.parse.urlsplit(a.attrib['href']).path)
dest = os.path.join(destination, fn)
if fn.endswith('.src.rpm'):
# skip source RPM
continue
DownloadHelper.download_file(a.attrib['href'], dest)
if fn.endswith('.rpm'):
rpms.append(dest)
elif fn.endswith('.log.gz'):
extracted = dest.replace('.log.gz', '.log')
try:
with gzip.open(dest, 'rb') as archive:
with open(extracted, 'wb') as f:
f.write(archive.read())
except (IOError, EOFError):
raise RebaseHelperError(
'Failed to extract {}'.format(dest))
logs.append(extracted)
return rpms, logs
def download_build(cls, client, build_id, destination):
logger.info('Downloading packages and logs for build %d', build_id)
try:
result = client.get_build_details(build_id)
except copr.client.exceptions.CoprRequestException as e:
raise RebaseHelperError(
'Failed to get copr build details for {}: {}'.format(build_id, str(e)))
rpms = []
logs = []
for _, url in result.data['results_by_chroot'].items():
url = url if url.endswith('/') else url + '/'
d = pyquery.PyQuery(url)
d.make_links_absolute()
for a in d('a[href$=\'.rpm\'], a[href$=\'.log.gz\']'):
fn = os.path.basename(urllib.parse.urlsplit(a.attrib['href']).path)
dest = os.path.join(destination, fn)
if fn.endswith('.src.rpm'):
# skip source RPM
continue
DownloadHelper.download_file(a.attrib['href'], dest)
if fn.endswith('.rpm'):
rpms.append(dest)
elif fn.endswith('.log.gz'):
extracted = dest.replace('.log.gz', '.log')
try:
with gzip.open(dest, 'rb') as archive:
with open(extracted, 'wb') as f:
f.write(archive.read())
except (IOError, EOFError):
raise RebaseHelperError(
'Failed to extract {}'.format(dest))
logs.append(extracted)
return rpms, logs
def download_remote_sources(self):
"""
Method that iterates over all sources and downloads ones, which contain URL instead of just a file.
:return: None
"""
try:
# try to download old sources from Fedora lookaside cache
LookasideCacheHelper.download(self.lookaside_cache_preset, os.path.dirname(self.path), self.header.name,
self.sources_location)
except LookasideCacheError as e:
logger.verbose("Downloading sources from lookaside cache failed. "
"Reason: %s.", str(e))
# filter out only sources with URL
remote_files = [source for source in self.sources if bool(urllib.parse.urlparse(source).scheme)]
# download any sources that are not yet downloaded
for remote_file in remote_files:
local_file = os.path.join(self.sources_location, os.path.basename(remote_file))
if not os.path.isfile(local_file):
logger.verbose("File '%s' doesn't exist locally, downloading it.", local_file)
try:
DownloadHelper.download_file(remote_file, local_file)
except DownloadError as e:
raise RebaseHelperError("Failed to download file from URL {}. "
"Reason: '{}'. ".format(remote_file, str(e))) from e
def _download_source(cls,
tool,
url,
package,
filename,
hashtype,
hsh,
target=None):
if target is None:
target = os.path.basename(filename)
if os.path.exists(target):
if cls._hash(target, hashtype) == hsh:
# nothing to do
return
else:
os.unlink(target)
if tool == 'fedpkg':
url = '{0}/{1}/{2}/{3}/{4}/{2}'.format(url, package, filename,
hashtype, hsh)
else:
url = '{0}/{1}/{2}/{3}/{2}'.format(url, package, filename, hsh)
try:
DownloadHelper.download_file(url, target)
except DownloadError as e:
raise LookasideCacheError(six.text_type(e))
def _get_initial_patches(self) -> Dict[str, List[PatchObject]]:
"""Returns a dict of patches from a spec file"""
patches_applied = []
patches_not_used = []
patches_list = [p for p in self.spc.sources if p[2] == 2]
strip_options = self._get_patch_strip_options(patches_list)
for patch, num, _ in patches_list:
is_url = bool(urllib.parse.urlparse(patch).scheme)
filename = os.path.basename(patch) if is_url else patch
patch_path = os.path.join(self.sources_location, filename)
if not os.path.exists(patch_path):
if is_url:
logger.info('Patch%s is remote, trying to download the patch', num)
try:
DownloadHelper.download_file(patch, filename)
except DownloadError:
logger.error('Could not download remote patch %s', patch)
continue
else:
logger.error('Patch %s does not exist', filename)
continue
patch_num = num
if patch_num in strip_options:
patches_applied.append(PatchObject(patch_path, patch_num, strip_options[patch_num]))
else:
patches_not_used.append(PatchObject(patch_path, patch_num, None))
patches_applied = sorted(patches_applied, key=lambda x: x.index)
return {"applied": patches_applied, "not_applied": patches_not_used}
def test_download_existing_file(self, url, content):
"""Test downloading existing file"""
local_file = 'local_file'
DownloadHelper.download_file(url, local_file)
assert os.path.isfile(local_file)
with open(local_file) as f:
assert f.readline().strip() == content
def test_download_existing_file(self, url, content):
"""Test downloading existing file"""
local_file = 'local_file'
DownloadHelper.download_file(url, local_file)
assert os.path.isfile(local_file)
with open(local_file) as f:
assert f.readline().strip() == content
def download_task_results(cls, session, tasklist, destination):
"""Downloads packages and logs of finished Koji tasks.
Args:
session (koji.ClientSession): Active Koji session instance.
tasklist (list): List of task IDs.
destination (str): Path where to download files to.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
rpms = []
logs = []
for task_id in tasklist:
logger.info('Downloading packages and logs for task %s', task_id)
task = session.getTaskInfo(task_id, request=True)
if task['state'] in [koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']]:
logger.info('Task %s is still running!', task_id)
continue
elif task['state'] != koji.TASK_STATES['CLOSED']:
logger.info('Task %s did not complete successfully!', task_id)
if task['method'] == 'buildArch':
tasks = [task]
elif task['method'] == 'build':
opts = dict(parent=task_id, method='buildArch', decode=True,
state=[koji.TASK_STATES['CLOSED'], koji.TASK_STATES['FAILED']])
tasks = session.listTasks(opts=opts)
else:
logger.info('Task %s is not a build or buildArch task!', task_id)
continue
for task in tasks:
base_path = koji.pathinfo.taskrelpath(task['id'])
output = session.listTaskOutput(task['id'])
for filename in output:
local_path = os.path.join(destination, filename)
download = False
fn, ext = os.path.splitext(filename)
if ext == '.rpm':
if task['state'] != koji.TASK_STATES['CLOSED']:
continue
if local_path not in rpms:
nevra = RpmHelper.split_nevra(fn)
# FIXME: multiple arches
download = nevra['arch'] in ['noarch', 'x86_64']
if download:
rpms.append(local_path)
else:
if local_path not in logs:
download = True
logs.append(local_path)
if download:
logger.info('Downloading file %s', filename)
url = '/'.join([session.opts['topurl'], 'work', base_path, filename])
DownloadHelper.download_file(url, local_path)
return rpms, logs
def __iter__(self):
totalsize = len(self.data)
for offset in range(0, totalsize, self.chunksize):
transferred = min(offset + self.chunksize, totalsize)
if not self.check_only:
DownloadHelper.progress(totalsize, transferred,
self.start)
yield self.data[offset:transferred]
def test_progress(self, total, downloaded, output, monkeypatch):
"""
Test that progress of a download is shown correctly. Test the case when size parameters are passed as integers.
"""
buf = io.StringIO()
monkeypatch.setattr('sys.stdout', buf)
monkeypatch.setattr('time.time', lambda: 10.0)
DownloadHelper.progress(total, downloaded, 0.0)
assert buf.getvalue() == output
def test_progress(self, total, downloaded, output, monkeypatch):
"""
Test that progress of a download is shown correctly. Test the case when size parameters are passed as integers.
"""
buf = StringIO()
monkeypatch.setattr('sys.stdout', buf)
monkeypatch.setattr('time.time', lambda: 10.0)
DownloadHelper.progress(total, downloaded, 0.0)
assert buf.getvalue() == output
def _get_commit_hash_from_github(cls, spec_file):
"""
Tries to find a commit using Github API
:param spec_file: SPEC file to base the search on
:return: SHA of a commit, or None
"""
m = re.match(
r'^https?://github\.com/(?P<owner>[\w-]+)/(?P<project>[\w-]+)/.*$',
spec_file.sources[0])
if not m:
return None
baseurl = 'https://api.github.com/repos/{owner}/{project}'.format(
**m.groupdict())
# try to get tag name from a release matching version
r = DownloadHelper.request('{}/releases'.format(baseurl))
if r is None:
return None
if not r.ok:
if r.status_code == 403 and r.headers.get(
'X-RateLimit-Remaining') == '0':
logger.warning(
"Rate limit exceeded on Github API! Try again later.")
return None
data = r.json()
version = spec_file.header.version
tag_name = None
for release in data:
if version in release.get('name', ''):
tag_name = release.get('tag_name')
break
r = DownloadHelper.request('{}/tags'.format(baseurl))
if r is None:
return None
if not r.ok:
if r.status_code == 403 and r.headers.get(
'X-RateLimit-Remaining') == '0':
logger.warning(
"Rate limit exceeded on Github API! Try again later.")
return None
data = r.json()
for tag in data:
name = tag.get('name')
if tag_name:
if name != tag_name:
continue
else:
# no specific tag name, try common tag names
if name not in [version, 'v{}'.format(version)]:
continue
commit = tag.get('commit')
if commit:
return commit.get('sha')
return None
def download_build(cls, session, build_id, destination, arches):
"""Downloads RPMs and logs of a Koji build.
Args:
session (koji.ClientSession): Active Koji session instance.
build_id (str): Koji build ID.
destination (str): Path where to download files to.
arches (list): List of architectures to be downloaded.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
build = session.getBuild(build_id)
packages = session.listRPMs(buildID=build_id)
rpms = []
logs = []
os.makedirs(destination, exist_ok=True)
for pkg in packages:
if pkg['arch'] not in arches:
continue
filename = '.'.join([pkg['nvr'], pkg['arch'], 'rpm'])
local_path = os.path.join(destination, filename)
if local_path not in rpms:
url = '/'.join([
session.opts['topurl'],
'packages',
build['package_name'],
build['version'],
build['release'],
pkg['arch'],
filename])
DownloadHelper.download_file(url, local_path)
rpms.append(local_path)
if pkg['arch'] == 'src':
# No logs for SRPM in koji
continue
for logname in ['build.log', 'root.log', 'state.log']:
local_path = os.path.join(destination, logname)
if local_path not in logs:
url = '/'.join([
session.opts['topurl'],
'packages',
build['package_name'],
build['version'],
build['release'],
'data',
'logs',
pkg['arch'],
logname])
DownloadHelper.download_file(url, local_path)
logs.append(local_path)
return rpms, logs
def run(self):
if self.check_only:
return
n = 0
start = time.time()
while not self.stop_event.is_set():
DownloadHelper.progress(-1,
n * 256 * 1024,
start,
show_size=False)
n += 1
self.stop_event.wait(self.interval)
def __iter__(self):
if self.uploaded:
# ensure the progressbar is shown only once (HTTPSPNEGOAuth causes second request)
yield self.data
else:
totalsize = len(self.data)
for offset in range(0, totalsize, self.chunksize):
transferred = min(offset + self.chunksize, totalsize)
if not self.check_only:
DownloadHelper.progress(totalsize, transferred,
self.start)
yield self.data[offset:transferred]
self.uploaded = True
def _get_version(cls, package_name):
r = DownloadHelper.request('{}/{}/json'.format(cls.API_URL, package_name))
if r is None or not r.ok:
# try to strip python prefix
package_name = re.sub(r'^python[23]?-', '', package_name)
r = DownloadHelper.request('{}/{}/json'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
try:
return data['info']['version']
except KeyError:
return None
def _get_version(cls, package_name):
# special-case "ruby", as https://rubygems.org/api/v1/gems/ruby.json returns nonsense
if package_name == 'ruby':
return None
r = DownloadHelper.request('{}/{}.json'.format(cls.API_URL, package_name))
if r is None or not r.ok:
# try to strip rubygem prefix
package_name = re.sub(r'^rubygem-', '', package_name)
r = DownloadHelper.request('{}/{}.json'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
return data.get('version')
def _get_commit_hash_from_github(cls, spec_file):
"""
Tries to find a commit using Github API
:param spec_file: SPEC file to base the search on
:return: SHA of a commit, or None
"""
m = re.match(r'^https?://github\.com/(?P<owner>[\w-]+)/(?P<project>[\w-]+)/.*$', spec_file.sources[0])
if not m:
return None
baseurl = 'https://api.github.com/repos/{owner}/{project}'.format(**m.groupdict())
# try to get tag name from a release matching version
r = DownloadHelper.request('{}/releases'.format(baseurl))
if r is None:
return None
if not r.ok:
if r.status_code == 403 and r.headers.get('X-RateLimit-Remaining') == '0':
logger.warning("Rate limit exceeded on Github API! Try again later.")
return None
data = r.json()
version = spec_file.get_version()
tag_name = None
for release in data:
if version in release.get('name'):
tag_name = release.get('tag_name')
break
r = DownloadHelper.request('{}/tags'.format(baseurl))
if r is None:
return None
if not r.ok:
if r.status_code == 403 and r.headers.get('X-RateLimit-Remaining') == '0':
logger.warning("Rate limit exceeded on Github API! Try again later.")
return None
data = r.json()
for tag in data:
name = tag.get('name')
if tag_name:
if name != tag_name:
continue
else:
# no specific tag name, try common tag names
if name not in [version, 'v{}'.format(version)]:
continue
commit = tag.get('commit')
if commit:
return commit.get('sha')
return None
def _get_version_using_distro_api(cls, package_name):
r = DownloadHelper.request('{}/project/Fedora/{}'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
return data.get('version')
def _get_version(cls, package_name):
# special-case "ruby", as https://rubygems.org/api/v1/gems/ruby.json returns nonsense
if package_name == 'ruby':
return None
r = DownloadHelper.request('{}/{}.json'.format(cls.API_URL,
package_name))
if r is None or not r.ok:
# try to strip rubygem prefix
package_name = re.sub(r'^rubygem-', '', package_name)
r = DownloadHelper.request('{}/{}.json'.format(
cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
return data.get('version')
def _get_version(cls, package_name):
r = DownloadHelper.request('{}/{}/json'.format(cls.API_URL,
package_name))
if r is None or not r.ok:
# try to strip python prefix
package_name = re.sub(r'^python[23]?-', '', package_name)
r = DownloadHelper.request('{}/{}/json'.format(
cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
try:
return data['info']['version']
except KeyError:
return None
def download_remote_sources(self):
"""
Download the sources from the URL in the configuration (if the path in
the configuration match to the URL basename from SourceX) or from the one
from SourceX in specfile.
"""
# Fetch all sources defined in packit.yaml -> sources
for source in self.package_config.sources:
logger.info(f"Downloading source {source.path!r}.")
DownloadHelper.download_file(
source.url,
str(
Path(self.specfile.sources_location).joinpath(
source.path)),
)
self.specfile.download_remote_sources()
def download_build(cls, session, build_id, destination, arches):
"""Downloads RPMs and logs of a Koji build.
Args:
session (koji.ClientSession): Active Koji session instance.
build_id (str): Koji build ID.
destination (str): Path where to download files to.
arches (list): List of architectures to be downloaded.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
build = session.getBuild(build_id)
packages = session.listRPMs(buildID=build_id)
rpms = []
logs = []
os.makedirs(destination, exist_ok=True)
for pkg in packages:
if pkg['arch'] not in arches:
continue
filename = '.'.join([pkg['nvr'], pkg['arch'], 'rpm'])
local_path = os.path.join(destination, filename)
if local_path not in rpms:
url = '/'.join([
session.opts['topurl'], 'packages', build['package_name'],
build['version'], build['release'], pkg['arch'], filename
])
DownloadHelper.download_file(url, local_path)
rpms.append(local_path)
if pkg['arch'] == 'src':
# No logs for SRPM in koji
continue
for logname in ['build.log', 'root.log', 'state.log']:
local_path = os.path.join(destination, logname)
if local_path not in logs:
url = '/'.join([
session.opts['topurl'], 'packages',
build['package_name'], build['version'],
build['release'], 'data', 'logs', pkg['arch'], logname
])
DownloadHelper.download_file(url, local_path)
logs.append(local_path)
return rpms, logs
def test_keyboard_interrupt_situation(self, monkeypatch):
"""
Test that the local file is deleted in case KeyboardInterrupt is raised during the download
"""
KNOWN_URL = 'https://ftp.isc.org/isc/bind9/9.10.4-P1/srcid'
LOCAL_FILE = os.path.basename(KNOWN_URL)
def interrupter():
raise KeyboardInterrupt
# make sure that some function call inside tha actual download section raises the KeyboardInterrupt exception.
monkeypatch.setattr('time.time', interrupter)
with pytest.raises(KeyboardInterrupt):
DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE)
assert not os.path.exists(LOCAL_FILE)
def test_keyboard_interrupt_situation(self, monkeypatch):
"""
Test that the local file is deleted in case KeyboardInterrupt is raised during the download
"""
KNOWN_URL = 'https://ftp.isc.org/isc/bind9/9.10.4-P1/srcid'
LOCAL_FILE = os.path.basename(KNOWN_URL)
def interrupter():
raise KeyboardInterrupt
# make sure that some function call inside tha actual download section raises the KeyboardInterrupt exception.
monkeypatch.setattr('time.time', interrupter)
with pytest.raises(KeyboardInterrupt):
DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE)
assert not os.path.exists(LOCAL_FILE)
def _get_version(cls, package_name):
if package_name.startswith('ghc-'):
package_name = package_name.replace('ghc-', '', 1)
r = DownloadHelper.request('{}/package/{}/preferred'.format(cls.API_URL, package_name),
headers={'Accept': 'application/json'})
if r is None or not r.ok:
return None
data = r.json()
return data.get('normal-version')[0]
def download_remote_sources(self):
"""
Download the sources from the URL in the configuration (if the path in
the configuration match to the URL basename from SourceX) or from the one
from SourceX in specfile.
"""
# Fetch all sources defined in packit.yaml -> sources
for source in self.package_config.sources:
source_path = self.specfile.sourcedir.joinpath(source.path)
if not source_path.is_file():
logger.info(f"Downloading source {source.path!r}.")
DownloadHelper.download_file(
source.url,
str(source_path),
)
# Try to download sources defined in "sources" file from Fedora lookaside cache
try:
LookasideCacheHelper.download(
"fedpkg",
self.specfile.path.parent,
self.specfile.expanded_name,
self.specfile.sourcedir,
)
except LookasideCacheError as e:
logger.debug(
f"Downloading sources from lookaside cache failed: {e}.")
# Fetch all remote sources defined in the spec file
with self.specfile.sources() as sources, self.specfile.patches(
) as patches:
for source in sources + patches:
if source.remote:
source_path = self.specfile.sourcedir.joinpath(
source.expanded_filename)
if not source_path.is_file():
logger.info(f"Downloading source {source.filename!r}.")
DownloadHelper.download_file(
source.expanded_location,
str(source_path),
)
def _get_version(cls, package_name):
# gets the package name format needed in npm registry
if package_name.startswith('nodejs-'):
package_name = package_name.replace('nodejs-', '')
r = DownloadHelper.request('{}/{}'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
try:
return data.get('dist-tags').get('latest')
except TypeError:
return None
def _get_version(cls, package_name):
# gets the package name format needed in npm registry
if package_name.startswith('nodejs-'):
package_name = package_name.replace('nodejs-', '')
r = DownloadHelper.request('{}/{}'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
try:
return data.get('dist-tags').get('latest')
except TypeError:
return None
def download_build(cls, session, build_id, destination, arches):
"""Downloads RPMs and logs of a Koji build.
Args:
session (koji.ClientSession): Active Koji session instance.
build_id (str): Koji build ID.
destination (str): Path where to download files to.
arches (list): List of architectures to be downloaded.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
build = session.getBuild(build_id)
pathinfo = koji.PathInfo(topdir=session.opts['topurl'])
rpms: List[str] = []
logs: List[str] = []
os.makedirs(destination, exist_ok=True)
for pkg in session.listBuildRPMs(build_id):
if pkg['arch'] not in arches:
continue
rpmpath = pathinfo.rpm(pkg)
local_path = os.path.join(destination, os.path.basename(rpmpath))
if local_path not in rpms:
url = pathinfo.build(build) + '/' + rpmpath
DownloadHelper.download_file(url, local_path)
rpms.append(local_path)
for logfile in session.getBuildLogs(build_id):
if logfile['dir'] not in arches:
continue
local_path = os.path.join(destination, logfile['name'])
if local_path not in logs:
url = pathinfo.topdir + '/' + logfile['path']
DownloadHelper.download_file(url, local_path)
logs.append(local_path)
return rpms, logs
def _get_version_using_pattern_api(cls, package_name):
r = DownloadHelper.request('{}/projects'.format(cls.API_URL), params=dict(pattern=package_name))
if r is None or not r.ok:
return None
data = r.json()
try:
versions = [p['version'] for p in data['projects'] if p['name'] == package_name and p['version']]
except KeyError:
return None
if not versions:
return None
# there can be multiple matching projects, just return the highest version of all of them
return sorted(versions, key=parse_version, reverse=True)[0]
def _get_version(cls, package_name):
# sets package name to format used on metacpan
if package_name.startswith('perl-'):
package_name = package_name.replace('perl-', '', 1)
package_name = package_name.replace('-', '::')
r = DownloadHelper.request('{}/download_url/{}'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
if data.get('status') == 'latest':
return data.get('version')
return None
def _get_version(cls, package_name):
# sets package name to format used on metacpan
if package_name.startswith('perl-'):
package_name = package_name.replace('perl-', '', 1)
package_name = package_name.replace('-', '::')
r = DownloadHelper.request('{}/download_url/{}'.format(cls.API_URL, package_name))
if r is None or not r.ok:
return None
data = r.json()
if data.get('status') == 'latest':
return data.get('version')
return None
def test_download_non_existing_file(self, url):
"""Test downloading NON existing file"""
local_file = 'local_file'
with pytest.raises(DownloadError):
DownloadHelper.download_file(url, local_file)
assert not os.path.isfile(local_file)
def test_download_non_existing_file(self, url):
"""Test downloading NON existing file"""
local_file = 'local_file'
with pytest.raises(DownloadError):
DownloadHelper.download_file(url, local_file)
assert not os.path.isfile(local_file)
def progress(uploaded, total, chunksize, t1, t2): # pylint: disable=unused-argument
DownloadHelper.progress(total, uploaded, upload_start)
def download_task_results(cls, session, tasklist, destination):
"""Downloads packages and logs of finished Koji tasks.
Args:
session (koji.ClientSession): Active Koji session instance.
tasklist (list): List of task IDs.
destination (str): Path where to download files to.
Returns:
tuple: List of downloaded RPMs and list of downloaded logs.
Raises:
DownloadError: If download failed.
"""
rpms = []
logs = []
for task_id in tasklist:
logger.info('Downloading packages and logs for task %s', task_id)
task = session.getTaskInfo(task_id, request=True)
if task['state'] in [
koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']
]:
logger.info('Task %s is still running!', task_id)
continue
elif task['state'] != koji.TASK_STATES['CLOSED']:
logger.info('Task %s did not complete successfully!', task_id)
if task['method'] == 'buildArch':
tasks = [task]
elif task['method'] == 'build':
opts = dict(parent=task_id,
method='buildArch',
decode=True,
state=[
koji.TASK_STATES['CLOSED'],
koji.TASK_STATES['FAILED']
])
tasks = session.listTasks(opts=opts)
else:
logger.info('Task %s is not a build or buildArch task!',
task_id)
continue
for task in tasks:
base_path = koji.pathinfo.taskrelpath(task['id'])
output = session.listTaskOutput(task['id'])
for filename in output:
local_path = os.path.join(destination, filename)
download = False
fn, ext = os.path.splitext(filename)
if ext == '.rpm':
if task['state'] != koji.TASK_STATES['CLOSED']:
continue
if local_path not in rpms:
nevra = RpmHelper.split_nevra(fn)
# FIXME: multiple arches
download = nevra['arch'] in ['noarch', 'x86_64']
if download:
rpms.append(local_path)
else:
if local_path not in logs:
download = True
logs.append(local_path)
if download:
logger.info('Downloading file %s', filename)
url = '/'.join([
session.opts['topurl'], 'work', base_path, filename
])
DownloadHelper.download_file(url, local_path)
return rpms, logs
def progress(uploaded, total, chunksize, t1, t2): # pylint: disable=unused-argument
DownloadHelper.progress(total, uploaded, upload_start)
