def download_build(cls, client, build_id, destination): logger.info('Downloading packages and logs for build %d', build_id) rpms = [] logs = [] for chroot in client.build_chroot_proxy.get_list(build_id): url = chroot.result_url url = url if url.endswith('/') else url + '/' d = pyquery.PyQuery(url) d.make_links_absolute() for a in d('a[href$=\'.rpm\'], a[href$=\'.log.gz\']'): fn = os.path.basename( urllib.parse.urlsplit(a.attrib['href']).path) dest = os.path.join(destination, chroot.name) os.makedirs(dest, exist_ok=True) dest = os.path.join(dest, fn) if fn.endswith('.src.rpm'): # skip source RPM continue DownloadHelper.download_file(a.attrib['href'], dest) if fn.endswith('.rpm'): rpms.append(dest) elif fn.endswith('.log.gz'): local_path = dest.replace('.log.gz', '.log') os.rename(dest, local_path) logs.append(local_path) return rpms, logs
def download_build(cls, client, build_id, destination): logger.info('Downloading packages and logs for build %d', build_id) try: result = client.get_build_details(build_id) except copr.client.exceptions.CoprRequestException as e: raise RebaseHelperError( 'Failed to get copr build details for {}: {}'.format( build_id, str(e))) rpms = [] logs = [] for _, url in six.iteritems(result.data['results_by_chroot']): url = url if url.endswith('/') else url + '/' d = pyquery.PyQuery(url) d.make_links_absolute() for a in d('a[href$=\'.rpm\'], a[href$=\'.log.gz\']'): fn = os.path.basename( urllib.parse.urlsplit(a.attrib['href']).path) dest = os.path.join(destination, fn) if fn.endswith('.src.rpm'): # skip source RPM continue DownloadHelper.download_file(a.attrib['href'], dest) if fn.endswith('.rpm'): rpms.append(dest) elif fn.endswith('.log.gz'): extracted = dest.replace('.log.gz', '.log') try: with gzip.open(dest, 'rb') as archive: with open(extracted, 'wb') as f: f.write(archive.read()) except (IOError, EOFError): raise RebaseHelperError( 'Failed to extract {}'.format(dest)) logs.append(extracted) return rpms, logs
def download_build(cls, client, build_id, destination): logger.info('Downloading packages and logs for build %d', build_id) try: result = client.get_build_details(build_id) except copr.client.exceptions.CoprRequestException as e: raise RebaseHelperError( 'Failed to get copr build details for {}: {}'.format(build_id, str(e))) rpms = [] logs = [] for _, url in result.data['results_by_chroot'].items(): url = url if url.endswith('/') else url + '/' d = pyquery.PyQuery(url) d.make_links_absolute() for a in d('a[href$=\'.rpm\'], a[href$=\'.log.gz\']'): fn = os.path.basename(urllib.parse.urlsplit(a.attrib['href']).path) dest = os.path.join(destination, fn) if fn.endswith('.src.rpm'): # skip source RPM continue DownloadHelper.download_file(a.attrib['href'], dest) if fn.endswith('.rpm'): rpms.append(dest) elif fn.endswith('.log.gz'): extracted = dest.replace('.log.gz', '.log') try: with gzip.open(dest, 'rb') as archive: with open(extracted, 'wb') as f: f.write(archive.read()) except (IOError, EOFError): raise RebaseHelperError( 'Failed to extract {}'.format(dest)) logs.append(extracted) return rpms, logs
def download_remote_sources(self): """ Method that iterates over all sources and downloads ones, which contain URL instead of just a file. :return: None """ try: # try to download old sources from Fedora lookaside cache LookasideCacheHelper.download(self.lookaside_cache_preset, os.path.dirname(self.path), self.header.name, self.sources_location) except LookasideCacheError as e: logger.verbose("Downloading sources from lookaside cache failed. " "Reason: %s.", str(e)) # filter out only sources with URL remote_files = [source for source in self.sources if bool(urllib.parse.urlparse(source).scheme)] # download any sources that are not yet downloaded for remote_file in remote_files: local_file = os.path.join(self.sources_location, os.path.basename(remote_file)) if not os.path.isfile(local_file): logger.verbose("File '%s' doesn't exist locally, downloading it.", local_file) try: DownloadHelper.download_file(remote_file, local_file) except DownloadError as e: raise RebaseHelperError("Failed to download file from URL {}. " "Reason: '{}'. ".format(remote_file, str(e))) from e
def _download_source(cls, tool, url, package, filename, hashtype, hsh, target=None): if target is None: target = os.path.basename(filename) if os.path.exists(target): if cls._hash(target, hashtype) == hsh: # nothing to do return else: os.unlink(target) if tool == 'fedpkg': url = '{0}/{1}/{2}/{3}/{4}/{2}'.format(url, package, filename, hashtype, hsh) else: url = '{0}/{1}/{2}/{3}/{2}'.format(url, package, filename, hsh) try: DownloadHelper.download_file(url, target) except DownloadError as e: raise LookasideCacheError(six.text_type(e))
def _get_initial_patches(self) -> Dict[str, List[PatchObject]]: """Returns a dict of patches from a spec file""" patches_applied = [] patches_not_used = [] patches_list = [p for p in self.spc.sources if p[2] == 2] strip_options = self._get_patch_strip_options(patches_list) for patch, num, _ in patches_list: is_url = bool(urllib.parse.urlparse(patch).scheme) filename = os.path.basename(patch) if is_url else patch patch_path = os.path.join(self.sources_location, filename) if not os.path.exists(patch_path): if is_url: logger.info('Patch%s is remote, trying to download the patch', num) try: DownloadHelper.download_file(patch, filename) except DownloadError: logger.error('Could not download remote patch %s', patch) continue else: logger.error('Patch %s does not exist', filename) continue patch_num = num if patch_num in strip_options: patches_applied.append(PatchObject(patch_path, patch_num, strip_options[patch_num])) else: patches_not_used.append(PatchObject(patch_path, patch_num, None)) patches_applied = sorted(patches_applied, key=lambda x: x.index) return {"applied": patches_applied, "not_applied": patches_not_used}
def test_download_existing_file(self, url, content): """Test downloading existing file""" local_file = 'local_file' DownloadHelper.download_file(url, local_file) assert os.path.isfile(local_file) with open(local_file) as f: assert f.readline().strip() == content
def download_task_results(cls, session, tasklist, destination): """Downloads packages and logs of finished Koji tasks. Args: session (koji.ClientSession): Active Koji session instance. tasklist (list): List of task IDs. destination (str): Path where to download files to. Returns: tuple: List of downloaded RPMs and list of downloaded logs. Raises: DownloadError: If download failed. """ rpms = [] logs = [] for task_id in tasklist: logger.info('Downloading packages and logs for task %s', task_id) task = session.getTaskInfo(task_id, request=True) if task['state'] in [koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN']]: logger.info('Task %s is still running!', task_id) continue elif task['state'] != koji.TASK_STATES['CLOSED']: logger.info('Task %s did not complete successfully!', task_id) if task['method'] == 'buildArch': tasks = [task] elif task['method'] == 'build': opts = dict(parent=task_id, method='buildArch', decode=True, state=[koji.TASK_STATES['CLOSED'], koji.TASK_STATES['FAILED']]) tasks = session.listTasks(opts=opts) else: logger.info('Task %s is not a build or buildArch task!', task_id) continue for task in tasks: base_path = koji.pathinfo.taskrelpath(task['id']) output = session.listTaskOutput(task['id']) for filename in output: local_path = os.path.join(destination, filename) download = False fn, ext = os.path.splitext(filename) if ext == '.rpm': if task['state'] != koji.TASK_STATES['CLOSED']: continue if local_path not in rpms: nevra = RpmHelper.split_nevra(fn) # FIXME: multiple arches download = nevra['arch'] in ['noarch', 'x86_64'] if download: rpms.append(local_path) else: if local_path not in logs: download = True logs.append(local_path) if download: logger.info('Downloading file %s', filename) url = '/'.join([session.opts['topurl'], 'work', base_path, filename]) DownloadHelper.download_file(url, local_path) return rpms, logs
def __iter__(self): totalsize = len(self.data) for offset in range(0, totalsize, self.chunksize): transferred = min(offset + self.chunksize, totalsize) if not self.check_only: DownloadHelper.progress(totalsize, transferred, self.start) yield self.data[offset:transferred]
def test_progress(self, total, downloaded, output, monkeypatch): """ Test that progress of a download is shown correctly. Test the case when size parameters are passed as integers. """ buf = io.StringIO() monkeypatch.setattr('sys.stdout', buf) monkeypatch.setattr('time.time', lambda: 10.0) DownloadHelper.progress(total, downloaded, 0.0) assert buf.getvalue() == output
def test_progress(self, total, downloaded, output, monkeypatch): """ Test that progress of a download is shown correctly. Test the case when size parameters are passed as integers. """ buf = StringIO() monkeypatch.setattr('sys.stdout', buf) monkeypatch.setattr('time.time', lambda: 10.0) DownloadHelper.progress(total, downloaded, 0.0) assert buf.getvalue() == output
def _get_commit_hash_from_github(cls, spec_file): """ Tries to find a commit using Github API :param spec_file: SPEC file to base the search on :return: SHA of a commit, or None """ m = re.match( r'^https?://github\.com/(?P<owner>[\w-]+)/(?P<project>[\w-]+)/.*$', spec_file.sources[0]) if not m: return None baseurl = 'https://api.github.com/repos/{owner}/{project}'.format( **m.groupdict()) # try to get tag name from a release matching version r = DownloadHelper.request('{}/releases'.format(baseurl)) if r is None: return None if not r.ok: if r.status_code == 403 and r.headers.get( 'X-RateLimit-Remaining') == '0': logger.warning( "Rate limit exceeded on Github API! Try again later.") return None data = r.json() version = spec_file.header.version tag_name = None for release in data: if version in release.get('name', ''): tag_name = release.get('tag_name') break r = DownloadHelper.request('{}/tags'.format(baseurl)) if r is None: return None if not r.ok: if r.status_code == 403 and r.headers.get( 'X-RateLimit-Remaining') == '0': logger.warning( "Rate limit exceeded on Github API! Try again later.") return None data = r.json() for tag in data: name = tag.get('name') if tag_name: if name != tag_name: continue else: # no specific tag name, try common tag names if name not in [version, 'v{}'.format(version)]: continue commit = tag.get('commit') if commit: return commit.get('sha') return None
def download_build(cls, session, build_id, destination, arches): """Downloads RPMs and logs of a Koji build. Args: session (koji.ClientSession): Active Koji session instance. build_id (str): Koji build ID. destination (str): Path where to download files to. arches (list): List of architectures to be downloaded. Returns: tuple: List of downloaded RPMs and list of downloaded logs. Raises: DownloadError: If download failed. """ build = session.getBuild(build_id) packages = session.listRPMs(buildID=build_id) rpms = [] logs = [] os.makedirs(destination, exist_ok=True) for pkg in packages: if pkg['arch'] not in arches: continue filename = '.'.join([pkg['nvr'], pkg['arch'], 'rpm']) local_path = os.path.join(destination, filename) if local_path not in rpms: url = '/'.join([ session.opts['topurl'], 'packages', build['package_name'], build['version'], build['release'], pkg['arch'], filename]) DownloadHelper.download_file(url, local_path) rpms.append(local_path) if pkg['arch'] == 'src': # No logs for SRPM in koji continue for logname in ['build.log', 'root.log', 'state.log']: local_path = os.path.join(destination, logname) if local_path not in logs: url = '/'.join([ session.opts['topurl'], 'packages', build['package_name'], build['version'], build['release'], 'data', 'logs', pkg['arch'], logname]) DownloadHelper.download_file(url, local_path) logs.append(local_path) return rpms, logs
def run(self): if self.check_only: return n = 0 start = time.time() while not self.stop_event.is_set(): DownloadHelper.progress(-1, n * 256 * 1024, start, show_size=False) n += 1 self.stop_event.wait(self.interval)
def __iter__(self): if self.uploaded: # ensure the progressbar is shown only once (HTTPSPNEGOAuth causes second request) yield self.data else: totalsize = len(self.data) for offset in range(0, totalsize, self.chunksize): transferred = min(offset + self.chunksize, totalsize) if not self.check_only: DownloadHelper.progress(totalsize, transferred, self.start) yield self.data[offset:transferred] self.uploaded = True
def _get_version(cls, package_name): r = DownloadHelper.request('{}/{}/json'.format(cls.API_URL, package_name)) if r is None or not r.ok: # try to strip python prefix package_name = re.sub(r'^python[23]?-', '', package_name) r = DownloadHelper.request('{}/{}/json'.format(cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() try: return data['info']['version'] except KeyError: return None
def _get_version(cls, package_name): # special-case "ruby", as https://rubygems.org/api/v1/gems/ruby.json returns nonsense if package_name == 'ruby': return None r = DownloadHelper.request('{}/{}.json'.format(cls.API_URL, package_name)) if r is None or not r.ok: # try to strip rubygem prefix package_name = re.sub(r'^rubygem-', '', package_name) r = DownloadHelper.request('{}/{}.json'.format(cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() return data.get('version')
def _get_commit_hash_from_github(cls, spec_file): """ Tries to find a commit using Github API :param spec_file: SPEC file to base the search on :return: SHA of a commit, or None """ m = re.match(r'^https?://github\.com/(?P<owner>[\w-]+)/(?P<project>[\w-]+)/.*$', spec_file.sources[0]) if not m: return None baseurl = 'https://api.github.com/repos/{owner}/{project}'.format(**m.groupdict()) # try to get tag name from a release matching version r = DownloadHelper.request('{}/releases'.format(baseurl)) if r is None: return None if not r.ok: if r.status_code == 403 and r.headers.get('X-RateLimit-Remaining') == '0': logger.warning("Rate limit exceeded on Github API! Try again later.") return None data = r.json() version = spec_file.get_version() tag_name = None for release in data: if version in release.get('name'): tag_name = release.get('tag_name') break r = DownloadHelper.request('{}/tags'.format(baseurl)) if r is None: return None if not r.ok: if r.status_code == 403 and r.headers.get('X-RateLimit-Remaining') == '0': logger.warning("Rate limit exceeded on Github API! Try again later.") return None data = r.json() for tag in data: name = tag.get('name') if tag_name: if name != tag_name: continue else: # no specific tag name, try common tag names if name not in [version, 'v{}'.format(version)]: continue commit = tag.get('commit') if commit: return commit.get('sha') return None
def _get_version_using_distro_api(cls, package_name): r = DownloadHelper.request('{}/project/Fedora/{}'.format(cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() return data.get('version')
def _get_version(cls, package_name): # special-case "ruby", as https://rubygems.org/api/v1/gems/ruby.json returns nonsense if package_name == 'ruby': return None r = DownloadHelper.request('{}/{}.json'.format(cls.API_URL, package_name)) if r is None or not r.ok: # try to strip rubygem prefix package_name = re.sub(r'^rubygem-', '', package_name) r = DownloadHelper.request('{}/{}.json'.format( cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() return data.get('version')
def _get_version(cls, package_name): r = DownloadHelper.request('{}/{}/json'.format(cls.API_URL, package_name)) if r is None or not r.ok: # try to strip python prefix package_name = re.sub(r'^python[23]?-', '', package_name) r = DownloadHelper.request('{}/{}/json'.format( cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() try: return data['info']['version'] except KeyError: return None
def download_remote_sources(self): """ Download the sources from the URL in the configuration (if the path in the configuration match to the URL basename from SourceX) or from the one from SourceX in specfile. """ # Fetch all sources defined in packit.yaml -> sources for source in self.package_config.sources: logger.info(f"Downloading source {source.path!r}.") DownloadHelper.download_file( source.url, str( Path(self.specfile.sources_location).joinpath( source.path)), ) self.specfile.download_remote_sources()
def download_build(cls, session, build_id, destination, arches): """Downloads RPMs and logs of a Koji build. Args: session (koji.ClientSession): Active Koji session instance. build_id (str): Koji build ID. destination (str): Path where to download files to. arches (list): List of architectures to be downloaded. Returns: tuple: List of downloaded RPMs and list of downloaded logs. Raises: DownloadError: If download failed. """ build = session.getBuild(build_id) packages = session.listRPMs(buildID=build_id) rpms = [] logs = [] os.makedirs(destination, exist_ok=True) for pkg in packages: if pkg['arch'] not in arches: continue filename = '.'.join([pkg['nvr'], pkg['arch'], 'rpm']) local_path = os.path.join(destination, filename) if local_path not in rpms: url = '/'.join([ session.opts['topurl'], 'packages', build['package_name'], build['version'], build['release'], pkg['arch'], filename ]) DownloadHelper.download_file(url, local_path) rpms.append(local_path) if pkg['arch'] == 'src': # No logs for SRPM in koji continue for logname in ['build.log', 'root.log', 'state.log']: local_path = os.path.join(destination, logname) if local_path not in logs: url = '/'.join([ session.opts['topurl'], 'packages', build['package_name'], build['version'], build['release'], 'data', 'logs', pkg['arch'], logname ]) DownloadHelper.download_file(url, local_path) logs.append(local_path) return rpms, logs
def test_keyboard_interrupt_situation(self, monkeypatch): """ Test that the local file is deleted in case KeyboardInterrupt is raised during the download """ KNOWN_URL = 'https://ftp.isc.org/isc/bind9/9.10.4-P1/srcid' LOCAL_FILE = os.path.basename(KNOWN_URL) def interrupter(): raise KeyboardInterrupt # make sure that some function call inside tha actual download section raises the KeyboardInterrupt exception. monkeypatch.setattr('time.time', interrupter) with pytest.raises(KeyboardInterrupt): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.exists(LOCAL_FILE)
def _get_version(cls, package_name): if package_name.startswith('ghc-'): package_name = package_name.replace('ghc-', '', 1) r = DownloadHelper.request('{}/package/{}/preferred'.format(cls.API_URL, package_name), headers={'Accept': 'application/json'}) if r is None or not r.ok: return None data = r.json() return data.get('normal-version')[0]
def download_remote_sources(self): """ Download the sources from the URL in the configuration (if the path in the configuration match to the URL basename from SourceX) or from the one from SourceX in specfile. """ # Fetch all sources defined in packit.yaml -> sources for source in self.package_config.sources: source_path = self.specfile.sourcedir.joinpath(source.path) if not source_path.is_file(): logger.info(f"Downloading source {source.path!r}.") DownloadHelper.download_file( source.url, str(source_path), ) # Try to download sources defined in "sources" file from Fedora lookaside cache try: LookasideCacheHelper.download( "fedpkg", self.specfile.path.parent, self.specfile.expanded_name, self.specfile.sourcedir, ) except LookasideCacheError as e: logger.debug( f"Downloading sources from lookaside cache failed: {e}.") # Fetch all remote sources defined in the spec file with self.specfile.sources() as sources, self.specfile.patches( ) as patches: for source in sources + patches: if source.remote: source_path = self.specfile.sourcedir.joinpath( source.expanded_filename) if not source_path.is_file(): logger.info(f"Downloading source {source.filename!r}.") DownloadHelper.download_file( source.expanded_location, str(source_path), )
def _get_version(cls, package_name): # gets the package name format needed in npm registry if package_name.startswith('nodejs-'): package_name = package_name.replace('nodejs-', '') r = DownloadHelper.request('{}/{}'.format(cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() try: return data.get('dist-tags').get('latest') except TypeError: return None
def download_build(cls, session, build_id, destination, arches): """Downloads RPMs and logs of a Koji build. Args: session (koji.ClientSession): Active Koji session instance. build_id (str): Koji build ID. destination (str): Path where to download files to. arches (list): List of architectures to be downloaded. Returns: tuple: List of downloaded RPMs and list of downloaded logs. Raises: DownloadError: If download failed. """ build = session.getBuild(build_id) pathinfo = koji.PathInfo(topdir=session.opts['topurl']) rpms: List[str] = [] logs: List[str] = [] os.makedirs(destination, exist_ok=True) for pkg in session.listBuildRPMs(build_id): if pkg['arch'] not in arches: continue rpmpath = pathinfo.rpm(pkg) local_path = os.path.join(destination, os.path.basename(rpmpath)) if local_path not in rpms: url = pathinfo.build(build) + '/' + rpmpath DownloadHelper.download_file(url, local_path) rpms.append(local_path) for logfile in session.getBuildLogs(build_id): if logfile['dir'] not in arches: continue local_path = os.path.join(destination, logfile['name']) if local_path not in logs: url = pathinfo.topdir + '/' + logfile['path'] DownloadHelper.download_file(url, local_path) logs.append(local_path) return rpms, logs
def _get_version_using_pattern_api(cls, package_name): r = DownloadHelper.request('{}/projects'.format(cls.API_URL), params=dict(pattern=package_name)) if r is None or not r.ok: return None data = r.json() try: versions = [p['version'] for p in data['projects'] if p['name'] == package_name and p['version']] except KeyError: return None if not versions: return None # there can be multiple matching projects, just return the highest version of all of them return sorted(versions, key=parse_version, reverse=True)[0]
def _get_version(cls, package_name): # sets package name to format used on metacpan if package_name.startswith('perl-'): package_name = package_name.replace('perl-', '', 1) package_name = package_name.replace('-', '::') r = DownloadHelper.request('{}/download_url/{}'.format(cls.API_URL, package_name)) if r is None or not r.ok: return None data = r.json() if data.get('status') == 'latest': return data.get('version') return None
def test_download_non_existing_file(self, url): """Test downloading NON existing file""" local_file = 'local_file' with pytest.raises(DownloadError): DownloadHelper.download_file(url, local_file) assert not os.path.isfile(local_file)
def progress(uploaded, total, chunksize, t1, t2): # pylint: disable=unused-argument DownloadHelper.progress(total, uploaded, upload_start)
def download_task_results(cls, session, tasklist, destination): """Downloads packages and logs of finished Koji tasks. Args: session (koji.ClientSession): Active Koji session instance. tasklist (list): List of task IDs. destination (str): Path where to download files to. Returns: tuple: List of downloaded RPMs and list of downloaded logs. Raises: DownloadError: If download failed. """ rpms = [] logs = [] for task_id in tasklist: logger.info('Downloading packages and logs for task %s', task_id) task = session.getTaskInfo(task_id, request=True) if task['state'] in [ koji.TASK_STATES['FREE'], koji.TASK_STATES['OPEN'] ]: logger.info('Task %s is still running!', task_id) continue elif task['state'] != koji.TASK_STATES['CLOSED']: logger.info('Task %s did not complete successfully!', task_id) if task['method'] == 'buildArch': tasks = [task] elif task['method'] == 'build': opts = dict(parent=task_id, method='buildArch', decode=True, state=[ koji.TASK_STATES['CLOSED'], koji.TASK_STATES['FAILED'] ]) tasks = session.listTasks(opts=opts) else: logger.info('Task %s is not a build or buildArch task!', task_id) continue for task in tasks: base_path = koji.pathinfo.taskrelpath(task['id']) output = session.listTaskOutput(task['id']) for filename in output: local_path = os.path.join(destination, filename) download = False fn, ext = os.path.splitext(filename) if ext == '.rpm': if task['state'] != koji.TASK_STATES['CLOSED']: continue if local_path not in rpms: nevra = RpmHelper.split_nevra(fn) # FIXME: multiple arches download = nevra['arch'] in ['noarch', 'x86_64'] if download: rpms.append(local_path) else: if local_path not in logs: download = True logs.append(local_path) if download: logger.info('Downloading file %s', filename) url = '/'.join([ session.opts['topurl'], 'work', base_path, filename ]) DownloadHelper.download_file(url, local_path) return rpms, logs