def _init_test( swh_scheduler: SchedulerInterface, debian_sources: Dict[Suite, SourcesText], requests_mock, ) -> Tuple[DebianLister, DebianSuitePkgSrcInfo]: lister = DebianLister( scheduler=swh_scheduler, mirror_url=_mirror_url, suites=list(debian_sources.keys()), components=_components, ) suite_pkg_info: DebianSuitePkgSrcInfo = {} for suite, sources in debian_sources.items(): suite_pkg_info[suite] = defaultdict(list) for pkg_src in Sources.iter_paragraphs(sources): suite_pkg_info[suite][pkg_src["Package"]].append(pkg_src) for idx_url, compression in lister.debian_index_urls( suite, _components[0]): if compression: requests_mock.get(idx_url, status_code=404) else: requests_mock.get(idx_url, text=sources) return lister, suite_pkg_info
def transport_response_simplified(self, response): """Decompress and parse the package index fetched in `transport_request`. For each package, we "pivot" the file list entries (Files, Checksums-Sha1, Checksums-Sha256), to return a files dict mapping filenames to their checksums. """ if self.decompressor: data = self.decompressor(response.raw) else: data = response.raw for src_pkg in Sources.iter_paragraphs(data.readlines()): files = defaultdict(dict) for field in src_pkg._multivalued_fields: if field.startswith('checksums-'): sum_name = field[len('checksums-'):] else: sum_name = 'md5sum' if field in src_pkg: for entry in src_pkg[field]: name = entry['name'] files[name]['name'] = entry['name'] files[name]['size'] = int(entry['size'], 10) files[name][sum_name] = entry[sum_name] yield { 'name': src_pkg['Package'], 'version': src_pkg['Version'], 'directory': src_pkg['Directory'], 'files': files, }
def extract_deb_packages(data, packages): """ Extract package metadata from debian Packages file """ extracted = extract(data) package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) sio = StringIO(extracted) for i, stanza in enumerate(Sources.iter_paragraphs(sio)): fullversion = Version(stanza['version']) arch = stanza['architecture'] name = stanza['package'] epoch = fullversion._BaseVersion__epoch if epoch is None: epoch = '' version = fullversion._BaseVersion__upstream_version release = fullversion._BaseVersion__debian_revision if release is None: release = '' progress_update_s.send(sender=None, index=i + 1) package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='D') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo\n')
async def iter_sources(url): async with ClientSession() as session: async with session.get(url) as resp: if resp.status != 200: raise Exception("URL %s returned response code %d" % (url, resp.status)) contents = await resp.read() if url.endswith(".gz"): contents = gzip.decompress(contents) for source in Sources.iter_paragraphs(contents): yield source
def find_dsc(self, source): sources = "{root}/dists/{suite}/{component}/source/Sources.gz".format( root=self.root, suite=source.suite.name, component=source.component.name ) for entry in Sources.iter_paragraphs(GzipFile(filename=sources)): if entry['Package'] == source.name and entry['Version'] == source.version: dsc = None for line in entry['Files']: if line['name'].endswith(".dsc"): dsc = line['name'] break return (entry['Directory'], dsc) raise Exception("Package not found in Sources.gz")
def spec_show(request, spec): tasks = BuildTask.objects.filter(specification=spec).select_related() packages = Package.objects.filter(specification=spec) # Combined build and task logs logs = [] logs += [(log.created, '', str(log), 'spec', log) for log in SpecificationLog.objects.filter(spec=spec)] logs += [(log.created, getattr(log.task.builder, 'name', None), str(log), 'task', log) for log in BuildTaskLog.objects.filter(task__specification=spec) \ .select_related(depth=3)] logs = reversed(sorted(logs)) task_logs = BuildTaskLog.objects.filter(task__specification=spec) # Source files sources = [] dsc = spec.dsc() if dsc is not None: dsc_path = os.path.join(settings.DOWNLOAD_TARGET, str(spec.id), dsc) if os.path.exists(dsc_path): dsc_size = os.stat(dsc_path).st_size sources.append((reverse('build_spec_source', args=[spec.id, dsc]), dsc, dsc_size)) src = Sources(open(dsc_path)) for info in src['Files']: fname = info['name'] size = int(info['size']) path = os.path.join(settings.DOWNLOAD_TARGET, str(spec.id), fname) if os.path.exists(path): sources.append((reverse('build_spec_source', args=[spec.id, fname]), fname, size)) context = { 'build': spec, 'tasks': tasks, 'packages': packages, 'logs': logs, 'sources': sources } return render_to_response('build/spec_show.html', context, context_instance=RequestContext(request))
def find_dsc(self, source): sources = "{root}/dists/{suite}/{component}/source/Sources.gz".format( root=self.root, suite=source.suite.name, component=source.component.name) for entry in Sources.iter_paragraphs(GzipFile(filename=sources)): if entry['Package'] == source.name and entry[ 'Version'] == source.version: dsc = None for line in entry['Files']: if line['name'].endswith(".dsc"): dsc = line['name'] break return (entry['Directory'], dsc) raise RepoPackageNotFound('{0}-{1}'.format(source.name, source.version))
def page_request(self, suite: Suite, component: Component) -> DebianPageType: """Return parsed package Sources file for a given debian suite and component.""" for url, compression in self.debian_index_urls(suite, component): response = requests.get(url, stream=True) logging.debug("Fetched URL: %s, status code: %s", url, response.status_code) if response.status_code == 200: break else: raise Exception("Could not retrieve sources index for %s/%s", suite, component) decompressor = decompressors.get(compression) if decompressor: data = decompressor(response.raw) else: data = response.raw return Sources.iter_paragraphs(data.readlines())
def _init_test( swh_scheduler: SchedulerInterface, debian_sources: Dict[Suite, SourcesText], requests_mock, ) -> Tuple[DebianLister, DebianSuitePkgSrcInfo]: lister = DebianLister( scheduler=swh_scheduler, mirror_url=_mirror_url, suites=list(debian_sources.keys()), components=_components, ) suite_pkg_info: DebianSuitePkgSrcInfo = {} for i, (suite, sources) in enumerate(debian_sources.items()): # ensure to generate a different date for each suite last_modified = formatdate(timeval=datetime.now().timestamp() + i, usegmt=True) suite_pkg_info[suite] = defaultdict(list) for pkg_src in Sources.iter_paragraphs(sources): suite_pkg_info[suite][pkg_src["Package"]].append(pkg_src) # backup package last update date global _last_modified _last_modified[pkg_src["Package"]] = last_modified for idx_url, compression in lister.debian_index_urls(suite, _components[0]): if compression: requests_mock.get(idx_url, status_code=404) else: requests_mock.get( idx_url, text=sources, headers={"Last-Modified": last_modified}, ) for idx_url, _ in lister.debian_index_urls(suite, _components[1]): requests_mock.get(idx_url, status_code=404) return lister, suite_pkg_info
def extract_deb_packages(data, url): """ Extract package metadata from debian Packages file """ extracted = extract(data, url) package_re = re.compile(b'^Package: ', re.M) plen = len(package_re.findall(extracted)) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) bio = BytesIO(extracted) for i, stanza in enumerate(Sources.iter_paragraphs(bio)): fullversion = Version(stanza['version']) arch = stanza['architecture'] name = stanza['package'] epoch = fullversion._BaseVersion__epoch if epoch is None: epoch = '' version = fullversion._BaseVersion__upstream_version release = fullversion._BaseVersion__debian_revision if release is None: release = '' progress_update_s.send(sender=None, index=i + 1) package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='D') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def download(self, target): dsc = os.path.basename(self.source) dsc_path = os.path.join(target, dsc) # Download .dsc file self.log.debug('Downloading dsc file: %s' % self.source) tmp, headers = urlretrieve(self.source) shutil.move(tmp, dsc_path) # Download required files src = Sources(open(dsc_path)) files = [info['name'] for info in src['Files']] for fname in files: url = os.path.join(self.base, fname) self.log.debug('Downloading source file: %s' % url) tmp, headers = urlretrieve(url) path = os.path.join(target, fname) shutil.move(tmp, path) self.log.debug('All files downloaded') return dsc
def page_request(self, suite: Suite, component: Component) -> DebianPageType: """Return parsed package Sources file for a given debian suite and component.""" for url, compression in self.debian_index_urls(suite, component): response = requests.get(url, stream=True) logging.debug("Fetched URL: %s, status code: %s", url, response.status_code) if response.status_code == 200: last_modified = response.headers.get("Last-Modified") self.last_sources_update = ( parsedate_to_datetime(last_modified) if last_modified else None) decompressor = decompressors.get(compression) if decompressor: data = decompressor(response.raw).readlines() else: data = response.raw.readlines() break else: data = "" logger.debug("Could not retrieve sources index for %s/%s", suite, component) return Sources.iter_paragraphs(data)
def download(self): # Prepare source package builder from irgsh.source import SourcePackageBuilder spec = self.spec srcpkg = SourcePackageBuilder( spec.source, spec.source_type, spec.source_opts, spec.orig, spec.extraorig_set.all().values_list('orig', flat=True)) orig_path = None logger = None try: # Download source and build source package build_dir = tempfile.mkdtemp('-irgsh-build') source_dir = tempfile.mkdtemp('-irgsh-build-source') # Prepare logger logdir = os.path.dirname(self.spec.source_log_path()) if not os.path.exists(logdir): os.makedirs(logdir) logname = os.path.join(logdir, 'source.log') logger = open(logname, 'wb') # Build source package spec.add_log(_('Downloading source code')) self.log.debug('[%s] Downloading source code' % (self.spec_id, )) self.set_status(101) self.dsc = srcpkg.build(build_dir, logger) spec.add_log(_('Source code downloaded')) self.log.debug('[%s] Source code downloaded' % (self.spec_id, )) # Extract source package and send package description dsc_file = os.path.join(build_dir, self.dsc) source = os.path.join(source_dir, 'source') cmd = 'dpkg-source -x %s %s' % (dsc_file, source) logger.write('# Extracting source package\n') logger.write('# Command: %s\n' % cmd) logger.flush() p = Popen(cmd.split(), stdout=logger, stderr=STDOUT) p.communicate() logger.write('\n') changelog = os.path.join(source, 'debian', 'changelog') control = os.path.join(source, 'debian', 'control') self.send_description(changelog, control) # Copy source packages logger.write('# Copying source files:\n') src = Sources(open(dsc_file)) files = [self.dsc] + [info['name'] for info in src['Files']] for fname in files: logger.write('# - %s\n' % os.path.basename(fname)) target = os.path.join(self.target, fname) if os.path.exists(target): os.unlink(target) shutil.move(os.path.join(build_dir, fname), target) os.chmod(target, 0644) return files except StandardError, e: import traceback traceback.print_stack() logger.write('# Exception happened: %s: %s' % (type(e), str(e))) send_tweet(self.spec, 'Failed to initialize') raise
def get_sources(self, dist, component): request = requests.get(self._get_source_url(dist, component)) data = io.BytesIO(request.content) stream = gzip.GzipFile(fileobj=data) yield from (Upload(x, self) for x in Sources.iter_paragraphs(stream))
elif ( f.endswith('/sources.deb822.gz') or f.endswith('.sources.deb822.gz') ): source_lists.add(f) else: runtime_package_lists.add(f) sources = {} # type: typing.Dict[str, typing.Dict[str, Version]] for f in source_lists: with GzipFile(f, 'rb') as gzip_reader: for source_stanza in Sources.iter_paragraphs( sequence=gzip_reader, encoding='utf-8', ): source = Source( source_stanza['package'], Version(source_stanza['version']), stanza=source_stanza, ) sources.setdefault(source.name, {})[source.version] = source for f in runtime_package_lists: test.diag('Examining runtime %s...' % f) with GzipFile(f, 'rb') as gzip_reader: for binary_stanza in Packages.iter_paragraphs( sequence=gzip_reader, encoding='utf-8', ):
def fetch_sources(self): fp = self.fetch_indexed_file('Sources') return Sources.iter_paragraphs(fp, use_apt_pkg=False)
def build(self, target, logger=None): '''Build source package. This function returns the .dsc filename ''' try: cwd = os.getcwd() build_path = tempfile.mkdtemp('-irgsh-srcpkg') # Prepare source directory self.slog(logger, '# Preparing source code directory') package, version = self.prepare_source(build_path, logger) source = '%s-%s' % (package, version) self.slog(logger) os.chdir(build_path) self.slog(logger, '# File listing') cmd = 'find -ls' p = Popen(cmd.split(), stdout=logger, stderr=STDOUT, preexec_fn=os.setsid) out, err = p.communicate() self.slog(logger) # Build self.log.debug('Building source package: ' \ 'source=%s type=%s opts=%s orig=%s extra_orig=%s' % \ (self.source, self.source_type, \ self.source_opts, self.orig, self.extra_orig)) self.slog(logger, '# Building source package') cmd = 'dpkg-source -b %s' % source self.slog(logger, '# Command:', cmd) p = Popen(cmd.split(), stdout=logger, stderr=STDOUT, preexec_fn=os.setsid) out, err = p.communicate() self.slog(logger, '# Return code:', p.returncode, '\n') if p.returncode != 0: raise SourcePackageBuildError(p.returncode, out, err) # Move result to the given target directory, # existing files will be replaced dsc = '%s_%s.dsc' % (package, version) files = [dsc] dsc_path = os.path.join(build_path, dsc) src = Sources(open(dsc_path)) if not src.has_key('Files'): raise KeyError, 'Invalid source package' files += [item['name'] for item in src['Files']] self.log.debug('Moving source package files: %s' % ', '.join(files)) for fname in files: target_path = os.path.join(target, fname) if os.path.exists(target_path): os.unlink(target_path) shutil.move(os.path.join(build_path, fname), target_path) os.chmod(target_path, 0644) self.log.debug('Source package built: %s' % dsc) return dsc finally: shutil.rmtree(build_path) os.chdir(cwd)
def _queue_sources_gz_pkgs(self, path): sources = os.path.join(path, 'Sources') with open(sources) as sources_file: for src in Sources.iter_paragraphs(sources_file): package = self.package_to_enqueue(src) super().watch(**package)