def generate_control_file(self): dsc = Dsc() build_deps = [] packages = OrderedDict() self.installs = {} self.symlinks = {} for item in self.generate_control_content(): log.debug(repr(item)) if isinstance(item, SourceControl): for key, value in item.items(): key = '-'.join([k.capitalize() for k in key.split('_')]) if key == 'Description': value = value.strip() \ .replace('\n\n', '\n.\n') \ .replace('\n', '\n ') dsc[key] = value elif isinstance(item, BuildDependency): build_deps.append(item.dependency) elif isinstance(item, Package): control = Deb822() for key, value in item._asdict().items(): key = '-'.join([k.capitalize() for k in key.split('_')]) if key == 'Description': value = value.strip() \ .replace('\n\n', '\n.\n') \ .replace('\n', '\n ') control[key] = value packages[item.package] = (control, [], []) self.installs[item.package] = [] self.symlinks[item.package] = [] elif isinstance(item, Dependency): packages[item.package][1].append(item.dependency) elif isinstance(item, Provide): packages[item.package][2].append(item.provide) elif isinstance(item, Symlink): self.symlinks[item.package].append((item.dest, item.src)) elif isinstance(item, FastBuild): if self.fast_build is None: self.fast_build = item.possible else: self.fast_build = self.fast_build and item.possible else: raise NotImplementedError( 'Got unexcepted action item' ' on control generation {!r}'.format(item)) if self.installs: build_deps.append('dh-exec') with open(self.debian_file('control'), 'wb') as control_file: dsc['Build-Depends'] = ', '.join(build_deps) dsc.dump(control_file) for control, deps, provides in packages.values(): if deps: control['Depends'] = ', '.join(deps) if provides: control['Provides'] = ', '.join(provides) control_file.write(b'\n') control.dump(control_file)
def recognizes(cls, file): if not super().recognizes(file): return False with open(file.path, 'rb') as f: dsc = Dsc(f) for d in dsc.get('Files'): md5 = hashlib.md5() # XXX: this will not work for containers in_dsc_path = os.path.join( os.path.dirname(file.path), d['Name'], ) if not os.path.exists(in_dsc_path): return False with open(in_dsc_path, 'rb') as f: for buf in iter(functools.partial(f.read, 32768), b''): md5.update(buf) if md5.hexdigest() != d['md5sum']: return False file._deb822 = dsc return True
def recognizes(cls, file): if not super().recognizes(file): return False with open(file.path, 'rb') as f: # We can parse .buildinfo files just like .dsc buildinfo = Dsc(f) if 'Checksums-Sha256' not in buildinfo: return False for d in buildinfo.get('Checksums-Sha256'): sha256 = hashlib.sha256() # XXX: this will not work for containers in_buildinfo_path = os.path.join( os.path.dirname(file.path), d['Name'], ) if not os.path.exists(in_buildinfo_path): return False with open(in_buildinfo_path, 'rb') as f: for buf in iter(functools.partial(f.read, 32768), b''): sha256.update(buf) if sha256.hexdigest() != d['sha256']: return False file._deb822 = buildinfo return True
def recognizes(file): if not DotDscFile.RE_FILE_EXTENSION.search(file.name): return False with open(file.path, 'rb') as f: dsc = Dsc(f) for d in dsc.get('Files'): md5 = hashlib.md5() # XXX: this will not work for containers in_dsc_path = os.path.join(os.path.dirname(file.path), d['Name']) if not os.path.exists(in_dsc_path): return False with open(in_dsc_path, 'rb') as f: for buf in iter(partial(f.read, 32768), b''): md5.update(buf) if md5.hexdigest() != d['md5sum']: return False file._deb822 = dsc return True
async def put_dsc_package(host: str, user: str, password: str, dist: str, file: str): with open(file) as fh: dsc = Dsc(fh) dir: str = os.path.dirname(file) files: List[str] = [dir + "/" + file["name"] for file in dsc["files"]] await post_package_multipart(f"{host}/includedsc/{dist}", "dsc", file, files, user, password)
def recognizes(file): if not DotBuildinfoFile.RE_FILE_EXTENSION.search(file.name): return False with open(file.path, 'rb') as f: # We can parse .buildinfo just like .dsc buildinfo = Dsc(f) if not 'Checksums-Sha256' in buildinfo: return False for d in buildinfo.get('Checksums-Sha256'): sha256 = hashlib.sha256() # XXX: this will not work for containers in_buildinfo_path = os.path.join(os.path.dirname(file.path), d['Name']) if not os.path.exists(in_buildinfo_path): return False with open(in_buildinfo_path, 'rb') as f: for buf in iter(partial(f.read, 32768), b''): sha256.update(buf) if sha256.hexdigest() != d['sha256']: return False file._deb822 = buildinfo return True
def getBaseDsc(self): dsc = Dsc() dsc["Architecture"] = "all" dsc["Version"] = "0.42" dsc["Source"] = "dulwich" dsc["Binary"] = "python-dulwich" dsc["Standards-Version"] = "0.2.2" dsc["Maintainer"] = "Jelmer Vernooij <*****@*****.**>" dsc["Files"] = [{ "md5sum": "5e8ba79b4074e2f305ddeaf2543afe83", "size": "182280", "name": "dulwich_0.42.tar.gz" }] return dsc
def make_dsc_file_available(self, filename, owner=None): d = os.path.dirname(filename) or os.curdir with open(filename) as reader: dsc = Dsc(reader) to = self.new_directory() self.argv.append('--copy={}:{}'.format( filename, '{}/{}'.format(to, os.path.basename(filename)))) for f in dsc['files']: self.argv.append('--copy={}:{}'.format( os.path.join(d, f['name']), '{}/{}'.format(to, f['name']))) return to, os.path.basename(filename)
def make_dsc_file_available(self, filename, owner=None): d = os.path.dirname(filename) or os.curdir with open(filename) as reader: dsc = Dsc(reader) to = self.new_directory() files = [to, '{}/{}'.format(to, os.path.basename(filename))] self.copy_to_guest(filename, files[-1]) for f in dsc['files']: files.append('{}/{}'.format(to, f['name'])) self.copy_to_guest(os.path.join(d, f['name']), files[-1]) if owner is not None: self.check_call(['chown', owner] + files) return files[0], os.path.basename(filename)
def get_package_metadata(package, extracted_path, keyrings, log=None): """Get the package metadata from the source package at dsc_path, extracted in extracted_path. Args: package: the package dict (with a dsc_path key) extracted_path: the path where the package got extracted keyrings: a list of keyrings to use for gpg actions log: a logging.Logger object Returns: a dict with the following keys history: list of (package_name, package_version) tuples parsed from the package changelog source_files: information about all the files in the source package """ ret = {} # Parse the dsc file to retrieve all the original artifact files dsc_path = package['dsc'] with open(dsc_path, 'rb') as dsc: parsed_dsc = Dsc(dsc) source_files = [get_file_info(dsc_path)] dsc_dir = os.path.dirname(dsc_path) for file in parsed_dsc['files']: file_path = os.path.join(dsc_dir, file['name']) file_info = get_file_info(file_path) source_files.append(file_info) ret['original_artifact'] = source_files # Parse the changelog to retrieve the rest of the package information changelog_path = os.path.join(extracted_path, b'debian/changelog') with open(changelog_path, 'rb') as changelog: try: parsed_changelog = Changelog(changelog) except UnicodeDecodeError: if log: log.warn('Unknown encoding for changelog %s,' ' falling back to iso' % changelog_path.decode('utf-8'), extra={ 'swh_type': 'deb_changelog_encoding', 'swh_name': package['name'], 'swh_version': str(package['version']), 'swh_changelog': changelog_path.decode('utf-8'), }) # need to reset as Changelog scrolls to the end of the file changelog.seek(0) parsed_changelog = Changelog(changelog, encoding='iso-8859-15') package_info = { 'name': package['name'], 'version': str(package['version']), 'lister_metadata': { 'lister': 'snapshot.debian.org', 'id': package['id'], }, 'changelog': { 'person': converters.uid_to_person(parsed_changelog.author), 'date': parse_date(parsed_changelog.date), 'history': [(block.package, str(block.version)) for block in parsed_changelog][1:], } } try: gpg_info = parsed_dsc.get_gpg_info(keyrings=keyrings) package_info['pgp_signature'] = get_gpg_info_signature(gpg_info) except ValueError: if log: log.info('Could not get PGP signature on package %s_%s' % (package['name'], package['version']), extra={ 'swh_type': 'deb_missing_signature', 'swh_name': package['name'], 'swh_version': str(package['version']), }) package_info['pgp_signature'] = None maintainers = [ converters.uid_to_person(parsed_dsc['Maintainer'], encode=False), ] maintainers.extend( converters.uid_to_person(person, encode=False) for person in UPLOADERS_SPLIT.split(parsed_dsc.get('Uploaders', '')) ) package_info['maintainers'] = maintainers ret['package_info'] = package_info return ret
def _autopkgtest(things, *, architecture, built_binaries, lxc_24bit_subnet, lxc_worker, mirrors, modes, qemu_ram_size, schroot_worker, storage, suite, vendor, worker, extra_repositories=()): binaries = [] sources = [] for thing in things: if os.path.exists(thing): if thing.endswith('.changes'): with open(thing) as reader: c = Changes(reader) for f in c['files']: n = os.path.join( os.path.dirname(thing) or os.curdir, f['name']) if f['name'].endswith('.deb'): binaries.append(n) elif f['name'].endswith('.dsc'): sources.append(Source(n, dsc=Dsc(open(n)))) elif thing.endswith('.dsc'): sources.append(Source(thing, dsc=Dsc(open(thing)))) elif thing.endswith('.deb'): binaries.append(thing) else: sources.append(Source(thing)) failures = set() for source in sources: source_dsc = None source_package = None if source.dsc is not None: source_dsc = source.name else: source_package = source.name if built_binaries is None: built_binaries = not binaries for failure in run_autopkgtest( architecture=architecture, binaries=binaries, built_binaries=built_binaries, components=(), extra_repositories=extra_repositories, lxc_24bit_subnet=lxc_24bit_subnet, lxc_worker=lxc_worker, mirrors=mirrors, modes=modes, qemu_ram_size=qemu_ram_size, schroot_worker=schroot_worker, source_dsc=source_dsc, source_package=source_package, storage=storage, suite=suite, vendor=vendor, worker=worker, ): source.failures.append(failure) failures.add(source) return failures
def sbuild(self): self.worker.check_call([ 'install', '-d', '-m755', '-osbuild', '-gsbuild', '{}/out'.format(self.worker.scratch) ]) sbuild_version = Version( self.worker.check_output( ['dpkg-query', '-W', '-f${Version}', 'sbuild'], universal_newlines=True).rstrip('\n')) logger.info('Building architecture: %s', self.arch) if self.arch in ('all', 'source'): logger.info('(on %s)', self.worker.dpkg_architecture) use_arch = self.worker.dpkg_architecture else: use_arch = self.arch hierarchy = self.suite.hierarchy sbuild_tarball = ('sbuild-{vendor}-{base}-{arch}.tar.gz'.format( arch=use_arch, vendor=self.buildable.vendor, base=hierarchy[-1], )) self.worker.copy_to_guest(os.path.join(self.storage, sbuild_tarball), '{}/in/{}'.format(self.worker.scratch, sbuild_tarball), cache=True) chroot = '{base}-{arch}-sbuild'.format(base=hierarchy[-1], arch=use_arch) with TemporaryDirectory() as tmp: with AtomicWriter(os.path.join(tmp, 'sbuild.conf')) as writer: writer.write( textwrap.dedent(''' [{chroot}] type=file description=An autobuilder file={scratch}/in/{sbuild_tarball} groups=root,sbuild root-groups=root,sbuild profile=sbuild ''').format(chroot=chroot, sbuild_tarball=sbuild_tarball, scratch=self.worker.scratch)) self.worker.copy_to_guest( os.path.join(tmp, 'sbuild.conf'), '/etc/schroot/chroot.d/{}'.format(chroot)) # Backwards compatibility goo for Debian jessie buildd backport: # it can't do "sbuild hello", only "sbuild hello_2.10-1" if (self.buildable.source_from_archive and self.buildable.version is None and sbuild_version < Version('0.69.0')): lines = self.worker.check_output( [ 'schroot', '-c', chroot, '--', 'sh', '-c', 'apt-get update >&2 && ' 'apt-cache showsrc --only-source "$1" | ' 'sed -ne "s/^Version: *//p"', 'sh', # argv[0] self.buildable.source_package ], universal_newlines=True).strip().splitlines() self.buildable.version = sorted(map(Version, lines))[-1] self.buildable.buildable = '{}_{}'.format( self.buildable.source_package, self.buildable.version, ) argv = [ self.worker.command_wrapper, '--chdir', '{}/out'.format(self.worker.scratch), '--', 'runuser', '-u', 'sbuild', '--', 'sbuild', '-c', chroot, '-d', str(self.buildable.nominal_suite), '--no-run-lintian', ] for x in self.dpkg_buildpackage_options: argv.append('--debbuildopt=' + x) for x in self.dpkg_source_options: argv.append('--dpkg-source-opt=' + x) for child in hierarchy[:-1]: argv.append('--extra-repository') argv.append('deb {} {} {}'.format( child.mirror, child.apt_suite, ' '.join( set(self.components or child.components) & child.all_components))) if child.sbuild_resolver: argv.extend(child.sbuild_resolver) for x in self.extra_repositories: argv.append('--extra-repository') argv.append(x) if self.arch == 'all': logger.info('Architecture: all') argv.append('-A') # Backwards compatibility goo for Debian jessie buildd backport if sbuild_version < Version('0.69.0'): argv.append('--arch-all-only') else: argv.append('--no-arch-any') elif self.arch == self.buildable.together_with: logger.info('Architecture: %s + all', self.arch) argv.append('-A') argv.append('--arch') argv.append(self.arch) elif self.arch == 'source': logger.info('Source-only') # Backwards compatibility goo for Debian jessie buildd backport if sbuild_version < Version('0.69.0'): # If we only build 'all', and we don't build 'all', # then logically we build nothing (except source). argv.append('--arch-all-only') argv.append('--no-arch-all') # Urgh. This sbuild expects to find foo_1_amd64.changes # even for a source-only build (because it doesn't really # support source-only builds), so we have to cheat. # sbuild splits the command on spaces so we need to have # a one-liner that doesn't contain embedded whitespace. # Luckily, Perl can be written as line-noise. argv.append('--finished-build-commands=perl -e ' + '$arch=qx(dpkg\\x20--print-architecture);' + 'chomp($arch);' + 'chdir(shift);' + 'foreach(glob("../*_source.changes")){' + '$orig=$_;' + 's/_source\\.changes$/_${arch}.changes/;' + 'print("Renaming\\x20$orig\\x20to\\x20$_\\n");' + 'rename($orig,$_)||die("$!");' + '}' + ' %p') else: argv.append('--no-arch-any') argv.append('--source') else: logger.info('Architecture: %s only', self.arch) argv.append('--arch') argv.append(self.arch) if self.buildable.dsc_name is not None: if 'source' in self.buildable.changes_produced: argv.append('{}/out/{}'.format( self.worker.scratch, os.path.basename(self.buildable.dsc_name))) else: argv.append('{}/in/{}'.format( self.worker.scratch, os.path.basename(self.buildable.dsc_name))) elif self.buildable.source_from_archive: argv.append(self.buildable.buildable) else: # Build a clean source package as a side-effect of the first # build (in practice this will be the 'source' build). if '--source' not in argv: argv.append('--source') # jessie sbuild doesn't support --no-clean-source so build # the temporary source package ourselves. self.worker.check_call([ self.worker.command_wrapper, '--chdir', '{}/in/{}_source'.format(self.worker.scratch, self.buildable.product_prefix), '--', 'dpkg-source', '-b', '.' ]) argv.append('{}/in/{}.dsc'.format(self.worker.scratch, self.buildable.product_prefix)) logger.info('Running %r', argv) try: self.worker.check_call(argv) finally: # Note that we mix use_arch and arch here: an Architecture: all # build produces foo_1.2_amd64.build, which we rename. # We also check for foo_amd64.build because # that's what comes out if we do "vectis sbuild --suite=sid hello". for prefix in (self.buildable.source_package, self.buildable.product_prefix): product = '{}/out/{}_{}.build'.format(self.worker.scratch, prefix, use_arch) product = self.worker.check_output( ['readlink', '-f', product], universal_newlines=True).rstrip('\n') if (self.worker.call(['test', '-e', product]) == 0 and self.output_builds is not None): logger.info('Copying %s back to host as %s_%s.build...', product, self.buildable.product_prefix, self.arch) copied_back = os.path.join( self.output_builds, '{}_{}_{}.build'.format( self.buildable.product_prefix, self.arch, time.strftime('%Y%m%dt%H%M%S', time.gmtime()))) self.worker.copy_to_host(product, copied_back) self.buildable.logs[self.arch] = copied_back symlink = os.path.join( self.output_builds, '{}_{}.build'.format(self.buildable.product_prefix, self.arch)) try: os.remove(symlink) except FileNotFoundError: pass os.symlink(os.path.abspath(copied_back), symlink) break else: logger.warning('Did not find build log at %s', product) logger.warning( 'Possible build logs:\n%s', self.worker.check_call([ 'sh', '-c', 'cd "$1"; ls -l *.build || :', 'sh', # argv[0] self.worker.scratch ])) if self.arch == 'source' and self.buildable.source_from_archive: dscs = self.worker.check_output( [ 'sh', '-c', 'exec ls "$1"/out/*.dsc', 'sh', # argv[0] self.worker.scratch ], universal_newlines=True) dscs = dscs.splitlines() if len(dscs) != 1: raise CannotHappen('sbuild --source produced more than one ' '.dsc file from {!r}'.format( self.buildable)) product = dscs[0] with TemporaryDirectory() as tmp: copied_back = os.path.join( tmp, '{}.dsc'.format(self.buildable.buildable)) self.worker.copy_to_host(product, copied_back) self.buildable.dsc = Dsc(open(copied_back)) self.buildable.source_package = self.buildable.dsc['source'] self.buildable.version = Version(self.buildable.dsc['version']) self.buildable.arch_wildcards = set( self.buildable.dsc['architecture'].split()) self.buildable.binary_packages = [ p.strip() for p in self.buildable.dsc['binary'].split(',') ] if self.arch == 'source' and self.output_builds is not None: # Make sure the orig.tar.* are in the out directory, because # we will be building from the rebuilt source in future self.worker.check_call([ 'sh', '-c', 'ln -nsf "$1"/in/*.orig.tar.* "$1"/out/', 'sh', # argv[0] self.worker.scratch ]) if self.output_builds is None: return for product_arch in (self.arch, self.worker.dpkg_architecture): product = '{}/out/{}_{}.changes'.format( self.worker.scratch, self.buildable.product_prefix, product_arch) if self.worker.call(['test', '-e', product]) == 0: break else: raise CannotHappen('sbuild produced no .changes file from ' '{!r}'.format(self.buildable)) logger.info('Copying %s back to host...', product) copied_back = os.path.join( self.output_builds, '{}_{}.changes'.format(self.buildable.product_prefix, self.arch)) self.worker.copy_to_host(product, copied_back) self.buildable.changes_produced[self.arch] = copied_back changes_out = Changes(open(copied_back)) if self.arch == 'source': self.buildable.dsc_name = None self.buildable.sourceful_changes_name = copied_back for f in changes_out['files']: if f['name'].endswith('.dsc'): # expect to find exactly one .dsc file assert self.buildable.dsc_name is None self.buildable.dsc_name = os.path.join( self.output_builds, f['name']) assert self.buildable.dsc_name is not None # Save some space self.worker.check_call([ 'rm', '-fr', '{}/in/{}_source/'.format(self.worker.scratch, self.buildable.product_prefix) ]) for f in changes_out['files']: assert '/' not in f['name'] assert not f['name'].startswith('.') logger.info('Additionally copying %s back to host...', f['name']) product = '{}/out/{}'.format(self.worker.scratch, f['name']) copied_back = os.path.join(self.output_builds, f['name']) self.worker.copy_to_host(product, copied_back)
def _sbuild(self, chroot, sbuild_options=()): sbuild_version = self.worker.dpkg_version('sbuild') # Backwards compatibility goo for Debian jessie buildd backport: # it can't do "sbuild hello", only "sbuild hello_2.10-1". if (self.buildable.source_from_archive and self.buildable.source_version is None and sbuild_version < Version('0.69.0')): lines = chroot.check_output( [ 'sh', '-c', 'apt-get update >&2 && ' '( apt-cache showsrc --only-source "$1" || ' ' apt-cache showsrc "$1" ) | ' 'sed -ne "s/^Version: *//p"', 'sh', # argv[0] self.buildable.source_package, ], universal_newlines=True).strip().splitlines() self.buildable.source_version = sorted(map(Version, lines))[-1] self.buildable.buildable = '{}_{}'.format( self.buildable.source_package, self.buildable.source_version, ) argv = [ self.worker.command_wrapper, '--chdir', '{}/out'.format(self.worker.scratch), '--', 'runuser', '-u', 'sbuild', '--', 'env', ] for k, v in sorted(self.environ.items()): argv.append('{}={}'.format(k, v)) argv.extend(( 'sbuild', '-c', chroot.chroot, '-d', str(self.buildable.nominal_suite), '--no-run-lintian', )) if self.profiles: argv.append('--profiles={}'.format(','.join(self.profiles))) for x in self.dpkg_buildpackage_options: argv.append('--debbuildopt=' + x) for child in chroot.suite.hierarchy[:-1]: # The schroot already has the apt sources, we just need the # resolver if child.sbuild_resolver: argv.extend(child.sbuild_resolver) break if self.arch == 'all': logger.info('Architecture: all') argv.append('-A') # Backwards compatibility goo for Debian jessie buildd backport if sbuild_version < Version('0.69.0'): argv.append('--arch-all-only') else: argv.append('--no-arch-any') elif self.arch == self.buildable.indep_together_with: logger.info('Architecture: %s + all', self.arch) argv.append('-A') argv.append('--arch') argv.append(self.arch) elif self.arch == 'source': logger.info('Source-only') argv.append('--no-arch-any') if sbuild_version < Version('0.69.0'): # Backwards compatibility for Debian jessie buildd backport, # and for sbuild in Ubuntu xenial. # sbuild < 0.69.0 expects to find foo_1_amd64.changes # even for a source-only build (because it doesn't really # support source-only builds), so we have to cheat. perl = ("'" + '$arch = qx(dpkg\\x20--print-architecture);\n' + 'chomp($arch);\n' + 'chdir(shift);\n' + 'foreach(glob("../*_source.changes")) {\n' + ' $orig = $_;\n' + ' s/_source\\.changes$/_${arch}.changes/;\n' + ' print("Renaming\\x20$orig\\x20to\\x20$_\\n");\n' + ' rename($orig,$_) || die("$!");\n' + '}\n' + "'") argv.append( '--finished-build-commands=perl -e {} %p'.format(perl)) else: logger.info('Architecture: %s only', self.arch) argv.append('--arch') argv.append(self.arch) if self.arch in ('source', self.buildable.source_together_with): # Build a clean source package as a side-effect of one # build. argv.append('--source') for x in self.dpkg_source_options: argv.append('--debbuildopt=--source-option={}'.format(x)) if self.buildable.binary_version_suffix: argv.append('--append-to-version={}'.format( self.buildable.binary_version_suffix)) for x in sbuild_options: argv.append(x) if self.buildable.dsc_name is not None: if 'source' in self.buildable.changes_produced: # We rebuilt the source already. Use the rebuilt version # for all subsequent builds. argv.append('{}/out/{}'.format( self.worker.scratch, os.path.basename(self.buildable.dsc_name))) else: # We got a .dsc from outside Vectis and are not # rebuilding it. argv.append('{}/in/{}'.format( self.worker.scratch, os.path.basename(self.buildable.dsc_name))) elif self.buildable.source_from_archive: argv.append(self.buildable.buildable) else: # jessie sbuild doesn't support --no-clean-source so build # the temporary source package ourselves. ds_argv = [ self.worker.command_wrapper, '--chdir', '{}/in/{}_source'.format(self.worker.scratch, self.buildable.product_prefix), '--', 'dpkg-source', ] for x in self.dpkg_source_options: ds_argv.append(x) ds_argv.extend(('-b', '.')) self.worker.check_call(ds_argv) argv.append('{}/in/{}.dsc'.format(self.worker.scratch, self.buildable.product_prefix)) logger.info('Running %r', argv) try: self.worker.check_call(argv) finally: # Note that we mix chroot.dpkg_architecture and arch here: an # Architecture: all build produces foo_1.2_amd64.build, which we # rename. # We also check for foo_amd64.build because # that's what comes out if we do "vectis sbuild --suite=sid hello". for prefix in (self.buildable.source_package, self.buildable.product_prefix): product = '{}/out/{}_{}.build'.format(self.worker.scratch, prefix, chroot.dpkg_architecture) product = self.worker.check_output( ['readlink', '-f', product], universal_newlines=True).rstrip('\n') if (self.worker.call(['test', '-e', product]) == 0 and self.output_dir is not None): logger.info('Copying %s back to host as %s_%s.build...', product, self.buildable.product_prefix, self.arch) copied_back = os.path.join( self.output_dir, '{}_{}_{}.build'.format( self.buildable.product_prefix, self.arch, time.strftime('%Y%m%dt%H%M%S', time.gmtime()))) self.worker.copy_to_host(product, copied_back) self.buildable.logs[self.arch] = copied_back symlink = os.path.join( self.output_dir, '{}_{}.build'.format(self.buildable.product_prefix, self.arch)) try: os.remove(symlink) except FileNotFoundError: pass os.symlink(os.path.abspath(copied_back), symlink) break else: logger.warning('Did not find build log at %s', product) logger.warning( 'Possible build logs:\n%s', self.worker.check_call([ 'sh', '-c', 'cd "$1"; ls -l *.build || :', 'sh', # argv[0] self.worker.scratch ])) if self.arch == 'source' and self.buildable.source_from_archive: dscs = self.worker.check_output( [ 'sh', '-c', 'exec ls "$1"/out/*.dsc', 'sh', # argv[0] self.worker.scratch ], universal_newlines=True) dscs = dscs.splitlines() if len(dscs) != 1: raise CannotHappen('sbuild --source produced more than one ' '.dsc file from {!r}'.format( self.buildable)) product = dscs[0] with TemporaryDirectory(prefix='vectis-sbuild-') as tmp: copied_back = os.path.join( tmp, '{}.dsc'.format(self.buildable.buildable)) self.worker.copy_to_host(product, copied_back) self.buildable.dsc = Dsc(open(copied_back)) self.buildable.source_package = self.buildable.dsc['source'] self.buildable.source_version = Version( self.buildable.dsc['version']) self.buildable.arch_wildcards = set( self.buildable.dsc['architecture'].split()) self.buildable.binary_packages = [ p.strip() for p in self.buildable.dsc['binary'].split(',') ] if self.arch == 'source' and self.output_dir is not None: # Make sure the orig.tar.* are in the out directory, because # we will be building from the rebuilt source in future self.worker.check_call([ 'sh', '-c', 'ln -nsf "$1"/in/*.orig.tar.* "$1"/out/', 'sh', # argv[0] self.worker.scratch ]) if self.output_dir is None: return product_arch = None for candidate in (self.arch, self.worker.dpkg_architecture): product = '{}/out/{}_{}.changes'.format( self.worker.scratch, self.buildable.product_prefix, candidate) if self.worker.call(['test', '-e', product]) == 0: product_arch = candidate break else: raise CannotHappen( 'sbuild produced no .changes file from {!r}'.format( self.buildable)) copied_back = self.copy_back_product( '{}_{}.changes'.format(self.buildable.product_prefix, product_arch), '{}_{}.changes'.format(self.buildable.product_prefix, self.arch)) if copied_back is not None: self.buildable.changes_produced[self.arch] = copied_back changes_out = Changes(open(copied_back)) if 'source' in changes_out['architecture'].split(): self.buildable.dsc_name = None self.buildable.sourceful_changes_name = copied_back for f in changes_out['files']: if f['name'].endswith('.dsc'): # expect to find exactly one .dsc file assert self.buildable.dsc_name is None self.buildable.dsc_name = os.path.join( self.output_dir, f['name']) assert self.buildable.dsc_name is not None # Save some space self.worker.check_call([ 'rm', '-fr', '{}/in/{}_source/'.format(self.worker.scratch, self.buildable.product_prefix) ]) dsc = None for f in changes_out['files']: copied_back = self.copy_back_product(f['name']) if copied_back is not None and f['name'].endswith('.dsc'): dsc = Dsc(open(copied_back)) if dsc is not None: if self.buildable.dsc is None: self.buildable.dsc = dsc for f in dsc['files']: # The orig.tar.* might not have come back. Copy that too, # if necessary. self.copy_back_product(f['name'], skip_if_exists=True)
def __init__(self, buildable, *, binary_version_suffix='', link_builds=(), orig_dirs=('..', ), output_dir=None, output_parent, vendor): self.buildable = buildable self._product_prefix = None self._source_version = None self.arch_wildcards = set() self.archs = [] self.autopkgtest_failures = [] self.binary_packages = [] self.binary_version_suffix = binary_version_suffix self.changes_produced = {} self.dirname = None self.dsc = None self.dsc_name = None self.indep = False self.indep_together_with = None self.link_builds = link_builds self.logs = {} self.merged_changes = OrderedDict() self.nominal_suite = None self.orig_dirs = orig_dirs self.output_dir = output_dir self.piuparts_failures = [] self.source_from_archive = False self.source_package = None self.source_together_with = None self.sourceful_changes_name = None self.suite = None self.vendor = vendor if os.path.exists(self.buildable): if os.path.isdir(self.buildable): changelog = os.path.join(self.buildable, 'debian', 'changelog') changelog = Changelog(open(changelog)) self.source_package = changelog.get_package() self.nominal_suite = changelog.distributions self._source_version = Version(changelog.version) control = os.path.join(self.buildable, 'debian', 'control') if len(changelog.distributions.split()) != 1: raise ArgumentError( 'Cannot build for multiple distributions at once') for paragraph in Deb822.iter_paragraphs(open(control)): self.arch_wildcards |= set( paragraph.get('architecture', '').split()) binary = paragraph.get('package') if binary is not None: self.binary_packages.append(binary) elif self.buildable.endswith('.changes'): self.dirname = os.path.dirname(self.buildable) or os.curdir self.sourceful_changes_name = self.buildable sourceful_changes = Changes(open(self.buildable)) if 'source' not in sourceful_changes['architecture'].split(): raise ArgumentError( 'Changes file {!r} must be sourceful'.format( self.buildable)) self.nominal_suite = sourceful_changes['distribution'] for f in sourceful_changes['files']: if f['name'].endswith('.dsc'): if self.dsc_name is not None: raise ArgumentError( 'Changes file {!r} contained more than one ' '.dsc file'.format(self.buildable)) self.dsc_name = os.path.join(self.dirname, f['name']) if self.dsc_name is None: raise ArgumentError( 'Changes file {!r} did not contain a .dsc file'.format( self.buildable)) self.dsc = Dsc(open(self.dsc_name)) elif self.buildable.endswith('.dsc'): self.dirname = os.path.dirname(self.buildable) or os.curdir self.dsc_name = self.buildable self.dsc = Dsc(open(self.dsc_name)) else: raise ArgumentError( 'buildable must be .changes, .dsc or directory, not ' '{!r}'.format(self.buildable)) else: self.source_from_archive = True if '_' in self.buildable: source, version = self.buildable.split('_', 1) else: source = self.buildable version = None self.source_package = source if version is not None: self._source_version = Version(version) if self.dsc is not None: self.source_package = self.dsc['source'] self._source_version = Version(self.dsc['version']) self.arch_wildcards = set(self.dsc['architecture'].split()) self.binary_packages = [ p.strip() for p in self.dsc['binary'].split(',') ] if self._source_version is not None: self._binary_version = Version( str(self._source_version) + self.binary_version_suffix) timestamp = time.strftime('%Y%m%dt%H%M%S', time.gmtime()) if self.output_dir is None: if self._binary_version is None: dirname = '{}_{}'.format(self.source_package, timestamp) else: dirname = '{}_{}_{}'.format(self.source_package, self._binary_version, timestamp) self.output_dir = os.path.join(output_parent, dirname) # For convenience, create a symbolic link for the latest build of # each source package: hello_latest -> hello_2.10-1_20170319t102623 unversioned_symlink = os.path.join(output_parent, self.source_package + '_latest') with suppress(FileNotFoundError): os.unlink(unversioned_symlink) os.symlink(dirname, unversioned_symlink) # If we know the version, also create a symbolic link for the # latest build of each source/version pair: # hello_2.10-1 -> hello_2.10-1_20170319t102623 if self._binary_version is not None: versioned_symlink = os.path.join( output_parent, '{}_{}'.format(self.source_package, self._binary_version)) with suppress(FileNotFoundError): os.unlink(versioned_symlink) os.symlink(dirname, versioned_symlink) # It's OK if the output directory exists but is empty. with suppress(FileNotFoundError): os.rmdir(self.output_dir) # Otherwise, if someone already created this, we'll just crash out. os.mkdir(self.output_dir) if self.dsc is not None: abs_file = os.path.abspath(self.dsc_name) abs_dir, base = os.path.split(abs_file) os.symlink(abs_file, os.path.join(self.output_dir, base)) for l in self.link_builds: symlink = os.path.join(l, base) with suppress(FileNotFoundError): os.unlink(symlink) os.symlink(abs_file, symlink) for f in self.dsc['files']: abs_file = os.path.join(abs_dir, f['name']) os.symlink(abs_file, os.path.join(self.output_dir, f['name'])) for l in self.link_builds: symlink = os.path.join(l, f['name']) with suppress(FileNotFoundError): os.unlink(symlink) os.symlink(abs_file, symlink)
def get_package_metadata(package, dsc_path, extracted_path): """Get the package metadata from the source package at dsc_path, extracted in extracted_path. Args: package: the package dict (with a dsc_path key) dsc_path: path to the package's dsc file extracted_path: the path where the package got extracted Returns: dict: a dictionary with the following keys: - history: list of (package_name, package_version) tuples parsed from the package changelog - source_files: information about all the files in the source package """ ret = {} with open(dsc_path, 'rb') as dsc: parsed_dsc = Dsc(dsc) source_files = [get_file_info(dsc_path)] dsc_dir = os.path.dirname(dsc_path) for filename in package['files']: file_path = os.path.join(dsc_dir, filename) file_info = get_file_info(file_path) source_files.append(file_info) ret['original_artifact'] = source_files # Parse the changelog to retrieve the rest of the package information changelog_path = os.path.join(extracted_path, 'debian/changelog') with open(changelog_path, 'rb') as changelog: try: parsed_changelog = Changelog(changelog) except UnicodeDecodeError: log.warn('Unknown encoding for changelog %s,' ' falling back to iso' % changelog_path.decode('utf-8'), extra={ 'swh_type': 'deb_changelog_encoding', 'swh_name': package['name'], 'swh_version': str(package['version']), 'swh_changelog': changelog_path.decode('utf-8'), }) # need to reset as Changelog scrolls to the end of the file changelog.seek(0) parsed_changelog = Changelog(changelog, encoding='iso-8859-15') package_info = { 'name': package['name'], 'version': str(package['version']), 'changelog': { 'person': converters.uid_to_person(parsed_changelog.author), 'date': parse_date(parsed_changelog.date), 'history': [(block.package, str(block.version)) for block in parsed_changelog][1:], } } maintainers = [ converters.uid_to_person(parsed_dsc['Maintainer'], encode=False), ] maintainers.extend( converters.uid_to_person(person, encode=False) for person in UPLOADERS_SPLIT.split(parsed_dsc.get('Uploaders', '')) ) package_info['maintainers'] = maintainers ret['package_info'] = package_info return ret
def __init__(self, buildable, *, vendor): self.buildable = buildable self._product_prefix = None self.arch_wildcards = set() self.archs = [] self.binary_packages = [] self.changes_produced = {} self.dirname = None self.dsc = None self.dsc_name = None self.indep = False self.logs = {} self.merged_changes = OrderedDict() self.nominal_suite = None self.source_from_archive = False self.source_package = None self.sourceful_changes_name = None self.suite = None self.together_with = None self.vendor = vendor self._version = None if os.path.exists(self.buildable): if os.path.isdir(self.buildable): changelog = os.path.join(self.buildable, 'debian', 'changelog') changelog = Changelog(open(changelog)) self.source_package = changelog.get_package() self.nominal_suite = changelog.distributions self._version = Version(changelog.version) control = os.path.join(self.buildable, 'debian', 'control') if len(changelog.distributions.split()) != 1: raise ArgumentError('Cannot build for multiple ' 'distributions at once') for paragraph in Deb822.iter_paragraphs(open(control)): self.arch_wildcards |= set( paragraph.get('architecture', '').split()) binary = paragraph.get('package') if binary is not None: self.binary_packages.append(binary) elif self.buildable.endswith('.changes'): self.dirname = os.path.dirname(self.buildable) self.sourceful_changes_name = self.buildable sourceful_changes = Changes(open(self.buildable)) if 'source' not in sourceful_changes['architecture']: raise ArgumentError('Changes file {!r} must be ' 'sourceful'.format(self.buildable)) self.nominal_suite = sourceful_changes['distribution'] for f in sourceful_changes['files']: if f['name'].endswith('.dsc'): if self.dsc_name is not None: raise ArgumentError('Changes file {!r} contained ' 'more than one .dsc ' 'file'.format(self.buildable)) self.dsc_name = os.path.join(self.dirname, f['name']) if self.dsc_name is None: raise ArgumentError('Changes file {!r} did not contain a ' '.dsc file'.format(self.buildable)) self.dsc = Dsc(open(self.dsc_name)) elif self.buildable.endswith('.dsc'): self.dirname = os.path.dirname(self.buildable) self.dsc_name = self.buildable self.dsc = Dsc(open(self.dsc_name)) else: raise ArgumentError('buildable must be .changes, .dsc or ' 'directory, not {!r}'.format( self.buildable)) else: self.source_from_archive = True if '_' in self.buildable: source, version = self.buildable.split('_', 1) else: source = self.buildable version = None self.source_package = source if version is not None: self._version = Version(version) if self.dsc is not None: self.source_package = self.dsc['source'] self._version = Version(self.dsc['version']) self.arch_wildcards = set(self.dsc['architecture'].split()) self.binary_packages = [ p.strip() for p in self.dsc['binary'].split(',') ]
try: dsc_name = [name for name in filelist if name.endswith(".dsc")][0] except IndexError: # raise SourceError("No dsc file found") return [] try: dsc = self.obs.getFile(action["sourceproject"], action["sourcepackage"], dsc_name, action["sourcerevision"]) except Exception, exobj: raise SourceError( "Failed to fetch dsc file %s/%s/%s rev %s: %s" % (action["sourceproject"], action["sourcepackage"], dsc_name, action["sourcerevision"], exobj)) try: dsc = Dsc(dsc) sources = [fentry["name"] for fentry in dsc["files"]] except Exception, exobj: raise SourceError("Failed to parse dsc file: %s" % exobj) return sources @CheckActionProcessor("check_package_is_complete_sources") def check_source_files(self, action, _wid, filelist): """Check that filelist and spec sources match""" sources = set() msg = "" try: sources.update(self.get_rpm_sources(action, filelist)) except SourceError, exobj: msg += str(exobj) try:
def get_intrinsic_package_metadata( p_info: DebianPackageInfo, dsc_path: str, extracted_path: str) -> IntrinsicPackageMetadata: """Get the package metadata from the source package at dsc_path, extracted in extracted_path. Args: p_info: the package information dsc_path: path to the package's dsc file extracted_path: the path where the package got extracted Returns: dict: a dictionary with the following keys: - history: list of (package_name, package_version) tuples parsed from the package changelog """ with open(dsc_path, "rb") as dsc: parsed_dsc = Dsc(dsc) # Parse the changelog to retrieve the rest of the package information changelog_path = path.join(extracted_path, "debian/changelog") with open(changelog_path, "rb") as changelog_file: try: parsed_changelog = Changelog(changelog_file) except UnicodeDecodeError: logger.warning( "Unknown encoding for changelog %s," " falling back to iso" % changelog_path, extra={ "swh_type": "deb_changelog_encoding", "swh_name": p_info.name, "swh_version": str(p_info.version), "swh_changelog": changelog_path, }, ) # need to reset as Changelog scrolls to the end of the file changelog_file.seek(0) parsed_changelog = Changelog(changelog_file, encoding="iso-8859-15") history: List[Tuple[str, str]] = [] for block in parsed_changelog: assert block.package is not None history.append((block.package, str(block.version))) changelog = DebianPackageChangelog( person=uid_to_person(parsed_changelog.author), date=parse_date(parsed_changelog.date).isoformat(), history=history[1:], ) maintainers = [ uid_to_person(parsed_dsc["Maintainer"]), ] maintainers.extend( uid_to_person(person) for person in UPLOADERS_SPLIT.split(parsed_dsc.get("Uploaders", ""))) return IntrinsicPackageMetadata( name=p_info.name, version=str(p_info.intrinsic_version), changelog=changelog, maintainers=maintainers, )