def test_gpg_info(self): if not (os.path.exists('/usr/bin/gpgv') and os.path.exists('/usr/share/keyrings/debian-keyring.gpg')): return unparsed_with_gpg = SIGNED_CHECKSUM_CHANGES_FILE % CHECKSUM_CHANGES_FILE deb822_from_str = deb822.Dsc(unparsed_with_gpg) result_from_str = deb822_from_str.get_gpg_info() deb822_from_file = deb822.Dsc(StringIO(unparsed_with_gpg)) result_from_file = deb822_from_file.get_gpg_info() deb822_from_lines = deb822.Dsc(unparsed_with_gpg.splitlines()) result_from_lines = deb822_from_lines.get_gpg_info() valid = { 'GOODSIG': ['D14219877A786561', 'John Wright <*****@*****.**>'], 'VALIDSIG': [ '8FEFE900783CF175827C2F65D14219877A786561', '2008-05-01', '1209623566', '0', '3', '0', '17', '2', '01', '8FEFE900783CF175827C2F65D14219877A786561' ], 'SIG_ID': ['j3UjSpdky92fcQISbm8W5PlwC/g', '2008-05-01', '1209623566'], } for result in result_from_str, result_from_file, result_from_lines: # The second part of the GOODSIG field could change if the primary # uid changes, so avoid checking that. Also, the first part of the # SIG_ID field has undergone at least one algorithm changein gpg, # so don't bother testing that either. self.assertEqual(set(result.keys()), set(valid.keys())) self.assertEqual(result['GOODSIG'][0], valid['GOODSIG'][0]) self.assertEqual(result['VALIDSIG'], valid['VALIDSIG']) self.assertEqual(result['SIG_ID'][1:], valid['SIG_ID'][1:])
def _extract(self): """ Copy the files to a temporary directory and run dpkg-source -x on the dsc file to extract them. """ log.debug('Copying files to a temp directory to run dpkg-source -x on the dsc file') self.tempdir = tempfile.mkdtemp() log.debug('Temp dir is: %s', self.tempdir) for filename in self.changes.get_files(): log.debug('Copying: %s', filename) shutil.copy(os.path.join(self.config['debexpo.upload.incoming'], filename), self.tempdir) # If the original tarball was pulled from Debian or from the repository, that # also needs to be copied into this directory. dsc = deb822.Dsc(file(self.changes.get_dsc())) for item in dsc['Files']: if item['name'] not in self.changes.get_files(): src_file = os.path.join(self.config['debexpo.upload.incoming'], item['name']) repository_src_file = os.path.join(self.config['debexpo.repository'], self.changes.get_pool_path(), item['name']) if os.path.exists(src_file): shutil.copy(src_file, self.tempdir) elif os.path.exists(repository_src_file): shutil.copy(repository_src_file, self.tempdir) else: log.critical("Trying to copy non-existing file %s" % (src_file)) shutil.copy(os.path.join(self.config['debexpo.upload.incoming'], self.changes_file), self.tempdir) self.oldcurdir = os.path.abspath(os.path.curdir) os.chdir(self.tempdir) os.system('dpkg-source -x %s extracted' % self.changes.get_dsc())
def test_control_fields(self): """ Checks whether additional debian/control fields are present. """ log.debug( 'Checking whether additional debian/control fields are present') try: dsc = deb822.Dsc(file(self.changes.get_dsc())) except: log.critical('Could not open dsc file; skipping plugin') return data = {} severity = constants.PLUGIN_SEVERITY_WARNING outcome = "No Homepage field present" for item in fields: if item in dsc: data[item] = dsc[item] if "Homepage" in data: severity = constants.PLUGIN_SEVERITY_INFO if len(data) > 1: outcome = "Homepage and VCS control fields present" else: outcome = "Homepage control field present" self.failed(outcome, data, severity)
def test_multivalued_field_contains_newline(self): """Multivalued field components are not allowed to contain newlines""" d = deb822.Dsc() # We don't check at set time, since one could easily modify the list # without deb822 knowing. We instead check at get time. d['Files'] = [{'md5sum': 'deadbeef', 'size': '9605', 'name': 'bad\n'}] self.assertRaises(ValueError, d.get_as_string, 'files')
def forge_changes_file(fname, dist, **kwargs): dsc = deb822.Dsc(open(fname, 'r')) changes = deb822.Changes() changes['Format'] = '1.8' changes['Date'] = email.utils.formatdate(time.mktime( dt.datetime.utcnow().timetuple()), usegmt=True) for key in [ 'Source', 'Version', 'Maintainer', 'Checksums-Sha1', 'Checksums-Sha256', 'Files' ]: if not dsc.has_key(key): raise MissingChangesFieldException(key) changes[key] = dsc[key] for algo, key, h, s, f in file_info(fname): if algo == 'md5': algo = 'md5sum' entry = deb822.Deb822Dict() entry[algo] = h entry['size'] = s entry['name'] = f changes[key].append(entry) for entry in changes['Files']: entry['section'] = 'not-implemented' entry['priority'] = 'not-implemented' changes['Distribution'] = dist changes['Urgency'] = 'low' changes['Changed-By'] = 'Archive Rebuilder <*****@*****.**>' changes['Architecture'] = 'source' changes['Binary'] = 'not implemented either' changes['Description'] = """This feature is not implemented. This is a pretty damn hard to deal with right now. I might write this later.""" changes['Changes'] = """ {source} ({version}) {dist}; urgency={urgency} . * This is a fake ChangeLog entry used by ricky to force a rebuild on debuild.me.""".format( source=changes['Source'], version=changes['Version'], urgency=changes['Urgency'], dist=dist, ) for k, v in kwargs.items(): changes[k] = v return changes
def test_maintainer_email(self): """ Tests whether the maintainer email is the same as the uploader email. """ if self.user_id is not None: log.debug( 'Checking whether the maintainer email is the same as the uploader email' ) user = meta.session.query(User).get(self.user_id) if user is not None: maintainer_name, maintainer_email = email.utils.parseaddr( self.changes['Maintainer']) uploader_emails = [] dsc = deb822.Dsc(file(self.changes.get_dsc())) if 'Uploaders' in dsc: for uploader_name, uploader_email in email.utils.getaddresses( [dsc['Uploaders']]): uploader_emails.append(uploader_email) severity = constants.PLUGIN_SEVERITY_INFO if user.email == maintainer_email: log.debug('"Maintainer" email is the same as the uploader') outcome = '"Maintainer" email is the same as the uploader' elif user.email in uploader_emails: log.debug( 'The uploader is in the package\'s "Uploaders" field') outcome = 'The uploader is in the package\'s "Uploaders" field' else: log.warning('%s != %s' % (user.email, maintainer_email)) outcome = 'The uploader is not in the package\'s "Maintainer" or "Uploaders" fields' severity = constants.PLUGIN_SEVERITY_WARNING data = { 'user-is-maintainer': (severity == constants.PLUGIN_SEVERITY_INFO), 'user-email': user.email, 'maintainer-email': maintainer_email, 'uploader-emails': uploader_emails, } self.failed(outcome, data, severity) else: log.warning( 'Could not get the uploader\'s user details from the database')
def _dsc_to_sources(self, package_file): """ Reads the contents of a dsc file and converts it to a Sources file entry. ``file`` Filename of the dsc to read. """ filename = os.path.join(self.repository, package_file.filename) package_version = package_file.source_package.package_version package = package_version.package if not os.path.isfile(filename): log.critical('Cannot find file %s' % filename) return '' if not package: log.critical('For some reason package is None...') return '' # Read the dsc file. dsc = deb822.Dsc(file(filename)) # There are a few differences between a dsc file and a Sources entry, listed and acted # upon below: # Firstly, the "Source" field in the dsc is simply renamed to "Package". dsc['Package'] = dsc.pop('Source') # There needs to be a "Directory" field to tell the package manager where to download the # package from. This is in the format (for the test package in the component "main"): # pool/main/t/test dsc['Directory'] = str( 'pool/%s/%s' % (package_version.component, get_package_dir(package.name))) # The dsc file, its size, and its md5sum needs to be added to the "Files" field. This is # unsurprisingly not in the original dsc file! dsc['Files'].append({ 'md5sum': package_file.md5sum, 'size': str(package_file.size), 'name': str(package_file.filename.split('/')[-1]) }) # Get a nice rfc822 output of this dsc, now Sources, entry. return dsc.dump()
def find_orig_tarball(self, changes_file): """ Look to see whether there is an orig tarball present, if the dsc refers to one. This method returns a triple (filename, file_found, location_hint), returning the (expected) name of the original tarball, and whether it was found in the local repository ```changes_file``` The changes file to parse for the orig.tar (note the dsc file referenced must exist) Returns a tuple (orig_filename, orig_filename_was_found) """ orig_name = None if not changes_file.get_dsc() or open(changes_file.get_dsc()) == None: return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND) dscfile = open(changes_file.get_dsc()) dsc = deb822.Dsc(dscfile) for file in dsc['Files']: if (file['name'].endswith('orig.tar.gz') or file['name'].endswith('orig.tar.bz2') or file['name'].endswith('orig.tar.xz')): # We know how the orig.tar.gz should be called - at least. orig_name = file['name'] full_filename = os.path.join( pylons.config['debexpo.repository'], changes_file.get_pool_path(), orig_name) # tar.gz was found in the local directory if os.path.isfile(file['name']): sum = md5sum(file['name']) if sum == file['md5sum']: return (orig_name, constants.ORIG_TARBALL_LOCATION_LOCAL) # tar.gz was found, but does not seem to be the same file # tar.gz was found in the repository elif os.path.isfile(full_filename): return (orig_name, constants.ORIG_TARBALL_LOCATION_REPOSITORY) # tar.gz was expected but not found at all else: return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND) # We should neve end up here return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
def test_build_system(self): """ Finds the build system of the package. """ log.debug('Finding the package\'s build system') dsc = deb822.Dsc(file(self.changes.get_dsc())) data = {} severity = constants.PLUGIN_SEVERITY_INFO build_depends = dsc.get('Build-Depends', '') if 'cdbs' in build_depends: outcome = "Package uses CDBS" data["build-system"] = "cdbs" elif 'debhelper' in build_depends: data["build-system"] = "debhelper" # Retrieve the debhelper compat level compatpath = os.path.join(self.tempdir, "extracted/debian/compat") try: with open(compatpath, "rb") as f: compat_level = int(f.read().strip()) except IOError: compat_level = None data["compat-level"] = compat_level # Warn on old compatibility levels if compat_level is None or compat_level <= 4: outcome = "Package uses debhelper with an old compatibility level" severity = constants.PLUGIN_SEVERITY_WARNING else: outcome = "Package uses debhelper" else: outcome = "Package uses an unknown build system" data["build-system"] = "unknown" severity = constants.PLUGIN_SEVERITY_WARNING self.failed(outcome, data, severity)
def test_orig_tarball(self): """ Check whether there is an original tarball referenced by the dsc file, but not actually in the package upload. This procedure is skipped to avoid denial of service attacks when a package is larger than the configured size """ # Set download of files, when the expected file size of the orig.tar.gz is larger # than the configured threshold. # XXX TODO: This size should be the 75% quartile, that is the value where 75% of all # packages are smaller than that. For now, blindly assume this as 15M size = 15728640 log.debug( 'Checking whether an orig tarball mentioned in the dsc is missing') dsc = deb822.Dsc(file(self.changes.get_dsc())) filecheck = CheckFiles() if filecheck.is_native_package(self.changes): log.debug('No orig.tar.gz file found; native package?') return # An orig.tar.gz was found in the dsc, and also in the upload. (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes) if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND: log.debug('%s found successfully', orig) return if not orig: log.debug("Couldn't determine name of the orig.tar.gz?") return for dscfile in dsc['Files']: dscfile['size'] = int(dscfile['size']) if orig == dscfile['name']: if dscfile['size'] > size: log.warning( "Skipping eventual download of orig.tar.gz %s: size %d > %d" % (dscfile['name'], dscfile['size'], size)) return orig = dscfile break else: log.debug( "dsc does not reference our expected orig.tar.gz name '%s'" % (orig)) return log.debug('Could not find %s; looking in Debian for it', orig['name']) url = os.path.join(pylons.config['debexpo.debian_mirror'], self.changes.get_pool_path(), orig['name']) log.debug('Trying to fetch %s' % url) out = urllib.urlopen(url) contents = out.read() f = open(orig['name'], "wb") f.write(contents) f.close() if md5sum(orig['name']) == orig['md5sum']: log.debug('Tarball %s taken from Debian' % orig['name']) self.info('tarball-taken-from-debian', None) else: log.error('Tarball %s not found in Debian' % orig['name']) os.unlink(orig['name'])
def _get_dsc_obj(self): val = dict(deb822.Dsc(open(self.get_dsc(), 'r'))) val = self._obj_strip(val) return val
def read_dsc(dsc): if dsc[0] == '/': dsc = dsc[1:] with open(os.path.join(debian, dsc)) as d: return deb822.Dsc(d)
def get_dsc_obj(self): return deb822.Dsc(open(self.get_dsc(), 'r'))
def parse_dsc(dsc_file): """ Parse a dsc file into a Dsc class. """ with open(dsc_file) as f: return deb822.Dsc(f)