def test_gpg_info(self):
if not (os.path.exists('/usr/bin/gpgv')
and os.path.exists('/usr/share/keyrings/debian-keyring.gpg')):
return
unparsed_with_gpg = SIGNED_CHECKSUM_CHANGES_FILE % CHECKSUM_CHANGES_FILE
deb822_from_str = deb822.Dsc(unparsed_with_gpg)
result_from_str = deb822_from_str.get_gpg_info()
deb822_from_file = deb822.Dsc(StringIO(unparsed_with_gpg))
result_from_file = deb822_from_file.get_gpg_info()
deb822_from_lines = deb822.Dsc(unparsed_with_gpg.splitlines())
result_from_lines = deb822_from_lines.get_gpg_info()
valid = {
'GOODSIG':
['D14219877A786561', 'John Wright <*****@*****.**>'],
'VALIDSIG': [
'8FEFE900783CF175827C2F65D14219877A786561', '2008-05-01',
'1209623566', '0', '3', '0', '17', '2', '01',
'8FEFE900783CF175827C2F65D14219877A786561'
],
'SIG_ID':
['j3UjSpdky92fcQISbm8W5PlwC/g', '2008-05-01', '1209623566'],
}
for result in result_from_str, result_from_file, result_from_lines:
# The second part of the GOODSIG field could change if the primary
# uid changes, so avoid checking that. Also, the first part of the
# SIG_ID field has undergone at least one algorithm changein gpg,
# so don't bother testing that either.
self.assertEqual(set(result.keys()), set(valid.keys()))
self.assertEqual(result['GOODSIG'][0], valid['GOODSIG'][0])
self.assertEqual(result['VALIDSIG'], valid['VALIDSIG'])
self.assertEqual(result['SIG_ID'][1:], valid['SIG_ID'][1:])
def _extract(self):
"""
Copy the files to a temporary directory and run dpkg-source -x on the dsc file
to extract them.
"""
log.debug('Copying files to a temp directory to run dpkg-source -x on the dsc file')
self.tempdir = tempfile.mkdtemp()
log.debug('Temp dir is: %s', self.tempdir)
for filename in self.changes.get_files():
log.debug('Copying: %s', filename)
shutil.copy(os.path.join(self.config['debexpo.upload.incoming'], filename), self.tempdir)
# If the original tarball was pulled from Debian or from the repository, that
# also needs to be copied into this directory.
dsc = deb822.Dsc(file(self.changes.get_dsc()))
for item in dsc['Files']:
if item['name'] not in self.changes.get_files():
src_file = os.path.join(self.config['debexpo.upload.incoming'], item['name'])
repository_src_file = os.path.join(self.config['debexpo.repository'], self.changes.get_pool_path(), item['name'])
if os.path.exists(src_file):
shutil.copy(src_file, self.tempdir)
elif os.path.exists(repository_src_file):
shutil.copy(repository_src_file, self.tempdir)
else:
log.critical("Trying to copy non-existing file %s" % (src_file))
shutil.copy(os.path.join(self.config['debexpo.upload.incoming'], self.changes_file), self.tempdir)
self.oldcurdir = os.path.abspath(os.path.curdir)
os.chdir(self.tempdir)
os.system('dpkg-source -x %s extracted' % self.changes.get_dsc())
def test_control_fields(self):
"""
Checks whether additional debian/control fields are present.
"""
log.debug(
'Checking whether additional debian/control fields are present')
try:
dsc = deb822.Dsc(file(self.changes.get_dsc()))
except:
log.critical('Could not open dsc file; skipping plugin')
return
data = {}
severity = constants.PLUGIN_SEVERITY_WARNING
outcome = "No Homepage field present"
for item in fields:
if item in dsc:
data[item] = dsc[item]
if "Homepage" in data:
severity = constants.PLUGIN_SEVERITY_INFO
if len(data) > 1:
outcome = "Homepage and VCS control fields present"
else:
outcome = "Homepage control field present"
self.failed(outcome, data, severity)
def test_multivalued_field_contains_newline(self):
"""Multivalued field components are not allowed to contain newlines"""
d = deb822.Dsc()
# We don't check at set time, since one could easily modify the list
# without deb822 knowing. We instead check at get time.
d['Files'] = [{'md5sum': 'deadbeef', 'size': '9605', 'name': 'bad\n'}]
self.assertRaises(ValueError, d.get_as_string, 'files')
def forge_changes_file(fname, dist, **kwargs):
dsc = deb822.Dsc(open(fname, 'r'))
changes = deb822.Changes()
changes['Format'] = '1.8'
changes['Date'] = email.utils.formatdate(time.mktime(
dt.datetime.utcnow().timetuple()),
usegmt=True)
for key in [
'Source', 'Version', 'Maintainer', 'Checksums-Sha1',
'Checksums-Sha256', 'Files'
]:
if not dsc.has_key(key):
raise MissingChangesFieldException(key)
changes[key] = dsc[key]
for algo, key, h, s, f in file_info(fname):
if algo == 'md5':
algo = 'md5sum'
entry = deb822.Deb822Dict()
entry[algo] = h
entry['size'] = s
entry['name'] = f
changes[key].append(entry)
for entry in changes['Files']:
entry['section'] = 'not-implemented'
entry['priority'] = 'not-implemented'
changes['Distribution'] = dist
changes['Urgency'] = 'low'
changes['Changed-By'] = 'Archive Rebuilder <*****@*****.**>'
changes['Architecture'] = 'source'
changes['Binary'] = 'not implemented either'
changes['Description'] = """This feature is not implemented.
This is a pretty damn hard to deal with right now. I might write this
later."""
changes['Changes'] = """
{source} ({version}) {dist}; urgency={urgency}
.
* This is a fake ChangeLog entry used by ricky to force a rebuild
on debuild.me.""".format(
source=changes['Source'],
version=changes['Version'],
urgency=changes['Urgency'],
dist=dist,
)
for k, v in kwargs.items():
changes[k] = v
return changes
def test_maintainer_email(self):
"""
Tests whether the maintainer email is the same as the uploader email.
"""
if self.user_id is not None:
log.debug(
'Checking whether the maintainer email is the same as the uploader email'
)
user = meta.session.query(User).get(self.user_id)
if user is not None:
maintainer_name, maintainer_email = email.utils.parseaddr(
self.changes['Maintainer'])
uploader_emails = []
dsc = deb822.Dsc(file(self.changes.get_dsc()))
if 'Uploaders' in dsc:
for uploader_name, uploader_email in email.utils.getaddresses(
[dsc['Uploaders']]):
uploader_emails.append(uploader_email)
severity = constants.PLUGIN_SEVERITY_INFO
if user.email == maintainer_email:
log.debug('"Maintainer" email is the same as the uploader')
outcome = '"Maintainer" email is the same as the uploader'
elif user.email in uploader_emails:
log.debug(
'The uploader is in the package\'s "Uploaders" field')
outcome = 'The uploader is in the package\'s "Uploaders" field'
else:
log.warning('%s != %s' % (user.email, maintainer_email))
outcome = 'The uploader is not in the package\'s "Maintainer" or "Uploaders" fields'
severity = constants.PLUGIN_SEVERITY_WARNING
data = {
'user-is-maintainer':
(severity == constants.PLUGIN_SEVERITY_INFO),
'user-email':
user.email,
'maintainer-email':
maintainer_email,
'uploader-emails':
uploader_emails,
}
self.failed(outcome, data, severity)
else:
log.warning(
'Could not get the uploader\'s user details from the database')
def _dsc_to_sources(self, package_file):
"""
Reads the contents of a dsc file and converts it to a Sources file entry.
``file``
Filename of the dsc to read.
"""
filename = os.path.join(self.repository, package_file.filename)
package_version = package_file.source_package.package_version
package = package_version.package
if not os.path.isfile(filename):
log.critical('Cannot find file %s' % filename)
return ''
if not package:
log.critical('For some reason package is None...')
return ''
# Read the dsc file.
dsc = deb822.Dsc(file(filename))
# There are a few differences between a dsc file and a Sources entry, listed and acted
# upon below:
# Firstly, the "Source" field in the dsc is simply renamed to "Package".
dsc['Package'] = dsc.pop('Source')
# There needs to be a "Directory" field to tell the package manager where to download the
# package from. This is in the format (for the test package in the component "main"):
# pool/main/t/test
dsc['Directory'] = str(
'pool/%s/%s' %
(package_version.component, get_package_dir(package.name)))
# The dsc file, its size, and its md5sum needs to be added to the "Files" field. This is
# unsurprisingly not in the original dsc file!
dsc['Files'].append({
'md5sum': package_file.md5sum,
'size': str(package_file.size),
'name': str(package_file.filename.split('/')[-1])
})
# Get a nice rfc822 output of this dsc, now Sources, entry.
return dsc.dump()
def find_orig_tarball(self, changes_file):
"""
Look to see whether there is an orig tarball present, if the dsc refers to one.
This method returns a triple (filename, file_found, location_hint), returning the (expected)
name of the original tarball, and whether it was found in the local repository
```changes_file```
The changes file to parse for the orig.tar (note the dsc file referenced must exist)
Returns a tuple (orig_filename, orig_filename_was_found)
"""
orig_name = None
if not changes_file.get_dsc() or open(changes_file.get_dsc()) == None:
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
dscfile = open(changes_file.get_dsc())
dsc = deb822.Dsc(dscfile)
for file in dsc['Files']:
if (file['name'].endswith('orig.tar.gz')
or file['name'].endswith('orig.tar.bz2')
or file['name'].endswith('orig.tar.xz')):
# We know how the orig.tar.gz should be called - at least.
orig_name = file['name']
full_filename = os.path.join(
pylons.config['debexpo.repository'],
changes_file.get_pool_path(), orig_name)
# tar.gz was found in the local directory
if os.path.isfile(file['name']):
sum = md5sum(file['name'])
if sum == file['md5sum']:
return (orig_name,
constants.ORIG_TARBALL_LOCATION_LOCAL)
# tar.gz was found, but does not seem to be the same file
# tar.gz was found in the repository
elif os.path.isfile(full_filename):
return (orig_name,
constants.ORIG_TARBALL_LOCATION_REPOSITORY)
# tar.gz was expected but not found at all
else:
return (orig_name,
constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
# We should neve end up here
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
def test_build_system(self):
"""
Finds the build system of the package.
"""
log.debug('Finding the package\'s build system')
dsc = deb822.Dsc(file(self.changes.get_dsc()))
data = {}
severity = constants.PLUGIN_SEVERITY_INFO
build_depends = dsc.get('Build-Depends', '')
if 'cdbs' in build_depends:
outcome = "Package uses CDBS"
data["build-system"] = "cdbs"
elif 'debhelper' in build_depends:
data["build-system"] = "debhelper"
# Retrieve the debhelper compat level
compatpath = os.path.join(self.tempdir, "extracted/debian/compat")
try:
with open(compatpath, "rb") as f:
compat_level = int(f.read().strip())
except IOError:
compat_level = None
data["compat-level"] = compat_level
# Warn on old compatibility levels
if compat_level is None or compat_level <= 4:
outcome = "Package uses debhelper with an old compatibility level"
severity = constants.PLUGIN_SEVERITY_WARNING
else:
outcome = "Package uses debhelper"
else:
outcome = "Package uses an unknown build system"
data["build-system"] = "unknown"
severity = constants.PLUGIN_SEVERITY_WARNING
self.failed(outcome, data, severity)
def test_orig_tarball(self):
"""
Check whether there is an original tarball referenced by the dsc file, but not
actually in the package upload.
This procedure is skipped to avoid denial of service attacks when a package is
larger than the configured size
"""
# Set download of files, when the expected file size of the orig.tar.gz is larger
# than the configured threshold.
# XXX TODO: This size should be the 75% quartile, that is the value where 75% of all
# packages are smaller than that. For now, blindly assume this as 15M
size = 15728640
log.debug(
'Checking whether an orig tarball mentioned in the dsc is missing')
dsc = deb822.Dsc(file(self.changes.get_dsc()))
filecheck = CheckFiles()
if filecheck.is_native_package(self.changes):
log.debug('No orig.tar.gz file found; native package?')
return
# An orig.tar.gz was found in the dsc, and also in the upload.
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
log.debug('%s found successfully', orig)
return
if not orig:
log.debug("Couldn't determine name of the orig.tar.gz?")
return
for dscfile in dsc['Files']:
dscfile['size'] = int(dscfile['size'])
if orig == dscfile['name']:
if dscfile['size'] > size:
log.warning(
"Skipping eventual download of orig.tar.gz %s: size %d > %d"
% (dscfile['name'], dscfile['size'], size))
return
orig = dscfile
break
else:
log.debug(
"dsc does not reference our expected orig.tar.gz name '%s'" %
(orig))
return
log.debug('Could not find %s; looking in Debian for it', orig['name'])
url = os.path.join(pylons.config['debexpo.debian_mirror'],
self.changes.get_pool_path(), orig['name'])
log.debug('Trying to fetch %s' % url)
out = urllib.urlopen(url)
contents = out.read()
f = open(orig['name'], "wb")
f.write(contents)
f.close()
if md5sum(orig['name']) == orig['md5sum']:
log.debug('Tarball %s taken from Debian' % orig['name'])
self.info('tarball-taken-from-debian', None)
else:
log.error('Tarball %s not found in Debian' % orig['name'])
os.unlink(orig['name'])
def _get_dsc_obj(self):
val = dict(deb822.Dsc(open(self.get_dsc(), 'r')))
val = self._obj_strip(val)
return val
def read_dsc(dsc):
if dsc[0] == '/':
dsc = dsc[1:]
with open(os.path.join(debian, dsc)) as d:
return deb822.Dsc(d)
def get_dsc_obj(self):
return deb822.Dsc(open(self.get_dsc(), 'r'))
def parse_dsc(dsc_file):
""" Parse a dsc file into a Dsc class. """
with open(dsc_file) as f:
return deb822.Dsc(f)