def find_orig_tarball(self, changes_file):
"""
Look to see whether there is an orig tarball present, if the dsc refers to one.
This method returns a triple (filename, file_found, location_hint), returning the (expected)
name of the original tarball, and whether it was found in the local repository
```changes_file```
The changes file to parse for the orig.tar (note the dsc file referenced must exist)
Returns a tuple (orig_filename, orig_filename_was_found)
"""
orig_name = None
if not changes_file.get_dsc() or open(changes_file.get_dsc()) == None:
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
dscfile = open(changes_file.get_dsc())
dsc = deb822.Dsc(dscfile)
for file in dsc['Files']:
if (file['name'].endswith('orig.tar.gz')
or file['name'].endswith('orig.tar.bz2')
or file['name'].endswith('orig.tar.xz')):
# We know how the orig.tar.gz should be called - at least.
orig_name = file['name']
full_filename = os.path.join(
pylons.config['debexpo.repository'],
changes_file.get_pool_path(), orig_name)
# tar.gz was found in the local directory
if os.path.isfile(file['name']):
sum = md5sum(file['name'])
if sum == file['md5sum']:
return (orig_name,
constants.ORIG_TARBALL_LOCATION_LOCAL)
# tar.gz was found, but does not seem to be the same file
# tar.gz was found in the repository
elif os.path.isfile(full_filename):
return (orig_name,
constants.ORIG_TARBALL_LOCATION_REPOSITORY)
# tar.gz was expected but not found at all
else:
return (orig_name,
constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
# We should neve end up here
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
def test_md5sum(self):
"""
Check each file's md5sum and make sure the md5sum in the changes file is the same
as the actual file's md5sum.
"""
for file in self.changes['Files']:
log.debug('Checking md5sum of %s' % file['name'])
filename = os.path.join(config['debexpo.upload.incoming'], file['name'])
if os.path.isfile(filename):
sum = md5sum(filename)
data = 'Changes file says md5sum of %s is: %s\n' % (file['name'], file['md5sum'])
data += 'Actual md5sum of %s is: %s' % (file['name'], sum)
if sum != file['md5sum']:
log.error('%s != %s' % (sum, file['md5sum']))
self.failed('md5sum-not-match', data, constants.PLUGIN_SEVERITY_ERROR)
else:
log.debug('Test passed')
self.passed('md5sum-match', None, constants.PLUGIN_SEVERITY_INFO)
def test_md5sum(self, changes_file):
"""
Check each file's md5sum and make sure the md5sum in the changes file is the same
as the actual file's md5sum.
```changes_file```
The changes file to parse for the orig.tar (note the dsc file referenced must exist)
"""
for file in changes_file['Files']:
log.debug('Checking md5sum of %s' % file['name'])
filename = os.path.join(pylons.config['debexpo.upload.incoming'], file['name'])
if not os.path.isfile(filename):
raise OSError("Missing file %s in incoming" % (file['name']))
sum = md5sum(filename)
if sum != file['md5sum']:
log.critical('%s != %s' % (sum, file['md5sum']))
raise OSError("MD5 sum mismatch in file %s: %s != %s" % (file['name'], sum, file['md5sum']))
return True
def find_orig_tarball(self, changes_file):
"""
Look to see whether there is an orig tarball present, if the dsc refers to one.
This method returns a triple (filename, file_found, location_hint), returning the (expected)
name of the original tarball, and whether it was found in the local repository
```changes_file```
The changes file to parse for the orig.tar (note the dsc file referenced must exist)
Returns a tuple (orig_filename, orig_filename_was_found)
"""
orig_name = None
if not changes_file.get_dsc() or open(changes_file.get_dsc()) == None:
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
dscfile = open(changes_file.get_dsc())
dsc = deb822.Dsc(dscfile)
for file in dsc['Files']:
if (file['name'].endswith('orig.tar.gz') or
file['name'].endswith('orig.tar.bz2') or
file['name'].endswith('orig.tar.xz')):
# We know how the orig.tar.gz should be called - at least.
orig_name = file['name']
full_filename = os.path.join(pylons.config['debexpo.repository'], changes_file.get_pool_path(), orig_name)
# tar.gz was found in the local directory
if os.path.isfile(file['name']):
sum = md5sum(file['name'])
if sum == file['md5sum']:
return (orig_name, constants.ORIG_TARBALL_LOCATION_LOCAL)
# tar.gz was found, but does not seem to be the same file
# tar.gz was found in the repository
elif os.path.isfile(full_filename):
return (orig_name, constants.ORIG_TARBALL_LOCATION_REPOSITORY)
# tar.gz was expected but not found at all
else:
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
# We should neve end up here
return (orig_name, constants.ORIG_TARBALL_LOCATION_NOT_FOUND)
def test_md5sum(self, changes_file):
"""
Check each file's md5sum and make sure the md5sum in the changes file is the same
as the actual file's md5sum.
```changes_file```
The changes file to parse for the orig.tar (note the dsc file referenced must exist)
"""
for file in changes_file['Files']:
log.debug('Checking md5sum of %s' % file['name'])
filename = os.path.join(pylons.config['debexpo.upload.incoming'],
file['name'])
if not os.path.isfile(filename):
raise OSError("Missing file %s in incoming" % (file['name']))
sum = md5sum(filename)
if sum != file['md5sum']:
log.critical('%s != %s' % (sum, file['md5sum']))
raise OSError("MD5 sum mismatch in file %s: %s != %s" %
(file['name'], sum, file['md5sum']))
return True
def test_md5sum(self):
"""
Check each file's md5sum and make sure the md5sum in the changes file is the same
as the actual file's md5sum.
"""
for file in self.changes['Files']:
log.debug('Checking md5sum of %s' % file['name'])
filename = os.path.join(config['debexpo.upload.incoming'],
file['name'])
if os.path.isfile(filename):
sum = md5sum(filename)
data = 'Changes file says md5sum of %s is: %s\n' % (
file['name'], file['md5sum'])
data += 'Actual md5sum of %s is: %s' % (file['name'], sum)
if sum != file['md5sum']:
log.error('%s != %s' % (sum, file['md5sum']))
self.failed('md5sum-not-match', data,
constants.PLUGIN_SEVERITY_ERROR)
else:
log.debug('Test passed')
self.passed('md5sum-match', None,
constants.PLUGIN_SEVERITY_INFO)
def _create_db_entries(self, qa):
"""
Create entries in the Database for the package upload.
"""
def _package_description(raw):
return raw[2:].replace(' - ', ' - ')
log.debug('Creating database entries')
# Parse component and section from field in changes
component, section = parse_section(self.changes['files'][0]['section'])
# Check whether package is already in the database
package_query = meta.session.query(Package).filter_by(name=self.changes['Source'])
if package_query.count() == 1:
log.debug('Package %s already exists in the database' % self.changes['Source'])
package = package_query.one()
# Update description to make sure it reflects the latest upload
package.description = _package_description(self.changes['Description'])
else:
log.debug('Package %s is new to the system' % self.changes['Source'])
package = Package(name=self.changes['Source'], user=self.user)
package.description = _package_description(self.changes['Description'])
package.needs_sponsor = 0
meta.session.add(package)
# No need to check whether there is the same source name and same version as an existing
# entry in the database as the upload controller tested whether similar filenames existed
# in the repository. The only way this would be wrong is if the filename had a different
# version in than the Version field in changes..
try:
closes = self.changes['Closes']
except KeyError:
closes = None
# TODO: fix these magic numbers
if qa.stop():
qa_status = 1
else:
qa_status = 0
maintainer_matches = re.compile(r'(.*) <(.*)>').match(self.changes['Changed-By'])
maintainer = maintainer_matches.group(2)
package_version = PackageVersion(package=package, version=self.changes['Version'],
section=section, distribution=self.changes['Distribution'], qa_status=qa_status,
component=component, priority=self.changes.get_priority(), closes=closes,
uploaded=datetime.now(), maintainer=maintainer)
meta.session.add(package_version)
source_package = SourcePackage(package_version=package_version)
meta.session.add(source_package)
binary_package = None
# Add PackageFile objects to the database for each uploaded file
for file in self.files:
filename = os.path.join(self.changes.get_pool_path(), file)
# This exception should be never caught.
# It implies something went wrong before, as we expect a file which does not exist
try:
sum = md5sum(os.path.join(pylons.config['debexpo.repository'], filename))
except AttributeError as e:
self._fail("Could not calculate MD5 sum: %s" % (e))
size = os.stat(os.path.join(pylons.config['debexpo.repository'], filename))[ST_SIZE]
# Check for binary or source package file
if file.endswith('.deb'):
# Only create a BinaryPackage if there actually binary package files
if binary_package is None:
binary_package = BinaryPackage(package_version=package_version, arch=file[:-4].split('_')[-1])
meta.session.add(binary_package)
meta.session.add(PackageFile(filename=filename, binary_package=binary_package, size=size, md5sum=sum))
else:
meta.session.add(PackageFile(filename=filename, source_package=source_package, size=size, md5sum=sum))
meta.session.commit()
log.warning("Finished adding PackageFile objects.")
# Add PackageInfo objects to the database for the package_version
for result in qa.result:
meta.session.add(PackageInfo(package_version=package_version, from_plugin=result.from_plugin,
outcome=result.outcome, rich_data=result.data, severity=result.severity))
# Commit all changes to the database
meta.session.commit()
log.debug('Committed package data to the database')
subscribers = meta.session.query(PackageSubscription).filter_by(package=self.changes['Source']).filter(\
PackageSubscription.level <= constants.SUBSCRIPTION_LEVEL_UPLOADS).all()
if len(subscribers) > 0:
email = Email('package_uploaded')
self.send_email(email, [s.user.email for s in subscribers], package=self.changes['Source'],
version=self.changes['Version'], user=self.user)
log.debug('Sent out package subscription emails')
# Send success email to uploader
email = Email('successful_upload')
dsc_url = pylons.config[
'debexpo.server'] + '/debian/' + self.changes.get_pool_path() + '/' + self.changes.get_dsc()
rfs_url = pylons.config['debexpo.server'] + url('rfs', packagename=self.changes['Source'])
self.send_email(email, [self.user.email], package=self.changes['Source'],
dsc_url=dsc_url, rfs_url=rfs_url)
def test_orig_tarball(self):
"""
Check whether there is an original tarball referenced by the dsc file, but not
actually in the package upload.
This procedure is skipped to avoid denial of service attacks when a package is
larger than the configured size
"""
# Set download of files, when the expected file size of the orig.tar.gz is larger
# than the configured threshold.
# XXX TODO: This size should be the 75% quartile, that is the value where 75% of all
# packages are smaller than that. For now, blindly assume this as 15M
size = 15728640
log.debug(
'Checking whether an orig tarball mentioned in the dsc is missing')
dsc = deb822.Dsc(file(self.changes.get_dsc()))
filecheck = CheckFiles()
if filecheck.is_native_package(self.changes):
log.debug('No orig.tar.gz file found; native package?')
return
# An orig.tar.gz was found in the dsc, and also in the upload.
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
log.debug('%s found successfully', orig)
return
if not orig:
log.debug("Couldn't determine name of the orig.tar.gz?")
return
for dscfile in dsc['Files']:
dscfile['size'] = int(dscfile['size'])
if orig == dscfile['name']:
if dscfile['size'] > size:
log.warning(
"Skipping eventual download of orig.tar.gz %s: size %d > %d"
% (dscfile['name'], dscfile['size'], size))
return
orig = dscfile
break
else:
log.debug(
"dsc does not reference our expected orig.tar.gz name '%s'" %
(orig))
return
log.debug('Could not find %s; looking in Debian for it', orig['name'])
url = os.path.join(pylons.config['debexpo.debian_mirror'],
self.changes.get_pool_path(), orig['name'])
log.debug('Trying to fetch %s' % url)
out = urllib.urlopen(url)
contents = out.read()
f = open(orig['name'], "wb")
f.write(contents)
f.close()
if md5sum(orig['name']) == orig['md5sum']:
log.debug('Tarball %s taken from Debian' % orig['name'])
self.info('tarball-taken-from-debian', None)
else:
log.error('Tarball %s not found in Debian' % orig['name'])
os.unlink(orig['name'])
def test_orig_tarball(self):
"""
Check whether there is an original tarball referenced by the dsc file, but not
actually in the package upload.
This procedure is skipped to avoid denial of service attacks when a package is
larger than the configured size
"""
# Set download of files, when the expected file size of the orig.tar.gz is larger
# than the configured threshold.
# XXX TODO: This size should be the 75% quartile, that is the value where 75% of all
# packages are smaller than that. For now, blindly assume this as 15M
size = 15728640
log.debug('Checking whether an orig tarball mentioned in the dsc is missing')
dsc = deb822.Dsc(file(self.changes.get_dsc()))
filecheck = CheckFiles()
if filecheck.is_native_package(self.changes):
log.debug('No orig.tar.gz file found; native package?')
return
# An orig.tar.gz was found in the dsc, and also in the upload.
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
log.debug('%s found successfully', orig)
return
if not orig:
log.debug("Couldn't determine name of the orig.tar.gz?")
return
for dscfile in dsc['Files']:
dscfile['size'] = int(dscfile['size'])
if orig == dscfile['name']:
if dscfile['size'] > size:
log.warning("Skipping eventual download of orig.tar.gz %s: size %d > %d" % (dscfile['name'], dscfile['size'], size))
return
orig = dscfile
break
else:
log.debug("dsc does not reference our expected orig.tar.gz name '%s'" % (orig))
return
log.debug('Could not find %s; looking in Debian for it', orig['name'])
url = os.path.join(pylons.config['debexpo.debian_mirror'], self.changes.get_pool_path(), orig['name'])
log.debug('Trying to fetch %s' % url)
out = urllib.urlopen(url)
contents = out.read()
f = open(orig['name'], "wb")
f.write(contents)
f.close()
if md5sum(orig['name']) == orig['md5sum']:
log.debug('Tarball %s taken from Debian' % orig['name'])
self.info('tarball-taken-from-debian', None)
else:
log.error('Tarball %s not found in Debian' % orig['name'])
os.unlink(orig['name'])
def _create_db_entries(self, qa):
"""
Create entries in the Database for the package upload.
"""
def _package_description(raw):
return raw[2:].replace(' - ', ' - ')
log.debug('Creating database entries')
# Parse component and section from field in changes
component, section = parse_section(self.changes['files'][0]['section'])
# Check whether package is already in the database
package_query = meta.session.query(Package).filter_by(
name=self.changes['Source'])
if package_query.count() == 1:
log.debug('Package %s already exists in the database' %
self.changes['Source'])
package = package_query.one()
# Update description to make sure it reflects the latest upload
package.description = _package_description(
self.changes['Description'])
else:
log.debug('Package %s is new to the system' %
self.changes['Source'])
package = Package(name=self.changes['Source'], user=self.user)
package.description = _package_description(
self.changes['Description'])
package.needs_sponsor = 0
meta.session.add(package)
# No need to check whether there is the same source name and same version as an existing
# entry in the database as the upload controller tested whether similar filenames existed
# in the repository. The only way this would be wrong is if the filename had a different
# version in than the Version field in changes..
try:
closes = self.changes['Closes']
except KeyError:
closes = None
# TODO: fix these magic numbers
if qa.stop():
qa_status = 1
else:
qa_status = 0
maintainer_matches = re.compile(r'(.*) <(.*)>').match(
self.changes['Changed-By'])
maintainer = maintainer_matches.group(2)
package_version = PackageVersion(
package=package,
version=self.changes['Version'],
section=section,
distribution=self.changes['Distribution'],
qa_status=qa_status,
component=component,
priority=self.changes.get_priority(),
closes=closes,
uploaded=datetime.now(),
maintainer=maintainer)
meta.session.add(package_version)
source_package = SourcePackage(package_version=package_version)
meta.session.add(source_package)
binary_package = None
# Add PackageFile objects to the database for each uploaded file
for file in self.files:
filename = os.path.join(self.changes.get_pool_path(), file)
# This exception should be never caught.
# It implies something went wrong before, as we expect a file which does not exist
try:
sum = md5sum(
os.path.join(pylons.config['debexpo.repository'],
filename))
except AttributeError as e:
self._fail("Could not calculate MD5 sum: %s" % (e))
size = os.stat(
os.path.join(pylons.config['debexpo.repository'],
filename))[ST_SIZE]
# Check for binary or source package file
if file.endswith('.deb'):
# Only create a BinaryPackage if there actually binary package files
if binary_package is None:
binary_package = BinaryPackage(
package_version=package_version,
arch=file[:-4].split('_')[-1])
meta.session.add(binary_package)
meta.session.add(
PackageFile(filename=filename,
binary_package=binary_package,
size=size,
md5sum=sum))
else:
meta.session.add(
PackageFile(filename=filename,
source_package=source_package,
size=size,
md5sum=sum))
meta.session.commit()
log.warning("Finished adding PackageFile objects.")
# Add PackageInfo objects to the database for the package_version
for result in qa.result:
meta.session.add(
PackageInfo(package_version=package_version,
from_plugin=result.from_plugin,
outcome=result.outcome,
rich_data=result.data,
severity=result.severity))
# Commit all changes to the database
meta.session.commit()
log.debug('Committed package data to the database')
subscribers = meta.session.query(PackageSubscription).filter_by(package=self.changes['Source']).filter(\
PackageSubscription.level <= constants.SUBSCRIPTION_LEVEL_UPLOADS).all()
if len(subscribers) > 0:
email = Email('package_uploaded')
self.send_email(email, [s.user.email for s in subscribers],
package=self.changes['Source'],
version=self.changes['Version'],
user=self.user)
log.debug('Sent out package subscription emails')
# Send success email to uploader
email = Email('successful_upload')
dsc_url = pylons.config[
'debexpo.server'] + '/debian/' + self.changes.get_pool_path(
) + '/' + self.changes.get_dsc()
rfs_url = pylons.config['debexpo.server'] + url(
'rfs', packagename=self.changes['Source'])
self.send_email(email, [self.user.email],
package=self.changes['Source'],
dsc_url=dsc_url,
rfs_url=rfs_url)
