def _create_dists(self, email, filename):
"""
Create a /dists/... file and return it in the appropraite format.
``email``
Email address to filter on.
``filename``
Filename requested. This should be something like:
"dists/unstable/main/source/Sources".
"""
splitfilename = filename.split('/')
if not len(splitfilename) == 5:
log.error('File not found')
abort(404, 'File not found')
dists, dist, component, arch, type = filename.split('/')
if arch.startswith('binary-'):
arch = arch[7:]
user = meta.session.query(User).filter_by(email=email).first()
if user is None:
log.error('User not found')
abort(404, 'User not found')
repo = Repository(config['debexpo.repository'])
if type.startswith('Sources'):
out_file = repo.get_sources_file(dist, component, user.id)
elif type.startswith('Packages'):
out_file = repo.get_packages_file(dist, component, arch, user.id)
else:
log.error('File not found')
abort(404, 'File not found')
if type.endswith('.gz'):
return zlib.compress(out_file)
elif type.endswith('.bz2'):
return bz2.compress(out_file)
else:
return out_file
def _create_dists(self, email, filename):
"""
Create a /dists/... file and return it in the appropraite format.
``email``
Email address to filter on.
``filename``
Filename requested. This should be something like:
"dists/unstable/main/source/Sources".
"""
splitfilename = filename.split('/')
if not len(splitfilename) == 5:
log.error('File not found')
abort(404, 'File not found')
dists, dist, component, arch, type = filename.split('/')
if arch.startswith('binary-'):
arch = arch[7:]
user = meta.session.query(User).filter_by(email=email).first()
if user is None:
log.error('User not found')
abort(404, 'User not found')
repo = Repository(config['debexpo.repository'])
if type.startswith('Sources'):
out_file = repo.get_sources_file(dist, component, user.id)
elif type.startswith('Packages'):
out_file = repo.get_packages_file(dist, component, arch, user.id)
else:
log.error('File not found')
abort(404, 'File not found')
if type.endswith('.gz'):
return zlib.compress(out_file)
elif type.endswith('.bz2'):
return bz2.compress(out_file)
else:
return out_file
def main(self):
"""
Parse the d-d-c email and take action.
"""
# Set up
self._setup()
if log is not None:
log.debug('d-d-c parser started with arguments: %s' % sys.argv[1:])
from debexpo.lib.changes import Changes
from debexpo.lib.repository import Repository
from debexpo.lib.plugins import Plugins
changes = Changes(string=self.email)
Plugins('post-upload-to-debian', changes, None)
# Refresh the Sources/Packages files.
log.debug('Updating Sources and Packages files')
r = Repository(self.config['debexpo.repository'])
r.update()
log.debug('Done')
def main(self):
"""
Parse the d-d-c email and take action.
"""
# Set up
self._setup()
if log is not None:
log.debug('d-d-c parser started with arguments: %s' % sys.argv[1:])
from debexpo.lib.changes import Changes
from debexpo.lib.repository import Repository
from debexpo.lib.plugins import Plugins
changes = Changes(string=self.email)
Plugins('post-upload-to-debian', changes, None)
# Refresh the Sources/Packages files.
log.debug('Updating Sources and Packages files')
r = Repository(self.config['debexpo.repository'])
r.update()
log.debug('Done')
def main(self):
"""
Actually start the import of the package.
Do several environment sanity checks, move files into the right place, and then
create the database entries for the imported package.
"""
# Set up importer
self._setup()
log.debug('Importer started with arguments: %s' % sys.argv[1:])
filecheck = CheckFiles()
gpg = get_gnupg()
# Try parsing the changes file, but fail if there's an error.
try:
self.changes = Changes(filename=self.changes_file)
filecheck.test_files_present(self.changes)
filecheck.test_md5sum(self.changes)
except Exception as e:
# XXX: The user won't ever see this message. The changes file was
# invalid, we don't know whom send it to
self._reject("Your changes file appears invalid. Refusing your upload\n%s" % (e.message))
if not self.skip_gpg:
# Next, find out whether the changes file was signed with a valid signature, if not reject immediately
v = gpg.verify_file(path=self.changes_file)
if v is None:
self._reject('Your upload does not appear to be signed')
if not v.is_valid:
self._reject('Your upload does not contain a valid signature. Output was:\n%s' % (gpg_out))
self._determine_uploader_by_gpg(v.key_id)
else:
self._determine_uploader_by_changedby_field()
if self.user is None:
# Creates a fake user object, but only if no user was found before
# This is useful to have a user object to send reject mails to
# generate user object, but only to send out reject message
self.user = User(id=-1, name=maintainer_realname, email=maintainer_email_address)
self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)
self.user_id = self.user.id
log.debug("User found in database. Has id: %s", self.user.id)
if self.user.dmup is False:
log.debug("User has not accepted the DMUP")
self._reject('You must accept the Debian Machine Usage Policies before uploading a package')
self.files = self.changes.get_files()
self.files_to_remove = []
distribution = self.changes['Distribution'].lower()
allowed_distributions = (
'oldstable', 'stable', 'unstable', 'experimental', 'stable-backports', 'oldstable-backports',
'oldstable-backports-sloppy', 'oldstable-security', 'stable-security', 'testing-security',
'stable-proposed-updates',
'testing-proposed-updates', 'sid', 'wheezy', 'squeeze', 'lenny', 'squeeze-backports', 'lenny-backports',
'lenny-security', 'lenny-backports-sloppy', 'lenny-volatile', 'squeeze-security', 'squeeze-updates',
'wheezy-security',
'unreleased')
if distribution not in allowed_distributions:
self._reject("You are not uploading to one of those Debian distributions: %s" %
(reduce(lambda x, xs: x + " " + xs, allowed_distributions)))
# Look whether the orig tarball is present, and if not, try and get it from
# the repository.
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
log.debug("Upload does not contain orig.tar.gz - trying to find it elsewhere")
if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
filename = os.path.join(pylons.config['debexpo.repository'],
self.changes.get_pool_path(), orig)
if os.path.isfile(filename):
log.debug("Found tar.gz in repository as %s" % (filename))
shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
# We need the orig.tar.gz for the import run, plugins need to extract the source package
# also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
# remove it later
self.files_to_remove.append(orig)
destdir = pylons.config['debexpo.repository']
# Check whether the files are already present
log.debug("Checking whether files are already in the repository")
toinstall = []
pool_dir = os.path.join(destdir, self.changes.get_pool_path())
log.debug("Pool directory: %s", pool_dir)
for file in self.files:
if os.path.isfile(file) and os.path.isfile(os.path.join(pool_dir, file)):
log.warning('%s is being installed even though it already exists' % file)
toinstall.append(file)
elif os.path.isfile(file):
log.debug('File %s is safe to install' % os.path.join(pool_dir, file))
toinstall.append(file)
# skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
# by doing nothing here for that case
# initiate the git storage
gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
if os.path.isdir(os.path.join(destdir, 'git', 'last')):
log.debug("folder exist, cleaning it")
shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
else:
log.debug("last folder doesn't exist, creating one")
os.makedirs(os.path.join(destdir, 'git', 'last', ''))
#building sources
log.debug("building sources!")
self._build_source(os.path.join(destdir, 'git', 'last', ''))
#removing older file
if os.path.isdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])):
for i in glob(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'], '*')):
if os.path.isdir(i):
shutil.rmtree(i)
else:
os.remove(i)
os.rmdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
shutil.copytree(os.path.join(destdir, 'git', 'last', 'extracted'),
os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])
fileToAdd = self._get_files(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
fileToAdd = self._clean_path(os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
gs.change(fileToAdd)
# Run post-upload plugins.
post_upload = Plugins('post-upload', self.changes, self.changes_file,
user_id=self.user_id)
if post_upload.stop():
log.critical('post-upload plugins failed')
self._remove_changes()
sys.exit(1)
# Check whether a post-upload plugin has got the orig tarball from somewhere.
if not orig_file_found and not filecheck.is_native_package(self.changes):
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
# When coming here it means:
# a) The uploader did not include a orig.tar.gz in his upload
# b) We couldn't find a orig.tar.gz in our repository
# c) No plugin could get the orig.tar.gz
# ... time to give up
if orig == None:
orig = "any original tarball (orig.tar.gz)"
self._reject("Rejecting incomplete upload. "
"You did not upload %s and we didn't find it on any of our alternative resources.\n"\
"If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
( orig ))
else:
toinstall.append(orig)
# Check whether the debexpo.repository variable is set
if 'debexpo.repository' not in pylons.config:
self._fail('debexpo.repository not set')
# Check whether debexpo.repository is a directory
if not os.path.isdir(pylons.config['debexpo.repository']):
self._fail('debexpo.repository is not a directory')
# Check whether debexpo.repository is writeable
if not os.access(pylons.config['debexpo.repository'], os.W_OK):
self._fail('debexpo.repository is not writeable')
qa = Plugins('qa', self.changes, self.changes_file, user_id=self.user_id)
if qa.stop():
self._reject('QA plugins failed the package')
# Loop through parent directories in the target installation directory to make sure they
# all exist. If not, create them.
for dir in self.changes.get_pool_path().split('/'):
destdir = os.path.join(destdir, dir)
if not os.path.isdir(destdir):
log.debug('Creating directory: %s' % destdir)
os.mkdir(destdir)
# Install files in repository
for file in toinstall:
log.debug("Installing new file %s" % (file))
shutil.move(file, os.path.join(destdir, file))
#git job is done, cleaning
shutil.rmtree(os.path.join(pylons.config['debexpo.repository'], 'git', 'last', ''))
self._remove_temporary_files()
# Create the database rows
self._create_db_entries(qa)
# Execute I'm happy to have post-successful-upload plugins
f = open(self.changes_file)
changes_contents = f.read()
f.close()
Plugins('post-successful-upload', self.changes, self.changes_file,
changes_contents=changes_contents)
# Remove the changes file
self._remove_changes()
# Refresh the Sources/Packages files.
log.debug('Updating Sources and Packages files')
r = Repository(pylons.config['debexpo.repository'])
r.update()
log.debug('Done')
return 0
def main(self):
"""
Actually start the import of the package.
Do several environment sanity checks, move files into the right place, and then
create the database entries for the imported package.
"""
# Set up importer
self._setup()
log.debug('Importer started with arguments: %s' % sys.argv[1:])
filecheck = CheckFiles()
gpg = get_gnupg()
# Try parsing the changes file, but fail if there's an error.
try:
self.changes = Changes(filename=self.changes_file)
filecheck.test_files_present(self.changes)
filecheck.test_md5sum(self.changes)
except Exception as e:
# XXX: The user won't ever see this message. The changes file was
# invalid, we don't know whom send it to
self._reject(
"Your changes file appears invalid. Refusing your upload\n%s" %
(e.message))
if not self.skip_gpg:
# Next, find out whether the changes file was signed with a valid signature, if not reject immediately
v = gpg.verify_file(path=self.changes_file)
if v is None:
self._reject('Your upload does not appear to be signed')
if not v.is_valid:
self._reject(
'Your upload does not contain a valid signature. Output was:\n%s'
% (gpg_out))
self._determine_uploader_by_gpg(v.key_id)
else:
self._determine_uploader_by_changedby_field()
if self.user is None:
# Creates a fake user object, but only if no user was found before
# This is useful to have a user object to send reject mails to
# generate user object, but only to send out reject message
self.user = User(id=-1,
name=maintainer_realname,
email=maintainer_email_address)
self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)
self.user_id = self.user.id
log.debug("User found in database. Has id: %s", self.user.id)
if self.user.dmup is False:
log.debug("User has not accepted the DMUP")
self._reject(
'You must accept the Debian Machine Usage Policies before uploading a package'
)
self.files = self.changes.get_files()
self.files_to_remove = []
distribution = self.changes['Distribution'].lower()
allowed_distributions = ('oldstable', 'stable', 'unstable',
'experimental', 'stable-backports',
'oldstable-backports',
'oldstable-backports-sloppy',
'oldstable-security', 'stable-security',
'testing-security', 'stable-proposed-updates',
'testing-proposed-updates', 'sid', 'wheezy',
'squeeze', 'lenny', 'squeeze-backports',
'lenny-backports', 'lenny-security',
'lenny-backports-sloppy', 'lenny-volatile',
'squeeze-security', 'squeeze-updates',
'wheezy-security', 'unreleased')
if distribution not in allowed_distributions:
self._reject(
"You are not uploading to one of those Debian distributions: %s"
% (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))
# Look whether the orig tarball is present, and if not, try and get it from
# the repository.
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
log.debug(
"Upload does not contain orig.tar.gz - trying to find it elsewhere"
)
if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
filename = os.path.join(pylons.config['debexpo.repository'],
self.changes.get_pool_path(), orig)
if os.path.isfile(filename):
log.debug("Found tar.gz in repository as %s" % (filename))
shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
# We need the orig.tar.gz for the import run, plugins need to extract the source package
# also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
# remove it later
self.files_to_remove.append(orig)
destdir = pylons.config['debexpo.repository']
# Check whether the files are already present
log.debug("Checking whether files are already in the repository")
toinstall = []
pool_dir = os.path.join(destdir, self.changes.get_pool_path())
log.debug("Pool directory: %s", pool_dir)
for file in self.files:
if os.path.isfile(file) and os.path.isfile(
os.path.join(pool_dir, file)):
log.warning(
'%s is being installed even though it already exists' %
file)
toinstall.append(file)
elif os.path.isfile(file):
log.debug('File %s is safe to install' %
os.path.join(pool_dir, file))
toinstall.append(file)
# skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
# by doing nothing here for that case
# initiate the git storage
gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
if os.path.isdir(os.path.join(destdir, 'git', 'last')):
log.debug("folder exist, cleaning it")
shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
else:
log.debug("last folder doesn't exist, creating one")
os.makedirs(os.path.join(destdir, 'git', 'last', ''))
#building sources
log.debug("building sources!")
self._build_source(os.path.join(destdir, 'git', 'last', ''))
#removing older file
if os.path.isdir(
os.path.join(destdir, 'git', self.changes['source'],
self.changes['source'])):
for i in glob(
os.path.join(destdir, 'git', self.changes['source'],
self.changes['source'], '*')):
if os.path.isdir(i):
shutil.rmtree(i)
else:
os.remove(i)
os.rmdir(
os.path.join(destdir, 'git', self.changes['source'],
self.changes['source']))
shutil.copytree(
os.path.join(destdir, 'git', 'last', 'extracted'),
os.path.join(destdir, 'git', self.changes['source'],
self.changes['source']))
os.path.join(destdir, 'git', self.changes['source'],
self.changes['source'])
fileToAdd = self._get_files(
os.path.join(destdir, 'git', self.changes['source'],
self.changes['source']))
fileToAdd = self._clean_path(
os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
gs.change(fileToAdd)
# Run post-upload plugins.
post_upload = Plugins('post-upload',
self.changes,
self.changes_file,
user_id=self.user_id)
if post_upload.stop():
log.critical('post-upload plugins failed')
self._remove_changes()
sys.exit(1)
# Check whether a post-upload plugin has got the orig tarball from somewhere.
if not orig_file_found and not filecheck.is_native_package(
self.changes):
(orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
# When coming here it means:
# a) The uploader did not include a orig.tar.gz in his upload
# b) We couldn't find a orig.tar.gz in our repository
# c) No plugin could get the orig.tar.gz
# ... time to give up
if orig == None:
orig = "any original tarball (orig.tar.gz)"
self._reject("Rejecting incomplete upload. "
"You did not upload %s and we didn't find it on any of our alternative resources.\n"\
"If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
( orig ))
else:
toinstall.append(orig)
# Check whether the debexpo.repository variable is set
if 'debexpo.repository' not in pylons.config:
self._fail('debexpo.repository not set')
# Check whether debexpo.repository is a directory
if not os.path.isdir(pylons.config['debexpo.repository']):
self._fail('debexpo.repository is not a directory')
# Check whether debexpo.repository is writeable
if not os.access(pylons.config['debexpo.repository'], os.W_OK):
self._fail('debexpo.repository is not writeable')
qa = Plugins('qa',
self.changes,
self.changes_file,
user_id=self.user_id)
if qa.stop():
self._reject('QA plugins failed the package')
# Loop through parent directories in the target installation directory to make sure they
# all exist. If not, create them.
for dir in self.changes.get_pool_path().split('/'):
destdir = os.path.join(destdir, dir)
if not os.path.isdir(destdir):
log.debug('Creating directory: %s' % destdir)
os.mkdir(destdir)
# Install files in repository
for file in toinstall:
log.debug("Installing new file %s" % (file))
shutil.move(file, os.path.join(destdir, file))
#git job is done, cleaning
shutil.rmtree(
os.path.join(pylons.config['debexpo.repository'], 'git', 'last',
''))
self._remove_temporary_files()
# Create the database rows
self._create_db_entries(qa)
# Execute I'm happy to have post-successful-upload plugins
f = open(self.changes_file)
changes_contents = f.read()
f.close()
Plugins('post-successful-upload',
self.changes,
self.changes_file,
changes_contents=changes_contents)
# Remove the changes file
self._remove_changes()
# Refresh the Sources/Packages files.
log.debug('Updating Sources and Packages files')
r = Repository(pylons.config['debexpo.repository'])
r.update()
log.debug('Done')
return 0