Exemplo n.º 1
0
    def __init__(self, methodName):
        """
        Class constructor. Starts a Changes class for use in other tests.
        """
        TestCase.__init__(self, methodName)

        self.changes = Changes(
            filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')
Exemplo n.º 2
0
class TestChangesController(TestCase):
    def __init__(self, methodName):
        """
        Class constructor. Starts a Changes class for use in other tests.
        """
        TestCase.__init__(self, methodName)

        self.changes = Changes(
            filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')

    def testInitialize(self):
        """
        Tests whether the two ways of creating a Changes object output the same result.
        """
        f = open('debexpo/tests/changes/synce-hal_0.1-1_source.changes')
        contents = f.read()
        f.close()

        a = Changes(
            filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')
        b = Changes(string=contents)

        self.assertEqual(a['Source'], b['Source'])

    def testGetitem(self):
        """
        Tests Changes.__getitem__.
        """
        self.assertEqual(self.changes['Source'], 'synce-hal')
        self.assertEqual(self.changes['Urgency'], 'low')

    def testGetFiles(self):
        """
        Tests Changes.get_files.
        """
        self.assertEqual(self.changes.get_files(), [
            'synce-hal_0.1-1.dsc', 'synce-hal_0.1.orig.tar.gz',
            'synce-hal_0.1-1.diff.gz'
        ])

    def testGetComponent(self):
        """
        Tests Changes.get_component().
        """
        self.assertEqual(self.changes.get_component(), 'main')

    def testGetPriority(self):
        """
        Tests Changes.get_priority().
        """
        self.assertEqual(self.changes.get_priority(), 'optional')

    def testGetDsc(self):
        """
        Tests Changes.get_dsc().
        """
        self.assertEqual(self.changes.get_dsc(), 'synce-hal_0.1-1.dsc')
Exemplo n.º 3
0
class TestChangesController(TestCase):

    def __init__(self, methodName):
        """
        Class constructor. Starts a Changes class for use in other tests.
        """
        TestCase.__init__(self, methodName)

        self.changes = Changes(filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')

    def testInitialize(self):
        """
        Tests whether the two ways of creating a Changes object output the same result.
        """
        f = open('debexpo/tests/changes/synce-hal_0.1-1_source.changes')
        contents = f.read()
        f.close()

        a = Changes(filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')
        b = Changes(string=contents)

        self.assertEqual(a['Source'], b['Source'])

    def testGetitem(self):
        """
        Tests Changes.__getitem__.
        """
        self.assertEqual(self.changes['Source'], 'synce-hal')
        self.assertEqual(self.changes['Urgency'], 'low')

    def testGetFiles(self):
        """
        Tests Changes.get_files.
        """
        self.assertEqual(self.changes.get_files(), ['synce-hal_0.1-1.dsc', 'synce-hal_0.1.orig.tar.gz', 'synce-hal_0.1-1.diff.gz'])

    def testGetComponent(self):
        """
        Tests Changes.get_component().
        """
        self.assertEqual(self.changes.get_component(), 'main')

    def testGetPriority(self):
        """
        Tests Changes.get_priority().
        """
        self.assertEqual(self.changes.get_priority(), 'optional')

    def testGetDsc(self):
        """
        Tests Changes.get_dsc().
        """
        self.assertEqual(self.changes.get_dsc(), 'synce-hal_0.1-1.dsc')
Exemplo n.º 4
0
    def testInitialize(self):
        """
        Tests whether the two ways of creating a Changes object output the same result.
        """
        f = open('debexpo/tests/changes/synce-hal_0.1-1_source.changes')
        contents = f.read()
        f.close()

        a = Changes(
            filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')
        b = Changes(string=contents)

        self.assertEqual(a['Source'], b['Source'])
Exemplo n.º 5
0
    def __init__(self, methodName):
        """
        Class constructor. Starts a Changes class for use in other tests.
        """
        TestCase.__init__(self, methodName)

        self.changes = Changes(filename='debexpo/tests/changes/synce-hal_0.1-1_source.changes')
Exemplo n.º 6
0
    def main(self):
        """
        Parse the d-d-c email and take action.
        """
        # Set up
        self._setup()

        if log is not None:
            log.debug('d-d-c parser started with arguments: %s' % sys.argv[1:])

        from debexpo.lib.changes import Changes
        from debexpo.lib.repository import Repository
        from debexpo.lib.plugins import Plugins

        changes = Changes(string=self.email)

        Plugins('post-upload-to-debian', changes, None)

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(self.config['debexpo.repository'])
        r.update()

        log.debug('Done')
Exemplo n.º 7
0
class Importer(object):
    """
    Class to handle the package that is uploaded and wants to be imported into the database.
    """

    def __init__(self, changes, ini, skip_email, skip_gpg):
        """
        Object constructor. Sets class fields to sane values.

        ``self``
            Object pointer.

        ``changes``
            Name `changes` file to import. This is given from the upload controller.

        ``ini``
            Path to debexpo configuration file. This is given from the upload controller.

        ``skip_email``
            If this is set to true, send no email.
        """
        self.changes_file_unqualified = changes
        self.ini_file = os.path.abspath(ini)
        self.actually_send_email = not bool(skip_email)
        self.skip_gpg = skip_gpg

        self.user_id = None
        self.changes = None
        self.user = None

    def send_email(self, email, *args, **kwargs):
        if self.actually_send_email:
            email.send(*args, **kwargs)
        else:
            logging.info("Skipping email send: %s %s", args, kwargs)

    @property
    def changes_file(self):
        incoming_dir = pylons.config['debexpo.upload.incoming']
        return os.path.join(incoming_dir, self.changes_file_unqualified)

    def _clean_path(self, pathToRemove, files):
        """
        remove a path from another using magic split
        ''pathToRemove''
            the path to remove
        ''filePath''
            the a list of original filePath
        """
        result = []
        for filePath in files:
            filePath = string.split(filePath, pathToRemove)
            filePath = filePath[1]
            filePath = string.split(filePath, os.sep)
            if filePath[0] == '':
                filePath.remove('')
            fileName = string.join(filePath, os.sep)
            result.append(fileName)
        return result

    def _get_files(self, path):
        """
        return all files in a path
        ''path''
            the path we are searching
        """
        result = []
        for f in os.listdir(path):
            if os.path.isdir(os.path.join(path, f)):
                result += self._get_files(os.path.join(path, f))
            else:
                result.append(os.path.join(path, f))
        return result

    def _build_source(self, gitpath):
        log.debug('Copying files to a temp directory to run dpkg-source -x on the dsc file')
        self.tempdir = gitpath
        log.debug('Temp dir is: %s', self.tempdir)
        for filename in self.changes.get_files():
            log.debug('Copying: %s', filename)
            shutil.copy(os.path.join(pylons.config['debexpo.upload.incoming'], filename), self.tempdir)

        # If the original tarball was pulled from Debian or from the repository, that
        # also needs to be copied into this directory.
        dsc = deb822.Dsc(file(self.changes.get_dsc()))
        for item in dsc['Files']:
            if item['name'] not in self.changes.get_files():
                src_file = os.path.join(pylons.config['debexpo.upload.incoming'], item['name'])
                repository_src_file = os.path.join(pylons.config['debexpo.repository'], self.changes.get_pool_path(),
                    item['name'])
                if os.path.exists(src_file):
                    shutil.copy(src_file, self.tempdir)
                elif os.path.exists(repository_src_file):
                    shutil.copy(repository_src_file, self.tempdir)
                else:
                    log.critical("Trying to copy non-existing file %s" % (src_file))

        shutil.copy(os.path.join(pylons.config['debexpo.upload.incoming'], self.changes_file), self.tempdir)
        self.oldcurdir = os.path.abspath(os.path.curdir)
        cwd = os.getcwd()
        os.chdir(self.tempdir)
        os.system('dpkg-source -x %s extracted' % self.changes.get_dsc())
        os.chdir(cwd)

    def _remove_changes(self):
        """
        Removes the `changes` file.
        """
        if os.path.exists(self.changes_file):
            os.remove(self.changes_file)

    def _remove_temporary_files(self):
        if hasattr(self, 'files_to_remove'):
            for file in self.files_to_remove:
                if os.path.exists(file):
                    os.remove(file)

    def _remove_files(self):
        """
        Removes all the files uploaded.
        """
        if hasattr(self, 'files'):
            for file in self.files:
                if os.path.exists(file):
                    os.remove(file)

        self._remove_changes()
        self._remove_temporary_files()

    def _fail(self, reason, use_log=True):
        """
        Fail the upload by sending a reason for failure to the log and then remove all
        uploaded files.

        A package is `fail`ed if there is a problem with debexpo, **not** if there's
        something wrong with the package.

        ``reason``
            String of why it failed.

        ``use_log``
            Whether to use the log. This should only be False when actually loading the log fails.
            In this case, the reason is printed to stderr.
        """
        if use_log:
            log.critical(reason)
        else:
            print >> sys.stderr, reason

        self._remove_files()

        if self.user is not None:
            email = Email('importer_fail_maintainer')
            package = self.changes.get('Source', '')

            self.send_email(email, [self.user.email], package=package)

        email = Email('importer_fail_admin')
        self.send_email(email, [pylons.config['debexpo.email']], message=reason)

        sys.exit(1)

    def _reject(self, reason):
        """
        Reject the package by sending a reason for failure to the log and then remove all
        uploaded files.

        A package is `reject`ed if there is a problem with the package.

        ``reason``
            String of why it failed.
        """
        log.error('Rejected: %s' % reason)

        self._remove_changes()
        self._remove_files()

        if self.user is not None:
            email = Email('importer_reject_maintainer')
            package = self.changes.get('Source', '')

            self.send_email(email, [self.user.email], package=package, message=reason)
        sys.exit(1)

    def _setup_logging(self):
        """
        Parse the config file and create the ``log`` object for other methods to log their
        actions.
        """
        global log

        # Parse the ini file to validate it
        parser = ConfigParser.ConfigParser()
        parser.read(self.ini_file)

        # Check for the presence of [loggers] in self.ini_file
        if not parser.has_section('loggers'):
            self._fail('Config file does not have [loggers] section', use_log=False)

        logging.config.fileConfig(self.ini_file)

        # Use "name.pid" to avoid importer confusions in the logs
        logger_name = 'debexpo.importer.%s' % os.getpid()
        log = logging.getLogger(logger_name)

    def _setup(self):
        """
        Set up logging, import pylons/paste/debexpo modules, parse config file, create config
        class and chdir to the incoming directory.
        """
        # Look for ini file
        if not os.path.isfile(self.ini_file):
            self._fail('Cannot find ini file')

        self._setup_logging()

        # Import debexpo root directory
        sys.path.append(os.path.dirname(self.ini_file))

        # Initialize Pylons app
        conf = appconfig('config:' + self.ini_file)
        pylons.config = load_environment(conf.global_conf, conf.local_conf)

        # Change into the incoming directory
        incoming_dir = pylons.config['debexpo.upload.incoming']
        logging.info("Changing dir to %s", incoming_dir)
        os.chdir(incoming_dir)

        # Look for the changes file
        if not os.path.isfile(self.changes_file):
            self._fail('Cannot find changes file')

    def _create_db_entries(self, qa):
        """
        Create entries in the Database for the package upload.
        """

        def _package_description(raw):
            return raw[2:].replace('      - ', ' - ')

        log.debug('Creating database entries')


        # Parse component and section from field in changes
        component, section = parse_section(self.changes['files'][0]['section'])

        # Check whether package is already in the database
        package_query = meta.session.query(Package).filter_by(name=self.changes['Source'])
        if package_query.count() == 1:
            log.debug('Package %s already exists in the database' % self.changes['Source'])
            package = package_query.one()
            # Update description to make sure it reflects the latest upload
            package.description = _package_description(self.changes['Description'])
        else:
            log.debug('Package %s is new to the system' % self.changes['Source'])
            package = Package(name=self.changes['Source'], user=self.user)
            package.description = _package_description(self.changes['Description'])
            package.needs_sponsor = 0
            meta.session.add(package)

        # No need to check whether there is the same source name and same version as an existing
        # entry in the database as the upload controller tested whether similar filenames existed
        # in the repository. The only way this would be wrong is if the filename had a different
        # version in than the Version field in changes..


        try:
            closes = self.changes['Closes']
        except KeyError:
            closes = None

        # TODO: fix these magic numbers
        if qa.stop():
            qa_status = 1
        else:
            qa_status = 0

        maintainer_matches = re.compile(r'(.*) <(.*)>').match(self.changes['Changed-By'])
        maintainer = maintainer_matches.group(2)

        package_version = PackageVersion(package=package, version=self.changes['Version'],
            section=section, distribution=self.changes['Distribution'], qa_status=qa_status,
            component=component, priority=self.changes.get_priority(), closes=closes,
            uploaded=datetime.now(), maintainer=maintainer)
        meta.session.add(package_version)

        source_package = SourcePackage(package_version=package_version)
        meta.session.add(source_package)

        binary_package = None

        # Add PackageFile objects to the database for each uploaded file
        for file in self.files:
            filename = os.path.join(self.changes.get_pool_path(), file)
            # This exception should be never caught.
            # It implies something went wrong before, as we expect a file which does not exist
            try:
                sum = md5sum(os.path.join(pylons.config['debexpo.repository'], filename))
            except AttributeError as e:
                self._fail("Could not calculate MD5 sum: %s" % (e))

            size = os.stat(os.path.join(pylons.config['debexpo.repository'], filename))[ST_SIZE]

            # Check for binary or source package file
            if file.endswith('.deb'):
                # Only create a BinaryPackage if there actually binary package files
                if binary_package is None:
                    binary_package = BinaryPackage(package_version=package_version, arch=file[:-4].split('_')[-1])
                    meta.session.add(binary_package)

                meta.session.add(PackageFile(filename=filename, binary_package=binary_package, size=size, md5sum=sum))
            else:
                meta.session.add(PackageFile(filename=filename, source_package=source_package, size=size, md5sum=sum))

        meta.session.commit()
        log.warning("Finished adding PackageFile objects.")

        # Add PackageInfo objects to the database for the package_version
        for result in qa.result:
            meta.session.add(PackageInfo(package_version=package_version, from_plugin=result.from_plugin,
                outcome=result.outcome, rich_data=result.data, severity=result.severity))

        # Commit all changes to the database
        meta.session.commit()
        log.debug('Committed package data to the database')

        subscribers = meta.session.query(PackageSubscription).filter_by(package=self.changes['Source']).filter(\
            PackageSubscription.level <= constants.SUBSCRIPTION_LEVEL_UPLOADS).all()

        if len(subscribers) > 0:
            email = Email('package_uploaded')
            self.send_email(email, [s.user.email for s in subscribers], package=self.changes['Source'],
                version=self.changes['Version'], user=self.user)

            log.debug('Sent out package subscription emails')

        # Send success email to uploader
        email = Email('successful_upload')
        dsc_url = pylons.config[
                  'debexpo.server'] + '/debian/' + self.changes.get_pool_path() + '/' + self.changes.get_dsc()
        rfs_url = pylons.config['debexpo.server'] + url('rfs', packagename=self.changes['Source'])
        self.send_email(email, [self.user.email], package=self.changes['Source'],
            dsc_url=dsc_url, rfs_url=rfs_url)

    def _find_user_by_email_address(self, email_address):
        """
        Searches user by email address
        """
        # XXX: Maybe model is more appropriate place for such a method
        self.user = meta.session.query(User).filter_by(email=email_address).filter_by(verification=None).first()
        return self.user


    def _determine_uploader_by_changedby_field(self):
        """
        Create a user object based on the Changed-By entry
        """
        maintainer_string = self.changes.get('Changed-By')
        log.debug("Determining user from 'Changed-By:' field: %s" % maintainer_string)
        maintainer_realname, maintainer_email_address = email.utils.parseaddr(maintainer_string)
        log.debug("Changed-By's email address is: %s", maintainer_email_address)
        self._find_user_by_email_address(maintainer_email_address)

    def _determine_uploader_by_gpg(self, gpg_keyid):
        """
        Create a user object based on the gpg output
        """

        try:
            self.user = meta.session.query(User).filter_by(gpg_id=gpg_keyid[-8:]).one()
        except exceptions.InvalidRequestError:
            return None
        else:
            log.debug("GPG signature mapped to user %s <%s>" % (self.user.name,
                                                                self.user.email))

    def main(self):
        """
        Actually start the import of the package.

        Do several environment sanity checks, move files into the right place, and then
        create the database entries for the imported package.
        """
        # Set up importer
        self._setup()

        log.debug('Importer started with arguments: %s' % sys.argv[1:])
        filecheck = CheckFiles()
        gpg = get_gnupg()

        # Try parsing the changes file, but fail if there's an error.
        try:
            self.changes = Changes(filename=self.changes_file)
            filecheck.test_files_present(self.changes)
            filecheck.test_md5sum(self.changes)
        except Exception as e:
            # XXX: The user won't ever see this message. The changes file was
            # invalid, we don't know whom send it to
            self._reject("Your changes file appears invalid. Refusing your upload\n%s" % (e.message))

        if not self.skip_gpg:
            # Next, find out whether the changes file was signed with a valid signature, if not reject immediately
            v = gpg.verify_file(path=self.changes_file)
            if v is None:
                self._reject('Your upload does not appear to be signed')

            if not v.is_valid:
                self._reject('Your upload does not contain a valid signature. Output was:\n%s' % (gpg_out))
            self._determine_uploader_by_gpg(v.key_id)
        else:
            self._determine_uploader_by_changedby_field()

        if self.user is None:
            # Creates a fake user object, but only if no user was found before
            # This is useful to have a user object to send reject mails to
            # generate user object, but only to send out reject message
            self.user = User(id=-1, name=maintainer_realname, email=maintainer_email_address)
            self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)

        self.user_id = self.user.id
        log.debug("User found in database. Has id: %s", self.user.id)

        if self.user.dmup is False:
            log.debug("User has not accepted the DMUP")
            self._reject('You must accept the Debian Machine Usage Policies before uploading a package')

        self.files = self.changes.get_files()
        self.files_to_remove = []

        distribution = self.changes['Distribution'].lower()
        allowed_distributions = (
            'oldstable', 'stable', 'unstable', 'experimental', 'stable-backports', 'oldstable-backports',
            'oldstable-backports-sloppy', 'oldstable-security', 'stable-security', 'testing-security',
            'stable-proposed-updates',
            'testing-proposed-updates', 'sid', 'wheezy', 'squeeze', 'lenny', 'squeeze-backports', 'lenny-backports',
            'lenny-security', 'lenny-backports-sloppy', 'lenny-volatile', 'squeeze-security', 'squeeze-updates',
            'wheezy-security',
            'unreleased')
        if distribution not in allowed_distributions:
            self._reject("You are not uploading to one of those Debian distributions: %s" %
                         (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))

        # Look whether the orig tarball is present, and if not, try and get it from
        # the repository.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
            log.debug("Upload does not contain orig.tar.gz - trying to find it elsewhere")
        if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
            filename = os.path.join(pylons.config['debexpo.repository'],
                self.changes.get_pool_path(), orig)
            if os.path.isfile(filename):
                log.debug("Found tar.gz in repository as %s" % (filename))
                shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
                # We need the orig.tar.gz for the import run, plugins need to extract the source package
                # also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
                # remove it later
                self.files_to_remove.append(orig)

        destdir = pylons.config['debexpo.repository']


        # Check whether the files are already present
        log.debug("Checking whether files are already in the repository")
        toinstall = []
        pool_dir = os.path.join(destdir, self.changes.get_pool_path())
        log.debug("Pool directory: %s", pool_dir)
        for file in self.files:
            if os.path.isfile(file) and os.path.isfile(os.path.join(pool_dir, file)):
                log.warning('%s is being installed even though it already exists' % file)
                toinstall.append(file)
            elif os.path.isfile(file):
                log.debug('File %s is safe to install' % os.path.join(pool_dir, file))
                toinstall.append(file)
                # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
                # by doing nothing here for that case



        # initiate the git storage
        gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
        if os.path.isdir(os.path.join(destdir, 'git', 'last')):
            log.debug("folder exist, cleaning it")
            shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
        else:
            log.debug("last folder doesn't exist, creating one")
        os.makedirs(os.path.join(destdir, 'git', 'last', ''))

        #building sources

        log.debug("building sources!")
        self._build_source(os.path.join(destdir, 'git', 'last', ''))

        #removing older file
        if os.path.isdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])):
            for i in glob(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'], '*')):
                if os.path.isdir(i):
                    shutil.rmtree(i)
                else:
                    os.remove(i)
            os.rmdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        shutil.copytree(os.path.join(destdir, 'git', 'last', 'extracted'),
            os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])
        fileToAdd = self._get_files(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        fileToAdd = self._clean_path(os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
        gs.change(fileToAdd)



        # Run post-upload plugins.
        post_upload = Plugins('post-upload', self.changes, self.changes_file,
            user_id=self.user_id)
        if post_upload.stop():
            log.critical('post-upload plugins failed')
            self._remove_changes()
            sys.exit(1)

        # Check whether a post-upload plugin has got the orig tarball from somewhere.
        if not orig_file_found and not filecheck.is_native_package(self.changes):
            (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                # When coming here it means:
                # a) The uploader did not include a orig.tar.gz in his upload
                # b) We couldn't find a orig.tar.gz in our repository
                # c) No plugin could get the orig.tar.gz
                # ... time to give up
                if orig == None:
                    orig = "any original tarball (orig.tar.gz)"
                self._reject("Rejecting incomplete upload. "
                             "You did not upload %s and we didn't find it on any of our alternative resources.\n"\
                             "If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
                             ( orig ))
            else:
                toinstall.append(orig)

        # Check whether the debexpo.repository variable is set
        if 'debexpo.repository' not in pylons.config:
            self._fail('debexpo.repository not set')

        # Check whether debexpo.repository is a directory
        if not os.path.isdir(pylons.config['debexpo.repository']):
            self._fail('debexpo.repository is not a directory')

        # Check whether debexpo.repository is writeable
        if not os.access(pylons.config['debexpo.repository'], os.W_OK):
            self._fail('debexpo.repository is not writeable')

        qa = Plugins('qa', self.changes, self.changes_file, user_id=self.user_id)
        if qa.stop():
            self._reject('QA plugins failed the package')

        # Loop through parent directories in the target installation directory to make sure they
        # all exist. If not, create them.
        for dir in self.changes.get_pool_path().split('/'):
            destdir = os.path.join(destdir, dir)

            if not os.path.isdir(destdir):
                log.debug('Creating directory: %s' % destdir)
                os.mkdir(destdir)

        # Install files in repository
        for file in toinstall:
            log.debug("Installing new file %s" % (file))
            shutil.move(file, os.path.join(destdir, file))
            #git job is done, cleaning
        shutil.rmtree(os.path.join(pylons.config['debexpo.repository'], 'git', 'last', ''))

        self._remove_temporary_files()
        # Create the database rows
        self._create_db_entries(qa)

        # Execute I'm happy to have post-successful-upload plugins
        f = open(self.changes_file)
        changes_contents = f.read()
        f.close()
        Plugins('post-successful-upload', self.changes, self.changes_file,
            changes_contents=changes_contents)

        # Remove the changes file
        self._remove_changes()

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(pylons.config['debexpo.repository'])
        r.update()

        log.debug('Done')
        return 0
Exemplo n.º 8
0
    def main(self):
        """
        Actually start the import of the package.

        Do several environment sanity checks, move files into the right place, and then
        create the database entries for the imported package.
        """
        # Set up importer
        self._setup()

        log.debug('Importer started with arguments: %s' % sys.argv[1:])
        filecheck = CheckFiles()
        gpg = get_gnupg()

        # Try parsing the changes file, but fail if there's an error.
        try:
            self.changes = Changes(filename=self.changes_file)
            filecheck.test_files_present(self.changes)
            filecheck.test_md5sum(self.changes)
        except Exception as e:
            # XXX: The user won't ever see this message. The changes file was
            # invalid, we don't know whom send it to
            self._reject("Your changes file appears invalid. Refusing your upload\n%s" % (e.message))

        if not self.skip_gpg:
            # Next, find out whether the changes file was signed with a valid signature, if not reject immediately
            v = gpg.verify_file(path=self.changes_file)
            if v is None:
                self._reject('Your upload does not appear to be signed')

            if not v.is_valid:
                self._reject('Your upload does not contain a valid signature. Output was:\n%s' % (gpg_out))
            self._determine_uploader_by_gpg(v.key_id)
        else:
            self._determine_uploader_by_changedby_field()

        if self.user is None:
            # Creates a fake user object, but only if no user was found before
            # This is useful to have a user object to send reject mails to
            # generate user object, but only to send out reject message
            self.user = User(id=-1, name=maintainer_realname, email=maintainer_email_address)
            self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)

        self.user_id = self.user.id
        log.debug("User found in database. Has id: %s", self.user.id)

        if self.user.dmup is False:
            log.debug("User has not accepted the DMUP")
            self._reject('You must accept the Debian Machine Usage Policies before uploading a package')

        self.files = self.changes.get_files()
        self.files_to_remove = []

        distribution = self.changes['Distribution'].lower()
        allowed_distributions = (
            'oldstable', 'stable', 'unstable', 'experimental', 'stable-backports', 'oldstable-backports',
            'oldstable-backports-sloppy', 'oldstable-security', 'stable-security', 'testing-security',
            'stable-proposed-updates',
            'testing-proposed-updates', 'sid', 'wheezy', 'squeeze', 'lenny', 'squeeze-backports', 'lenny-backports',
            'lenny-security', 'lenny-backports-sloppy', 'lenny-volatile', 'squeeze-security', 'squeeze-updates',
            'wheezy-security',
            'unreleased')
        if distribution not in allowed_distributions:
            self._reject("You are not uploading to one of those Debian distributions: %s" %
                         (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))

        # Look whether the orig tarball is present, and if not, try and get it from
        # the repository.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
            log.debug("Upload does not contain orig.tar.gz - trying to find it elsewhere")
        if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
            filename = os.path.join(pylons.config['debexpo.repository'],
                self.changes.get_pool_path(), orig)
            if os.path.isfile(filename):
                log.debug("Found tar.gz in repository as %s" % (filename))
                shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
                # We need the orig.tar.gz for the import run, plugins need to extract the source package
                # also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
                # remove it later
                self.files_to_remove.append(orig)

        destdir = pylons.config['debexpo.repository']


        # Check whether the files are already present
        log.debug("Checking whether files are already in the repository")
        toinstall = []
        pool_dir = os.path.join(destdir, self.changes.get_pool_path())
        log.debug("Pool directory: %s", pool_dir)
        for file in self.files:
            if os.path.isfile(file) and os.path.isfile(os.path.join(pool_dir, file)):
                log.warning('%s is being installed even though it already exists' % file)
                toinstall.append(file)
            elif os.path.isfile(file):
                log.debug('File %s is safe to install' % os.path.join(pool_dir, file))
                toinstall.append(file)
                # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
                # by doing nothing here for that case



        # initiate the git storage
        gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
        if os.path.isdir(os.path.join(destdir, 'git', 'last')):
            log.debug("folder exist, cleaning it")
            shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
        else:
            log.debug("last folder doesn't exist, creating one")
        os.makedirs(os.path.join(destdir, 'git', 'last', ''))

        #building sources

        log.debug("building sources!")
        self._build_source(os.path.join(destdir, 'git', 'last', ''))

        #removing older file
        if os.path.isdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])):
            for i in glob(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'], '*')):
                if os.path.isdir(i):
                    shutil.rmtree(i)
                else:
                    os.remove(i)
            os.rmdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        shutil.copytree(os.path.join(destdir, 'git', 'last', 'extracted'),
            os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])
        fileToAdd = self._get_files(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        fileToAdd = self._clean_path(os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
        gs.change(fileToAdd)



        # Run post-upload plugins.
        post_upload = Plugins('post-upload', self.changes, self.changes_file,
            user_id=self.user_id)
        if post_upload.stop():
            log.critical('post-upload plugins failed')
            self._remove_changes()
            sys.exit(1)

        # Check whether a post-upload plugin has got the orig tarball from somewhere.
        if not orig_file_found and not filecheck.is_native_package(self.changes):
            (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                # When coming here it means:
                # a) The uploader did not include a orig.tar.gz in his upload
                # b) We couldn't find a orig.tar.gz in our repository
                # c) No plugin could get the orig.tar.gz
                # ... time to give up
                if orig == None:
                    orig = "any original tarball (orig.tar.gz)"
                self._reject("Rejecting incomplete upload. "
                             "You did not upload %s and we didn't find it on any of our alternative resources.\n"\
                             "If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
                             ( orig ))
            else:
                toinstall.append(orig)

        # Check whether the debexpo.repository variable is set
        if 'debexpo.repository' not in pylons.config:
            self._fail('debexpo.repository not set')

        # Check whether debexpo.repository is a directory
        if not os.path.isdir(pylons.config['debexpo.repository']):
            self._fail('debexpo.repository is not a directory')

        # Check whether debexpo.repository is writeable
        if not os.access(pylons.config['debexpo.repository'], os.W_OK):
            self._fail('debexpo.repository is not writeable')

        qa = Plugins('qa', self.changes, self.changes_file, user_id=self.user_id)
        if qa.stop():
            self._reject('QA plugins failed the package')

        # Loop through parent directories in the target installation directory to make sure they
        # all exist. If not, create them.
        for dir in self.changes.get_pool_path().split('/'):
            destdir = os.path.join(destdir, dir)

            if not os.path.isdir(destdir):
                log.debug('Creating directory: %s' % destdir)
                os.mkdir(destdir)

        # Install files in repository
        for file in toinstall:
            log.debug("Installing new file %s" % (file))
            shutil.move(file, os.path.join(destdir, file))
            #git job is done, cleaning
        shutil.rmtree(os.path.join(pylons.config['debexpo.repository'], 'git', 'last', ''))

        self._remove_temporary_files()
        # Create the database rows
        self._create_db_entries(qa)

        # Execute I'm happy to have post-successful-upload plugins
        f = open(self.changes_file)
        changes_contents = f.read()
        f.close()
        Plugins('post-successful-upload', self.changes, self.changes_file,
            changes_contents=changes_contents)

        # Remove the changes file
        self._remove_changes()

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(pylons.config['debexpo.repository'])
        r.update()

        log.debug('Done')
        return 0
Exemplo n.º 9
0
    def invoke(self):
        """
        Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
        """
        if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(
                self.config['debexpo.upload.incoming']):
            self.log.critical("debexpo.upload.incoming was not configured")
            return

        # 1) Process uploads
        for file in glob.glob(
                os.path.join(
                    os.path.join(self.config['debexpo.upload.incoming'],
                                 "pub"), '*.changes')):
            try:
                changes = Changes(filename=file)
            except:
                self.log.error('Invalid changes file: %s' % (file))
                continue

            try:
                for filename in changes.get_files() + [
                        changes.get_filename(),
                ]:
                    source_file = os.path.join(
                        os.path.join(self.config['debexpo.upload.incoming'],
                                     "pub"), filename)
                    destination_file = os.path.join(
                        self.config['debexpo.upload.incoming'], filename)

                    if not os.path.exists(source_file):
                        self.log.debug(
                            "Source file %s does not exist - putting upload on hold"
                            % (source_file))
                        raise NotCompleteUpload
                    if os.path.exists(destination_file):
                        self.log.debug(
                            "Do not import %s: already exists on destination path - removing file instead"
                            % (source_file))
                        os.remove(source_file)
                        raise NotCompleteUpload
                    shutil.move(source_file,
                                self.config['debexpo.upload.incoming'])
            except NotCompleteUpload:
                continue

            self.log.info("Import upload: %s" % (file))
            importer = Importer(changes.get_filename(),
                                self.config['global_conf']['__file__'], False,
                                False)

            returncode = importer.main()
            if returncode != 0:
                self.log.critical(
                    "Importer failed to import package %s [err=%d]." %
                    (file, returncode))
            for filename in changes.get_files() + [
                    changes.get_filename(),
            ]:
                destination_file = os.path.join(
                    self.config['debexpo.upload.incoming'], filename)
                if os.path.exists(destination_file):
                    self.log.debug(
                        "Remove stale file %s - the importer probably crashed"
                        % (destination_file))
                    os.remove(destination_file)

        # 2) Mark unprocessed files and get rid of them after some time
        pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
        for file in glob.glob(os.path.join(pub, '*')):
            if self.files.allowed_upload(file):
                self.log.debug("Incomplete upload: %s" % (file))
                last_change = time.time() - os.stat(file).st_mtime
                # the file was uploaded more than 6 hours ago
                if last_change > 6 * 60 * 60:
                    self.log.warning(
                        "Remove old file: %s (last modified %.2f hours ago)" %
                        (file, last_change / 3600.))
                    os.remove(file)
            else:
                if os.path.isfile(file):
                    self.log.warning("Remove unknown file: %s" % (file))
                    os.remove(file)
Exemplo n.º 10
0
    def invoke(self):
        """
        Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
        """
        if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']):
            self.log.critical("debexpo.upload.incoming was not configured")
            return

        # 1) Process uploads
        for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ):
            try:
                changes = Changes(filename=file)
            except:
                self.log.error('Invalid changes file: %s' % (file))
                continue

            try:
                for filename in changes.get_files() + [ changes.get_filename(), ]:
                    source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename)
                    destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)

                    if not os.path.exists(source_file):
                            self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file))
                            raise NotCompleteUpload;
                    if os.path.exists(destination_file):
                            self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file))
                            os.remove(source_file)
                            raise NotCompleteUpload;
                    shutil.move(source_file, self.config['debexpo.upload.incoming'])
            except NotCompleteUpload:
                continue

            self.log.info("Import upload: %s" % (file))
	    importer = Importer(changes.get_filename(),
				self.config['global_conf']['__file__'],
				False,
				False)

	    returncode = importer.main()
            if returncode != 0:
                self.log.critical("Importer failed to import package %s [err=%d]." % (file, returncode))
            for filename in changes.get_files() + [ changes.get_filename(), ]:
                destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
                if os.path.exists(destination_file):
                    self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file))
                    os.remove(destination_file)

        # 2) Mark unprocessed files and get rid of them after some time
        pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
        for file in glob.glob( os.path.join(pub, '*') ):
            if self.files.allowed_upload(file):
                self.log.debug("Incomplete upload: %s" % (file))
                last_change = time.time() - os.stat(file).st_mtime
                # the file was uploaded more than 6 hours ago
                if last_change > 6 * 60 * 60:
                    self.log.warning("Remove old file: %s (last modified %.2f hours ago)" % (file, last_change / 3600.))
                    os.remove(file)
            else:
                if os.path.isfile(file):
                    self.log.warning("Remove unknown file: %s" % (file))
                    os.remove(file)
Exemplo n.º 11
0
    def invoke(self):
        """
        Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
        """
        if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']):
            self.log.critical("debexpo.upload.incoming was not configured")
            return

        # 1) Process uploads
        for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ):
            try:
                changes = Changes(filename=file)
            except:
                self.log.error('Invalid changes file: %s' % (file))
                continue

            try:
                for filename in changes.get_files() + [ changes.get_filename(), ]:
                    source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename)
                    destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)

                    if not os.path.exists(source_file):
                            self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file))
                            raise NotCompleteUpload;
                    if os.path.exists(destination_file):
                            self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file))
                            os.remove(source_file)
                            raise NotCompleteUpload;
                    shutil.move(source_file, self.config['debexpo.upload.incoming'])
            except NotCompleteUpload:
                continue

            self.log.info("Import upload: %s" % (file))
            command = [ self.config['debexpo.importer'], '-i', self.config['global_conf']['__file__'], '-c', changes.get_filename() ]
            self.log.debug("Executing: %s" % (" ".join(command)))
            proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (istdout, istderr) = proc.communicate()
            if proc.returncode != 0:
                self.log.critical("Importer failed to import package %s [err=%d]." % (file, proc.returncode))
                self.log.debug("Output was\n%s\n%s" % (istdout,istderr))
            for filename in changes.get_files() + [ changes.get_filename(), ]:
                destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
                if os.path.exists(destination_file):
                    self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file))
                    os.remove(destination_file)

        # 2) Mark unprocessed files and get rid of them after some time
        pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
        filenames = [name for (name, _) in self.stale_files]
        file_to_check = []
        for file in glob.glob( os.path.join(pub, '*') ):
            if self.files.allowed_upload(file):
                self.log.debug("Incomplete upload: %s" % (file))
                if not file in filenames:
                    self.stale_files.append((file,datetime.datetime.now()))
                else:
                    file_to_check.append(file)
            else:
                if os.path.isfile(file):
                    self.log.warning("Remove unknown file: %s" % (file))
                    os.remove(file)

        new_file_list = []
        for file in file_to_check:
            for (file_known, last_check) in self.stale_files:
                if file == file_known and (datetime.datetime.now() - last_check) > datetime.timedelta(hours = 6):
                    if os.path.isfile(file):
                        self.log.warning("Remove incomplete upload: %s" % (file))
                        os.remove(file)
                        continue
                new_file_list.append((file_known, last_check))

        self.stale_files = new_file_list
Exemplo n.º 12
0
class Importer(object):
    """
    Class to handle the package that is uploaded and wants to be imported into the database.
    """
    def __init__(self, changes, ini, skip_email, skip_gpg):
        """
        Object constructor. Sets class fields to sane values.

        ``self``
            Object pointer.

        ``changes``
            Name `changes` file to import. This is given from the upload controller.

        ``ini``
            Path to debexpo configuration file. This is given from the upload controller.

        ``skip_email``
            If this is set to true, send no email.
        """
        self.changes_file_unqualified = changes
        self.ini_file = os.path.abspath(ini)
        self.actually_send_email = not bool(skip_email)
        self.skip_gpg = skip_gpg

        self.user_id = None
        self.changes = None
        self.user = None

    def send_email(self, email, *args, **kwargs):
        if self.actually_send_email:
            email.send(*args, **kwargs)
        else:
            logging.info("Skipping email send: %s %s", args, kwargs)

    @property
    def changes_file(self):
        incoming_dir = pylons.config['debexpo.upload.incoming']
        return os.path.join(incoming_dir, self.changes_file_unqualified)

    def _clean_path(self, pathToRemove, files):
        """
        remove a path from another using magic split
        ''pathToRemove''
            the path to remove
        ''filePath''
            the a list of original filePath
        """
        result = []
        for filePath in files:
            filePath = string.split(filePath, pathToRemove)
            filePath = filePath[1]
            filePath = string.split(filePath, os.sep)
            if filePath[0] == '':
                filePath.remove('')
            fileName = string.join(filePath, os.sep)
            result.append(fileName)
        return result

    def _get_files(self, path):
        """
        return all files in a path
        ''path''
            the path we are searching
        """
        result = []
        for f in os.listdir(path):
            if os.path.isdir(os.path.join(path, f)):
                result += self._get_files(os.path.join(path, f))
            else:
                result.append(os.path.join(path, f))
        return result

    def _build_source(self, gitpath):
        log.debug(
            'Copying files to a temp directory to run dpkg-source -x on the dsc file'
        )
        self.tempdir = gitpath
        log.debug('Temp dir is: %s', self.tempdir)
        for filename in self.changes.get_files():
            log.debug('Copying: %s', filename)
            shutil.copy(
                os.path.join(pylons.config['debexpo.upload.incoming'],
                             filename), self.tempdir)

        # If the original tarball was pulled from Debian or from the repository, that
        # also needs to be copied into this directory.
        dsc = deb822.Dsc(file(self.changes.get_dsc()))
        for item in dsc['Files']:
            if item['name'] not in self.changes.get_files():
                src_file = os.path.join(
                    pylons.config['debexpo.upload.incoming'], item['name'])
                repository_src_file = os.path.join(
                    pylons.config['debexpo.repository'],
                    self.changes.get_pool_path(), item['name'])
                if os.path.exists(src_file):
                    shutil.copy(src_file, self.tempdir)
                elif os.path.exists(repository_src_file):
                    shutil.copy(repository_src_file, self.tempdir)
                else:
                    log.critical("Trying to copy non-existing file %s" %
                                 (src_file))

        shutil.copy(
            os.path.join(pylons.config['debexpo.upload.incoming'],
                         self.changes_file), self.tempdir)
        self.oldcurdir = os.path.abspath(os.path.curdir)
        cwd = os.getcwd()
        os.chdir(self.tempdir)
        os.system('dpkg-source -x %s extracted' % self.changes.get_dsc())
        os.chdir(cwd)

    def _remove_changes(self):
        """
        Removes the `changes` file.
        """
        if os.path.exists(self.changes_file):
            os.remove(self.changes_file)

    def _remove_temporary_files(self):
        if hasattr(self, 'files_to_remove'):
            for file in self.files_to_remove:
                if os.path.exists(file):
                    os.remove(file)

    def _remove_files(self):
        """
        Removes all the files uploaded.
        """
        if hasattr(self, 'files'):
            for file in self.files:
                if os.path.exists(file):
                    os.remove(file)

        self._remove_changes()
        self._remove_temporary_files()

    def _fail(self, reason, use_log=True):
        """
        Fail the upload by sending a reason for failure to the log and then remove all
        uploaded files.

        A package is `fail`ed if there is a problem with debexpo, **not** if there's
        something wrong with the package.

        ``reason``
            String of why it failed.

        ``use_log``
            Whether to use the log. This should only be False when actually loading the log fails.
            In this case, the reason is printed to stderr.
        """
        if use_log:
            log.critical(reason)
        else:
            print >> sys.stderr, reason

        self._remove_files()

        if self.user is not None:
            email = Email('importer_fail_maintainer')
            package = self.changes.get('Source', '')

            self.send_email(email, [self.user.email], package=package)

        email = Email('importer_fail_admin')
        self.send_email(email, [pylons.config['debexpo.email']],
                        message=reason)

        sys.exit(1)

    def _reject(self, reason):
        """
        Reject the package by sending a reason for failure to the log and then remove all
        uploaded files.

        A package is `reject`ed if there is a problem with the package.

        ``reason``
            String of why it failed.
        """
        log.error('Rejected: %s' % reason)

        self._remove_changes()
        self._remove_files()

        if self.user is not None:
            email = Email('importer_reject_maintainer')
            package = self.changes.get('Source', '')

            self.send_email(email, [self.user.email],
                            package=package,
                            message=reason)
        sys.exit(1)

    def _setup_logging(self):
        """
        Parse the config file and create the ``log`` object for other methods to log their
        actions.
        """
        global log

        # Parse the ini file to validate it
        parser = ConfigParser.ConfigParser()
        parser.read(self.ini_file)

        # Check for the presence of [loggers] in self.ini_file
        if not parser.has_section('loggers'):
            self._fail('Config file does not have [loggers] section',
                       use_log=False)

        logging.config.fileConfig(self.ini_file)

        # Use "name.pid" to avoid importer confusions in the logs
        logger_name = 'debexpo.importer.%s' % os.getpid()
        log = logging.getLogger(logger_name)

    def _setup(self):
        """
        Set up logging, import pylons/paste/debexpo modules, parse config file, create config
        class and chdir to the incoming directory.
        """
        # Look for ini file
        if not os.path.isfile(self.ini_file):
            self._fail('Cannot find ini file')

        self._setup_logging()

        # Import debexpo root directory
        sys.path.append(os.path.dirname(self.ini_file))

        # Initialize Pylons app
        conf = appconfig('config:' + self.ini_file)
        pylons.config = load_environment(conf.global_conf, conf.local_conf)

        # Change into the incoming directory
        incoming_dir = pylons.config['debexpo.upload.incoming']
        logging.info("Changing dir to %s", incoming_dir)
        os.chdir(incoming_dir)

        # Look for the changes file
        if not os.path.isfile(self.changes_file):
            self._fail('Cannot find changes file')

    def _create_db_entries(self, qa):
        """
        Create entries in the Database for the package upload.
        """
        def _package_description(raw):
            return raw[2:].replace('      - ', ' - ')

        log.debug('Creating database entries')

        # Parse component and section from field in changes
        component, section = parse_section(self.changes['files'][0]['section'])

        # Check whether package is already in the database
        package_query = meta.session.query(Package).filter_by(
            name=self.changes['Source'])
        if package_query.count() == 1:
            log.debug('Package %s already exists in the database' %
                      self.changes['Source'])
            package = package_query.one()
            # Update description to make sure it reflects the latest upload
            package.description = _package_description(
                self.changes['Description'])
        else:
            log.debug('Package %s is new to the system' %
                      self.changes['Source'])
            package = Package(name=self.changes['Source'], user=self.user)
            package.description = _package_description(
                self.changes['Description'])
            package.needs_sponsor = 0
            meta.session.add(package)

        # No need to check whether there is the same source name and same version as an existing
        # entry in the database as the upload controller tested whether similar filenames existed
        # in the repository. The only way this would be wrong is if the filename had a different
        # version in than the Version field in changes..

        try:
            closes = self.changes['Closes']
        except KeyError:
            closes = None

        # TODO: fix these magic numbers
        if qa.stop():
            qa_status = 1
        else:
            qa_status = 0

        maintainer_matches = re.compile(r'(.*) <(.*)>').match(
            self.changes['Changed-By'])
        maintainer = maintainer_matches.group(2)

        package_version = PackageVersion(
            package=package,
            version=self.changes['Version'],
            section=section,
            distribution=self.changes['Distribution'],
            qa_status=qa_status,
            component=component,
            priority=self.changes.get_priority(),
            closes=closes,
            uploaded=datetime.now(),
            maintainer=maintainer)
        meta.session.add(package_version)

        source_package = SourcePackage(package_version=package_version)
        meta.session.add(source_package)

        binary_package = None

        # Add PackageFile objects to the database for each uploaded file
        for file in self.files:
            filename = os.path.join(self.changes.get_pool_path(), file)
            # This exception should be never caught.
            # It implies something went wrong before, as we expect a file which does not exist
            try:
                sum = md5sum(
                    os.path.join(pylons.config['debexpo.repository'],
                                 filename))
            except AttributeError as e:
                self._fail("Could not calculate MD5 sum: %s" % (e))

            size = os.stat(
                os.path.join(pylons.config['debexpo.repository'],
                             filename))[ST_SIZE]

            # Check for binary or source package file
            if file.endswith('.deb'):
                # Only create a BinaryPackage if there actually binary package files
                if binary_package is None:
                    binary_package = BinaryPackage(
                        package_version=package_version,
                        arch=file[:-4].split('_')[-1])
                    meta.session.add(binary_package)

                meta.session.add(
                    PackageFile(filename=filename,
                                binary_package=binary_package,
                                size=size,
                                md5sum=sum))
            else:
                meta.session.add(
                    PackageFile(filename=filename,
                                source_package=source_package,
                                size=size,
                                md5sum=sum))

        meta.session.commit()
        log.warning("Finished adding PackageFile objects.")

        # Add PackageInfo objects to the database for the package_version
        for result in qa.result:
            meta.session.add(
                PackageInfo(package_version=package_version,
                            from_plugin=result.from_plugin,
                            outcome=result.outcome,
                            rich_data=result.data,
                            severity=result.severity))

        # Commit all changes to the database
        meta.session.commit()
        log.debug('Committed package data to the database')

        subscribers = meta.session.query(PackageSubscription).filter_by(package=self.changes['Source']).filter(\
            PackageSubscription.level <= constants.SUBSCRIPTION_LEVEL_UPLOADS).all()

        if len(subscribers) > 0:
            email = Email('package_uploaded')
            self.send_email(email, [s.user.email for s in subscribers],
                            package=self.changes['Source'],
                            version=self.changes['Version'],
                            user=self.user)

            log.debug('Sent out package subscription emails')

        # Send success email to uploader
        email = Email('successful_upload')
        dsc_url = pylons.config[
            'debexpo.server'] + '/debian/' + self.changes.get_pool_path(
            ) + '/' + self.changes.get_dsc()
        rfs_url = pylons.config['debexpo.server'] + url(
            'rfs', packagename=self.changes['Source'])
        self.send_email(email, [self.user.email],
                        package=self.changes['Source'],
                        dsc_url=dsc_url,
                        rfs_url=rfs_url)

    def _find_user_by_email_address(self, email_address):
        """
        Searches user by email address
        """
        # XXX: Maybe model is more appropriate place for such a method
        self.user = meta.session.query(User).filter_by(
            email=email_address).filter_by(verification=None).first()
        return self.user

    def _determine_uploader_by_changedby_field(self):
        """
        Create a user object based on the Changed-By entry
        """
        maintainer_string = self.changes.get('Changed-By')
        log.debug("Determining user from 'Changed-By:' field: %s" %
                  maintainer_string)
        maintainer_realname, maintainer_email_address = email.utils.parseaddr(
            maintainer_string)
        log.debug("Changed-By's email address is: %s",
                  maintainer_email_address)
        self._find_user_by_email_address(maintainer_email_address)

    def _determine_uploader_by_gpg(self, gpg_keyid):
        """
        Create a user object based on the gpg output
        """

        try:
            self.user = meta.session.query(User).filter_by(
                gpg_id=gpg_keyid[-8:]).one()
        except exceptions.InvalidRequestError:
            return None
        else:
            log.debug("GPG signature mapped to user %s <%s>" %
                      (self.user.name, self.user.email))

    def main(self):
        """
        Actually start the import of the package.

        Do several environment sanity checks, move files into the right place, and then
        create the database entries for the imported package.
        """
        # Set up importer
        self._setup()

        log.debug('Importer started with arguments: %s' % sys.argv[1:])
        filecheck = CheckFiles()
        gpg = get_gnupg()

        # Try parsing the changes file, but fail if there's an error.
        try:
            self.changes = Changes(filename=self.changes_file)
            filecheck.test_files_present(self.changes)
            filecheck.test_md5sum(self.changes)
        except Exception as e:
            # XXX: The user won't ever see this message. The changes file was
            # invalid, we don't know whom send it to
            self._reject(
                "Your changes file appears invalid. Refusing your upload\n%s" %
                (e.message))

        if not self.skip_gpg:
            # Next, find out whether the changes file was signed with a valid signature, if not reject immediately
            v = gpg.verify_file(path=self.changes_file)
            if v is None:
                self._reject('Your upload does not appear to be signed')

            if not v.is_valid:
                self._reject(
                    'Your upload does not contain a valid signature. Output was:\n%s'
                    % (gpg_out))
            self._determine_uploader_by_gpg(v.key_id)
        else:
            self._determine_uploader_by_changedby_field()

        if self.user is None:
            # Creates a fake user object, but only if no user was found before
            # This is useful to have a user object to send reject mails to
            # generate user object, but only to send out reject message
            self.user = User(id=-1,
                             name=maintainer_realname,
                             email=maintainer_email_address)
            self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)

        self.user_id = self.user.id
        log.debug("User found in database. Has id: %s", self.user.id)

        if self.user.dmup is False:
            log.debug("User has not accepted the DMUP")
            self._reject(
                'You must accept the Debian Machine Usage Policies before uploading a package'
            )

        self.files = self.changes.get_files()
        self.files_to_remove = []

        distribution = self.changes['Distribution'].lower()
        allowed_distributions = ('oldstable', 'stable', 'unstable',
                                 'experimental', 'stable-backports',
                                 'oldstable-backports',
                                 'oldstable-backports-sloppy',
                                 'oldstable-security', 'stable-security',
                                 'testing-security', 'stable-proposed-updates',
                                 'testing-proposed-updates', 'sid', 'wheezy',
                                 'squeeze', 'lenny', 'squeeze-backports',
                                 'lenny-backports', 'lenny-security',
                                 'lenny-backports-sloppy', 'lenny-volatile',
                                 'squeeze-security', 'squeeze-updates',
                                 'wheezy-security', 'unreleased')
        if distribution not in allowed_distributions:
            self._reject(
                "You are not uploading to one of those Debian distributions: %s"
                % (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))

        # Look whether the orig tarball is present, and if not, try and get it from
        # the repository.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
            log.debug(
                "Upload does not contain orig.tar.gz - trying to find it elsewhere"
            )
        if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
            filename = os.path.join(pylons.config['debexpo.repository'],
                                    self.changes.get_pool_path(), orig)
            if os.path.isfile(filename):
                log.debug("Found tar.gz in repository as %s" % (filename))
                shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
                # We need the orig.tar.gz for the import run, plugins need to extract the source package
                # also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
                # remove it later
                self.files_to_remove.append(orig)

        destdir = pylons.config['debexpo.repository']

        # Check whether the files are already present
        log.debug("Checking whether files are already in the repository")
        toinstall = []
        pool_dir = os.path.join(destdir, self.changes.get_pool_path())
        log.debug("Pool directory: %s", pool_dir)
        for file in self.files:
            if os.path.isfile(file) and os.path.isfile(
                    os.path.join(pool_dir, file)):
                log.warning(
                    '%s is being installed even though it already exists' %
                    file)
                toinstall.append(file)
            elif os.path.isfile(file):
                log.debug('File %s is safe to install' %
                          os.path.join(pool_dir, file))
                toinstall.append(file)
                # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
                # by doing nothing here for that case

        # initiate the git storage
        gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
        if os.path.isdir(os.path.join(destdir, 'git', 'last')):
            log.debug("folder exist, cleaning it")
            shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
        else:
            log.debug("last folder doesn't exist, creating one")
        os.makedirs(os.path.join(destdir, 'git', 'last', ''))

        #building sources

        log.debug("building sources!")
        self._build_source(os.path.join(destdir, 'git', 'last', ''))

        #removing older file
        if os.path.isdir(
                os.path.join(destdir, 'git', self.changes['source'],
                             self.changes['source'])):
            for i in glob(
                    os.path.join(destdir, 'git', self.changes['source'],
                                 self.changes['source'], '*')):
                if os.path.isdir(i):
                    shutil.rmtree(i)
                else:
                    os.remove(i)
            os.rmdir(
                os.path.join(destdir, 'git', self.changes['source'],
                             self.changes['source']))
        shutil.copytree(
            os.path.join(destdir, 'git', 'last', 'extracted'),
            os.path.join(destdir, 'git', self.changes['source'],
                         self.changes['source']))
        os.path.join(destdir, 'git', self.changes['source'],
                     self.changes['source'])
        fileToAdd = self._get_files(
            os.path.join(destdir, 'git', self.changes['source'],
                         self.changes['source']))
        fileToAdd = self._clean_path(
            os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
        gs.change(fileToAdd)

        # Run post-upload plugins.
        post_upload = Plugins('post-upload',
                              self.changes,
                              self.changes_file,
                              user_id=self.user_id)
        if post_upload.stop():
            log.critical('post-upload plugins failed')
            self._remove_changes()
            sys.exit(1)

        # Check whether a post-upload plugin has got the orig tarball from somewhere.
        if not orig_file_found and not filecheck.is_native_package(
                self.changes):
            (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                # When coming here it means:
                # a) The uploader did not include a orig.tar.gz in his upload
                # b) We couldn't find a orig.tar.gz in our repository
                # c) No plugin could get the orig.tar.gz
                # ... time to give up
                if orig == None:
                    orig = "any original tarball (orig.tar.gz)"
                self._reject("Rejecting incomplete upload. "
                             "You did not upload %s and we didn't find it on any of our alternative resources.\n"\
                             "If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
                             ( orig ))
            else:
                toinstall.append(orig)

        # Check whether the debexpo.repository variable is set
        if 'debexpo.repository' not in pylons.config:
            self._fail('debexpo.repository not set')

        # Check whether debexpo.repository is a directory
        if not os.path.isdir(pylons.config['debexpo.repository']):
            self._fail('debexpo.repository is not a directory')

        # Check whether debexpo.repository is writeable
        if not os.access(pylons.config['debexpo.repository'], os.W_OK):
            self._fail('debexpo.repository is not writeable')

        qa = Plugins('qa',
                     self.changes,
                     self.changes_file,
                     user_id=self.user_id)
        if qa.stop():
            self._reject('QA plugins failed the package')

        # Loop through parent directories in the target installation directory to make sure they
        # all exist. If not, create them.
        for dir in self.changes.get_pool_path().split('/'):
            destdir = os.path.join(destdir, dir)

            if not os.path.isdir(destdir):
                log.debug('Creating directory: %s' % destdir)
                os.mkdir(destdir)

        # Install files in repository
        for file in toinstall:
            log.debug("Installing new file %s" % (file))
            shutil.move(file, os.path.join(destdir, file))
            #git job is done, cleaning
        shutil.rmtree(
            os.path.join(pylons.config['debexpo.repository'], 'git', 'last',
                         ''))

        self._remove_temporary_files()
        # Create the database rows
        self._create_db_entries(qa)

        # Execute I'm happy to have post-successful-upload plugins
        f = open(self.changes_file)
        changes_contents = f.read()
        f.close()
        Plugins('post-successful-upload',
                self.changes,
                self.changes_file,
                changes_contents=changes_contents)

        # Remove the changes file
        self._remove_changes()

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(pylons.config['debexpo.repository'])
        r.update()

        log.debug('Done')
        return 0
Exemplo n.º 13
0
    def main(self):
        """
        Actually start the import of the package.

        Do several environment sanity checks, move files into the right place, and then
        create the database entries for the imported package.
        """
        # Set up importer
        self._setup()

        log.debug('Importer started with arguments: %s' % sys.argv[1:])
        filecheck = CheckFiles()
        gpg = get_gnupg()

        # Try parsing the changes file, but fail if there's an error.
        try:
            self.changes = Changes(filename=self.changes_file)
            filecheck.test_files_present(self.changes)
            filecheck.test_md5sum(self.changes)
        except Exception as e:
            # XXX: The user won't ever see this message. The changes file was
            # invalid, we don't know whom send it to
            self._reject(
                "Your changes file appears invalid. Refusing your upload\n%s" %
                (e.message))

        if not self.skip_gpg:
            # Next, find out whether the changes file was signed with a valid signature, if not reject immediately
            v = gpg.verify_file(path=self.changes_file)
            if v is None:
                self._reject('Your upload does not appear to be signed')

            if not v.is_valid:
                self._reject(
                    'Your upload does not contain a valid signature. Output was:\n%s'
                    % (gpg_out))
            self._determine_uploader_by_gpg(v.key_id)
        else:
            self._determine_uploader_by_changedby_field()

        if self.user is None:
            # Creates a fake user object, but only if no user was found before
            # This is useful to have a user object to send reject mails to
            # generate user object, but only to send out reject message
            self.user = User(id=-1,
                             name=maintainer_realname,
                             email=maintainer_email_address)
            self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)

        self.user_id = self.user.id
        log.debug("User found in database. Has id: %s", self.user.id)

        if self.user.dmup is False:
            log.debug("User has not accepted the DMUP")
            self._reject(
                'You must accept the Debian Machine Usage Policies before uploading a package'
            )

        self.files = self.changes.get_files()
        self.files_to_remove = []

        distribution = self.changes['Distribution'].lower()
        allowed_distributions = ('oldstable', 'stable', 'unstable',
                                 'experimental', 'stable-backports',
                                 'oldstable-backports',
                                 'oldstable-backports-sloppy',
                                 'oldstable-security', 'stable-security',
                                 'testing-security', 'stable-proposed-updates',
                                 'testing-proposed-updates', 'sid', 'wheezy',
                                 'squeeze', 'lenny', 'squeeze-backports',
                                 'lenny-backports', 'lenny-security',
                                 'lenny-backports-sloppy', 'lenny-volatile',
                                 'squeeze-security', 'squeeze-updates',
                                 'wheezy-security', 'unreleased')
        if distribution not in allowed_distributions:
            self._reject(
                "You are not uploading to one of those Debian distributions: %s"
                % (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))

        # Look whether the orig tarball is present, and if not, try and get it from
        # the repository.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
            log.debug(
                "Upload does not contain orig.tar.gz - trying to find it elsewhere"
            )
        if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
            filename = os.path.join(pylons.config['debexpo.repository'],
                                    self.changes.get_pool_path(), orig)
            if os.path.isfile(filename):
                log.debug("Found tar.gz in repository as %s" % (filename))
                shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
                # We need the orig.tar.gz for the import run, plugins need to extract the source package
                # also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
                # remove it later
                self.files_to_remove.append(orig)

        destdir = pylons.config['debexpo.repository']

        # Check whether the files are already present
        log.debug("Checking whether files are already in the repository")
        toinstall = []
        pool_dir = os.path.join(destdir, self.changes.get_pool_path())
        log.debug("Pool directory: %s", pool_dir)
        for file in self.files:
            if os.path.isfile(file) and os.path.isfile(
                    os.path.join(pool_dir, file)):
                log.warning(
                    '%s is being installed even though it already exists' %
                    file)
                toinstall.append(file)
            elif os.path.isfile(file):
                log.debug('File %s is safe to install' %
                          os.path.join(pool_dir, file))
                toinstall.append(file)
                # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
                # by doing nothing here for that case

        # initiate the git storage
        gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
        if os.path.isdir(os.path.join(destdir, 'git', 'last')):
            log.debug("folder exist, cleaning it")
            shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
        else:
            log.debug("last folder doesn't exist, creating one")
        os.makedirs(os.path.join(destdir, 'git', 'last', ''))

        #building sources

        log.debug("building sources!")
        self._build_source(os.path.join(destdir, 'git', 'last', ''))

        #removing older file
        if os.path.isdir(
                os.path.join(destdir, 'git', self.changes['source'],
                             self.changes['source'])):
            for i in glob(
                    os.path.join(destdir, 'git', self.changes['source'],
                                 self.changes['source'], '*')):
                if os.path.isdir(i):
                    shutil.rmtree(i)
                else:
                    os.remove(i)
            os.rmdir(
                os.path.join(destdir, 'git', self.changes['source'],
                             self.changes['source']))
        shutil.copytree(
            os.path.join(destdir, 'git', 'last', 'extracted'),
            os.path.join(destdir, 'git', self.changes['source'],
                         self.changes['source']))
        os.path.join(destdir, 'git', self.changes['source'],
                     self.changes['source'])
        fileToAdd = self._get_files(
            os.path.join(destdir, 'git', self.changes['source'],
                         self.changes['source']))
        fileToAdd = self._clean_path(
            os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
        gs.change(fileToAdd)

        # Run post-upload plugins.
        post_upload = Plugins('post-upload',
                              self.changes,
                              self.changes_file,
                              user_id=self.user_id)
        if post_upload.stop():
            log.critical('post-upload plugins failed')
            self._remove_changes()
            sys.exit(1)

        # Check whether a post-upload plugin has got the orig tarball from somewhere.
        if not orig_file_found and not filecheck.is_native_package(
                self.changes):
            (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                # When coming here it means:
                # a) The uploader did not include a orig.tar.gz in his upload
                # b) We couldn't find a orig.tar.gz in our repository
                # c) No plugin could get the orig.tar.gz
                # ... time to give up
                if orig == None:
                    orig = "any original tarball (orig.tar.gz)"
                self._reject("Rejecting incomplete upload. "
                             "You did not upload %s and we didn't find it on any of our alternative resources.\n"\
                             "If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
                             ( orig ))
            else:
                toinstall.append(orig)

        # Check whether the debexpo.repository variable is set
        if 'debexpo.repository' not in pylons.config:
            self._fail('debexpo.repository not set')

        # Check whether debexpo.repository is a directory
        if not os.path.isdir(pylons.config['debexpo.repository']):
            self._fail('debexpo.repository is not a directory')

        # Check whether debexpo.repository is writeable
        if not os.access(pylons.config['debexpo.repository'], os.W_OK):
            self._fail('debexpo.repository is not writeable')

        qa = Plugins('qa',
                     self.changes,
                     self.changes_file,
                     user_id=self.user_id)
        if qa.stop():
            self._reject('QA plugins failed the package')

        # Loop through parent directories in the target installation directory to make sure they
        # all exist. If not, create them.
        for dir in self.changes.get_pool_path().split('/'):
            destdir = os.path.join(destdir, dir)

            if not os.path.isdir(destdir):
                log.debug('Creating directory: %s' % destdir)
                os.mkdir(destdir)

        # Install files in repository
        for file in toinstall:
            log.debug("Installing new file %s" % (file))
            shutil.move(file, os.path.join(destdir, file))
            #git job is done, cleaning
        shutil.rmtree(
            os.path.join(pylons.config['debexpo.repository'], 'git', 'last',
                         ''))

        self._remove_temporary_files()
        # Create the database rows
        self._create_db_entries(qa)

        # Execute I'm happy to have post-successful-upload plugins
        f = open(self.changes_file)
        changes_contents = f.read()
        f.close()
        Plugins('post-successful-upload',
                self.changes,
                self.changes_file,
                changes_contents=changes_contents)

        # Remove the changes file
        self._remove_changes()

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(pylons.config['debexpo.repository'])
        r.update()

        log.debug('Done')
        return 0