예제 #1
0
    def main(self):
        """
        Actually start the import of the package.

        Do several environment sanity checks, move files into the right place, and then
        create the database entries for the imported package.
        """
        # Set up importer
        self._setup()

        log.debug('Importer started with arguments: %s' % sys.argv[1:])
        filecheck = CheckFiles()
        gpg = get_gnupg()

        # Try parsing the changes file, but fail if there's an error.
        try:
            self.changes = Changes(filename=self.changes_file)
            filecheck.test_files_present(self.changes)
            filecheck.test_md5sum(self.changes)
        except Exception as e:
            # XXX: The user won't ever see this message. The changes file was
            # invalid, we don't know whom send it to
            self._reject("Your changes file appears invalid. Refusing your upload\n%s" % (e.message))

        if not self.skip_gpg:
            # Next, find out whether the changes file was signed with a valid signature, if not reject immediately
            v = gpg.verify_file(path=self.changes_file)
            if v is None:
                self._reject('Your upload does not appear to be signed')

            if not v.is_valid:
                self._reject('Your upload does not contain a valid signature. Output was:\n%s' % (gpg_out))
            self._determine_uploader_by_gpg(v.key_id)
        else:
            self._determine_uploader_by_changedby_field()

        if self.user is None:
            # Creates a fake user object, but only if no user was found before
            # This is useful to have a user object to send reject mails to
            # generate user object, but only to send out reject message
            self.user = User(id=-1, name=maintainer_realname, email=maintainer_email_address)
            self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)

        self.user_id = self.user.id
        log.debug("User found in database. Has id: %s", self.user.id)

        if self.user.dmup is False:
            log.debug("User has not accepted the DMUP")
            self._reject('You must accept the Debian Machine Usage Policies before uploading a package')

        self.files = self.changes.get_files()
        self.files_to_remove = []

        distribution = self.changes['Distribution'].lower()
        allowed_distributions = (
            'oldstable', 'stable', 'unstable', 'experimental', 'stable-backports', 'oldstable-backports',
            'oldstable-backports-sloppy', 'oldstable-security', 'stable-security', 'testing-security',
            'stable-proposed-updates',
            'testing-proposed-updates', 'sid', 'wheezy', 'squeeze', 'lenny', 'squeeze-backports', 'lenny-backports',
            'lenny-security', 'lenny-backports-sloppy', 'lenny-volatile', 'squeeze-security', 'squeeze-updates',
            'wheezy-security',
            'unreleased')
        if distribution not in allowed_distributions:
            self._reject("You are not uploading to one of those Debian distributions: %s" %
                         (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))

        # Look whether the orig tarball is present, and if not, try and get it from
        # the repository.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
            log.debug("Upload does not contain orig.tar.gz - trying to find it elsewhere")
        if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
            filename = os.path.join(pylons.config['debexpo.repository'],
                self.changes.get_pool_path(), orig)
            if os.path.isfile(filename):
                log.debug("Found tar.gz in repository as %s" % (filename))
                shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
                # We need the orig.tar.gz for the import run, plugins need to extract the source package
                # also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
                # remove it later
                self.files_to_remove.append(orig)

        destdir = pylons.config['debexpo.repository']


        # Check whether the files are already present
        log.debug("Checking whether files are already in the repository")
        toinstall = []
        pool_dir = os.path.join(destdir, self.changes.get_pool_path())
        log.debug("Pool directory: %s", pool_dir)
        for file in self.files:
            if os.path.isfile(file) and os.path.isfile(os.path.join(pool_dir, file)):
                log.warning('%s is being installed even though it already exists' % file)
                toinstall.append(file)
            elif os.path.isfile(file):
                log.debug('File %s is safe to install' % os.path.join(pool_dir, file))
                toinstall.append(file)
                # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
                # by doing nothing here for that case



        # initiate the git storage
        gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
        if os.path.isdir(os.path.join(destdir, 'git', 'last')):
            log.debug("folder exist, cleaning it")
            shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
        else:
            log.debug("last folder doesn't exist, creating one")
        os.makedirs(os.path.join(destdir, 'git', 'last', ''))

        #building sources

        log.debug("building sources!")
        self._build_source(os.path.join(destdir, 'git', 'last', ''))

        #removing older file
        if os.path.isdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])):
            for i in glob(os.path.join(destdir, 'git', self.changes['source'], self.changes['source'], '*')):
                if os.path.isdir(i):
                    shutil.rmtree(i)
                else:
                    os.remove(i)
            os.rmdir(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        shutil.copytree(os.path.join(destdir, 'git', 'last', 'extracted'),
            os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        os.path.join(destdir, 'git', self.changes['source'], self.changes['source'])
        fileToAdd = self._get_files(os.path.join(destdir, 'git', self.changes['source'], self.changes['source']))
        fileToAdd = self._clean_path(os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
        gs.change(fileToAdd)



        # Run post-upload plugins.
        post_upload = Plugins('post-upload', self.changes, self.changes_file,
            user_id=self.user_id)
        if post_upload.stop():
            log.critical('post-upload plugins failed')
            self._remove_changes()
            sys.exit(1)

        # Check whether a post-upload plugin has got the orig tarball from somewhere.
        if not orig_file_found and not filecheck.is_native_package(self.changes):
            (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                # When coming here it means:
                # a) The uploader did not include a orig.tar.gz in his upload
                # b) We couldn't find a orig.tar.gz in our repository
                # c) No plugin could get the orig.tar.gz
                # ... time to give up
                if orig == None:
                    orig = "any original tarball (orig.tar.gz)"
                self._reject("Rejecting incomplete upload. "
                             "You did not upload %s and we didn't find it on any of our alternative resources.\n"\
                             "If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
                             ( orig ))
            else:
                toinstall.append(orig)

        # Check whether the debexpo.repository variable is set
        if 'debexpo.repository' not in pylons.config:
            self._fail('debexpo.repository not set')

        # Check whether debexpo.repository is a directory
        if not os.path.isdir(pylons.config['debexpo.repository']):
            self._fail('debexpo.repository is not a directory')

        # Check whether debexpo.repository is writeable
        if not os.access(pylons.config['debexpo.repository'], os.W_OK):
            self._fail('debexpo.repository is not writeable')

        qa = Plugins('qa', self.changes, self.changes_file, user_id=self.user_id)
        if qa.stop():
            self._reject('QA plugins failed the package')

        # Loop through parent directories in the target installation directory to make sure they
        # all exist. If not, create them.
        for dir in self.changes.get_pool_path().split('/'):
            destdir = os.path.join(destdir, dir)

            if not os.path.isdir(destdir):
                log.debug('Creating directory: %s' % destdir)
                os.mkdir(destdir)

        # Install files in repository
        for file in toinstall:
            log.debug("Installing new file %s" % (file))
            shutil.move(file, os.path.join(destdir, file))
            #git job is done, cleaning
        shutil.rmtree(os.path.join(pylons.config['debexpo.repository'], 'git', 'last', ''))

        self._remove_temporary_files()
        # Create the database rows
        self._create_db_entries(qa)

        # Execute I'm happy to have post-successful-upload plugins
        f = open(self.changes_file)
        changes_contents = f.read()
        f.close()
        Plugins('post-successful-upload', self.changes, self.changes_file,
            changes_contents=changes_contents)

        # Remove the changes file
        self._remove_changes()

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(pylons.config['debexpo.repository'])
        r.update()

        log.debug('Done')
        return 0
예제 #2
0
    def test_orig_tarball(self):
        """
        Check whether there is an original tarball referenced by the dsc file, but not
        actually in the package upload.

        This procedure is skipped to avoid denial of service attacks when a package is
        larger than the configured size
        """

        # Set download of files, when the expected file size of the orig.tar.gz is larger
        # than the configured threshold.
        # XXX TODO: This size should be the 75% quartile, that is the value where 75% of all
        #           packages are smaller than that. For now, blindly assume this as 15M
        size = 15728640
        log.debug('Checking whether an orig tarball mentioned in the dsc is missing')
        dsc = deb822.Dsc(file(self.changes.get_dsc()))
        filecheck = CheckFiles()

        if filecheck.is_native_package(self.changes):
            log.debug('No orig.tar.gz file found; native package?')
            return

        # An orig.tar.gz was found in the dsc, and also in the upload.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
            log.debug('%s found successfully', orig)
            return

        if not orig:
            log.debug("Couldn't determine name of the orig.tar.gz?")
            return

        for dscfile in dsc['Files']:
            dscfile['size'] = int(dscfile['size'])
            if orig == dscfile['name']:
                if dscfile['size'] > size:
                        log.warning("Skipping eventual download of orig.tar.gz %s: size %d > %d" % (dscfile['name'], dscfile['size'], size))
                        return
                orig = dscfile
                break
        else:
            log.debug("dsc does not reference our expected orig.tar.gz name '%s'" % (orig))
            return

        log.debug('Could not find %s; looking in Debian for it', orig['name'])

        url = os.path.join(pylons.config['debexpo.debian_mirror'], self.changes.get_pool_path(), orig['name'])
        log.debug('Trying to fetch %s' % url)
        out = urllib.urlopen(url)
        contents = out.read()

        f = open(orig['name'], "wb")
        f.write(contents)
        f.close()

        if md5sum(orig['name']) == orig['md5sum']:
            log.debug('Tarball %s taken from Debian' % orig['name'])
            self.info('tarball-taken-from-debian', None)
        else:
            log.error('Tarball %s not found in Debian' % orig['name'])
            os.unlink(orig['name'])
예제 #3
0
    def test_orig_tarball(self):
        """
        Check whether there is an original tarball referenced by the dsc file, but not
        actually in the package upload.

        This procedure is skipped to avoid denial of service attacks when a package is
        larger than the configured size
        """

        # Set download of files, when the expected file size of the orig.tar.gz is larger
        # than the configured threshold.
        # XXX TODO: This size should be the 75% quartile, that is the value where 75% of all
        #           packages are smaller than that. For now, blindly assume this as 15M
        size = 15728640
        log.debug(
            'Checking whether an orig tarball mentioned in the dsc is missing')
        dsc = deb822.Dsc(file(self.changes.get_dsc()))
        filecheck = CheckFiles()

        if filecheck.is_native_package(self.changes):
            log.debug('No orig.tar.gz file found; native package?')
            return

        # An orig.tar.gz was found in the dsc, and also in the upload.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found > constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
            log.debug('%s found successfully', orig)
            return

        if not orig:
            log.debug("Couldn't determine name of the orig.tar.gz?")
            return

        for dscfile in dsc['Files']:
            dscfile['size'] = int(dscfile['size'])
            if orig == dscfile['name']:
                if dscfile['size'] > size:
                    log.warning(
                        "Skipping eventual download of orig.tar.gz %s: size %d > %d"
                        % (dscfile['name'], dscfile['size'], size))
                    return
                orig = dscfile
                break
        else:
            log.debug(
                "dsc does not reference our expected orig.tar.gz name '%s'" %
                (orig))
            return

        log.debug('Could not find %s; looking in Debian for it', orig['name'])

        url = os.path.join(pylons.config['debexpo.debian_mirror'],
                           self.changes.get_pool_path(), orig['name'])
        log.debug('Trying to fetch %s' % url)
        out = urllib.urlopen(url)
        contents = out.read()

        f = open(orig['name'], "wb")
        f.write(contents)
        f.close()

        if md5sum(orig['name']) == orig['md5sum']:
            log.debug('Tarball %s taken from Debian' % orig['name'])
            self.info('tarball-taken-from-debian', None)
        else:
            log.error('Tarball %s not found in Debian' % orig['name'])
            os.unlink(orig['name'])
예제 #4
0
    def main(self):
        """
        Actually start the import of the package.

        Do several environment sanity checks, move files into the right place, and then
        create the database entries for the imported package.
        """
        # Set up importer
        self._setup()

        log.debug('Importer started with arguments: %s' % sys.argv[1:])
        filecheck = CheckFiles()
        gpg = get_gnupg()

        # Try parsing the changes file, but fail if there's an error.
        try:
            self.changes = Changes(filename=self.changes_file)
            filecheck.test_files_present(self.changes)
            filecheck.test_md5sum(self.changes)
        except Exception as e:
            # XXX: The user won't ever see this message. The changes file was
            # invalid, we don't know whom send it to
            self._reject(
                "Your changes file appears invalid. Refusing your upload\n%s" %
                (e.message))

        if not self.skip_gpg:
            # Next, find out whether the changes file was signed with a valid signature, if not reject immediately
            v = gpg.verify_file(path=self.changes_file)
            if v is None:
                self._reject('Your upload does not appear to be signed')

            if not v.is_valid:
                self._reject(
                    'Your upload does not contain a valid signature. Output was:\n%s'
                    % (gpg_out))
            self._determine_uploader_by_gpg(v.key_id)
        else:
            self._determine_uploader_by_changedby_field()

        if self.user is None:
            # Creates a fake user object, but only if no user was found before
            # This is useful to have a user object to send reject mails to
            # generate user object, but only to send out reject message
            self.user = User(id=-1,
                             name=maintainer_realname,
                             email=maintainer_email_address)
            self._reject('Couldn\'t find user %s. Exiting.' % self.user.email)

        self.user_id = self.user.id
        log.debug("User found in database. Has id: %s", self.user.id)

        if self.user.dmup is False:
            log.debug("User has not accepted the DMUP")
            self._reject(
                'You must accept the Debian Machine Usage Policies before uploading a package'
            )

        self.files = self.changes.get_files()
        self.files_to_remove = []

        distribution = self.changes['Distribution'].lower()
        allowed_distributions = ('oldstable', 'stable', 'unstable',
                                 'experimental', 'stable-backports',
                                 'oldstable-backports',
                                 'oldstable-backports-sloppy',
                                 'oldstable-security', 'stable-security',
                                 'testing-security', 'stable-proposed-updates',
                                 'testing-proposed-updates', 'sid', 'wheezy',
                                 'squeeze', 'lenny', 'squeeze-backports',
                                 'lenny-backports', 'lenny-security',
                                 'lenny-backports-sloppy', 'lenny-volatile',
                                 'squeeze-security', 'squeeze-updates',
                                 'wheezy-security', 'unreleased')
        if distribution not in allowed_distributions:
            self._reject(
                "You are not uploading to one of those Debian distributions: %s"
                % (reduce(lambda x, xs: x + " " + xs, allowed_distributions)))

        # Look whether the orig tarball is present, and if not, try and get it from
        # the repository.
        (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
        if orig_file_found != constants.ORIG_TARBALL_LOCATION_LOCAL:
            log.debug(
                "Upload does not contain orig.tar.gz - trying to find it elsewhere"
            )
        if orig and orig_file_found == constants.ORIG_TARBALL_LOCATION_REPOSITORY:
            filename = os.path.join(pylons.config['debexpo.repository'],
                                    self.changes.get_pool_path(), orig)
            if os.path.isfile(filename):
                log.debug("Found tar.gz in repository as %s" % (filename))
                shutil.copy(filename, pylons.config['debexpo.upload.incoming'])
                # We need the orig.tar.gz for the import run, plugins need to extract the source package
                # also Lintian needs it. However the orig.tar.gz is in the repository already, so we can
                # remove it later
                self.files_to_remove.append(orig)

        destdir = pylons.config['debexpo.repository']

        # Check whether the files are already present
        log.debug("Checking whether files are already in the repository")
        toinstall = []
        pool_dir = os.path.join(destdir, self.changes.get_pool_path())
        log.debug("Pool directory: %s", pool_dir)
        for file in self.files:
            if os.path.isfile(file) and os.path.isfile(
                    os.path.join(pool_dir, file)):
                log.warning(
                    '%s is being installed even though it already exists' %
                    file)
                toinstall.append(file)
            elif os.path.isfile(file):
                log.debug('File %s is safe to install' %
                          os.path.join(pool_dir, file))
                toinstall.append(file)
                # skip another corner case, where the dsc contains a orig.tar.gz but wasn't uploaded
                # by doing nothing here for that case

        # initiate the git storage
        gs = GitStorage(os.path.join(destdir, "git", self.changes['Source']))
        if os.path.isdir(os.path.join(destdir, 'git', 'last')):
            log.debug("folder exist, cleaning it")
            shutil.rmtree(os.path.join(destdir, "git", 'last', ''), True)
        else:
            log.debug("last folder doesn't exist, creating one")
        os.makedirs(os.path.join(destdir, 'git', 'last', ''))

        #building sources

        log.debug("building sources!")
        self._build_source(os.path.join(destdir, 'git', 'last', ''))

        #removing older file
        if os.path.isdir(
                os.path.join(destdir, 'git', self.changes['source'],
                             self.changes['source'])):
            for i in glob(
                    os.path.join(destdir, 'git', self.changes['source'],
                                 self.changes['source'], '*')):
                if os.path.isdir(i):
                    shutil.rmtree(i)
                else:
                    os.remove(i)
            os.rmdir(
                os.path.join(destdir, 'git', self.changes['source'],
                             self.changes['source']))
        shutil.copytree(
            os.path.join(destdir, 'git', 'last', 'extracted'),
            os.path.join(destdir, 'git', self.changes['source'],
                         self.changes['source']))
        os.path.join(destdir, 'git', self.changes['source'],
                     self.changes['source'])
        fileToAdd = self._get_files(
            os.path.join(destdir, 'git', self.changes['source'],
                         self.changes['source']))
        fileToAdd = self._clean_path(
            os.path.join(destdir, 'git', self.changes['source']), fileToAdd)
        gs.change(fileToAdd)

        # Run post-upload plugins.
        post_upload = Plugins('post-upload',
                              self.changes,
                              self.changes_file,
                              user_id=self.user_id)
        if post_upload.stop():
            log.critical('post-upload plugins failed')
            self._remove_changes()
            sys.exit(1)

        # Check whether a post-upload plugin has got the orig tarball from somewhere.
        if not orig_file_found and not filecheck.is_native_package(
                self.changes):
            (orig, orig_file_found) = filecheck.find_orig_tarball(self.changes)
            if orig_file_found == constants.ORIG_TARBALL_LOCATION_NOT_FOUND:
                # When coming here it means:
                # a) The uploader did not include a orig.tar.gz in his upload
                # b) We couldn't find a orig.tar.gz in our repository
                # c) No plugin could get the orig.tar.gz
                # ... time to give up
                if orig == None:
                    orig = "any original tarball (orig.tar.gz)"
                self._reject("Rejecting incomplete upload. "
                             "You did not upload %s and we didn't find it on any of our alternative resources.\n"\
                             "If you tried to upload a package which only increased the Debian revision part, make sure you include the full source (pass -sa to dpkg-buildpackage)" %
                             ( orig ))
            else:
                toinstall.append(orig)

        # Check whether the debexpo.repository variable is set
        if 'debexpo.repository' not in pylons.config:
            self._fail('debexpo.repository not set')

        # Check whether debexpo.repository is a directory
        if not os.path.isdir(pylons.config['debexpo.repository']):
            self._fail('debexpo.repository is not a directory')

        # Check whether debexpo.repository is writeable
        if not os.access(pylons.config['debexpo.repository'], os.W_OK):
            self._fail('debexpo.repository is not writeable')

        qa = Plugins('qa',
                     self.changes,
                     self.changes_file,
                     user_id=self.user_id)
        if qa.stop():
            self._reject('QA plugins failed the package')

        # Loop through parent directories in the target installation directory to make sure they
        # all exist. If not, create them.
        for dir in self.changes.get_pool_path().split('/'):
            destdir = os.path.join(destdir, dir)

            if not os.path.isdir(destdir):
                log.debug('Creating directory: %s' % destdir)
                os.mkdir(destdir)

        # Install files in repository
        for file in toinstall:
            log.debug("Installing new file %s" % (file))
            shutil.move(file, os.path.join(destdir, file))
            #git job is done, cleaning
        shutil.rmtree(
            os.path.join(pylons.config['debexpo.repository'], 'git', 'last',
                         ''))

        self._remove_temporary_files()
        # Create the database rows
        self._create_db_entries(qa)

        # Execute I'm happy to have post-successful-upload plugins
        f = open(self.changes_file)
        changes_contents = f.read()
        f.close()
        Plugins('post-successful-upload',
                self.changes,
                self.changes_file,
                changes_contents=changes_contents)

        # Remove the changes file
        self._remove_changes()

        # Refresh the Sources/Packages files.
        log.debug('Updating Sources and Packages files')
        r = Repository(pylons.config['debexpo.repository'])
        r.update()

        log.debug('Done')
        return 0