Exemple #1
0
def check_debs_in_upload(changes, profile, interface):
    """
    The ``check-debs`` checker is a stock dput checker that checks packages
    intended for upload for .deb packages.

    Profile key: ``foo``

    Example profile::

        {
            "skip": false,
            "enforce": "debs"
        }

    ``skip``    controls if the checker should drop out without checking
                for anything at all.

    ``enforce`` This controls what we check for. Valid values are
                "debs" or "source". Nonsense values will cause
                an abort.
    """
    debs = {}
    if 'check-debs' in profile:
        debs = profile['check-debs']

    if 'skip' in debs and debs['skip']:
        logger.debug("Skipping deb checker.")
        return

    enforce_debs = True
    if 'enforce' in debs:
        model = debs['enforce']
        if model == 'debs':
            enforce_debs = True
        elif model == 'source':
            enforce_debs = False
        else:
            logger.warning("Garbage value for check-debs/enforce - is %s,"
                           " valid values are `debs` and `source`. Skipping"
                           " checks." % (model))
            return
    else:
        logger.warning("No `enforce` key in check-debs. Skipping checks.")
        return

    has_debs = False
    for fil in changes.get_files():
        xtns = ['.deb', '.udeb']
        for xtn in xtns:
            if fil.endswith(xtn):
                has_debs = True

    if enforce_debs and not has_debs:
        raise BinaryUploadError(
            "There are no .debs in this upload, and we're enforcing them."
        )
    if not enforce_debs and has_debs:
        raise BinaryUploadError(
            "There are .debs in this upload, and enforcing they don't exist."
        )
Exemple #2
0
    def _auth(self, fqdn, ssh_kwargs, _first=0):
        if _first == 3:
            raise SftpUploadException("Failed to authenticate")
        try:
            self._sshclient.connect(fqdn, **ssh_kwargs)
            logger.debug("Logged in!")
        except socket.error as e:
            raise SftpUploadException("SFTP error uploading to %s: %s" % (
                fqdn,
                repr(e)
            ))
        except paramiko.AuthenticationException:
            logger.warning("Failed to auth. Prompting for a login pair.")
            # XXX: Ask for pw only
            user = self.interface.question('please login', 'Username')
            # 4 first error
            pw = self.interface.password(None, "Password")

            if user is not None:
                ssh_kwargs['username'] = user
            ssh_kwargs['password'] = pw
            self._auth(fqdn, ssh_kwargs, _first=_first + 1)
        except paramiko.SSHException as e:
            raise SftpUploadException("SFTP error uploading to %s: %s" % (
                fqdn,
                repr(e)
            ))
Exemple #3
0
def check_protected_distributions(changes, profile, interface):
    """
    The ``protected distributions`` checker is a stock dput checker that makes
    sure, users intending an upload for a special care archive (
    testing-proposed-updates, stable-security, etc.) did really follow the
    archive policies for that.

    Profile key: none

    """
    # XXX: This check does not contain code names yet. We need a global way
    #      to retrieve and share current code names.
    suite = changes["Distribution"]
    query_user = False
    release_team_suites = ["testing-proposed-updates", "proposed-updates", "stable", "testing"]
    if suite in release_team_suites:
        msg = "Are you sure to upload to %s? Did you coordinate with the " "Release Team before your upload?" % (suite)
        error_msg = "Aborting upload to Release Team managed suite upon " "request"
        query_user = True
    security_team_suites = ["stable-security", "oldstable-security", "testing-security"]
    if suite in security_team_suites:
        msg = "Are you sure to upload to %s? Did you coordinate with the " "Security Team before your upload?" % (suite)
        error_msg = "Aborting upload to Security Team managed suite upon " "request"
        query_user = True

    if query_user:
        logger.trace("Querying the user for input. The upload targets a " "protected distribution")
        if not interface.boolean("Protected Checker", msg, default=BUTTON_NO):
            raise BadDistributionError(error_msg)
        else:
            logger.warning("Uploading with explicit confirmation by the user")
    else:
        logger.trace("Nothing to do for checker protected_distributions")
Exemple #4
0
    def preload(self, configs):
        """
        See :meth:`dput.config.AbstractConfig.preload`
        """
        parser = configparser.ConfigParser()
        if configs is None:
            configs = dput.core.DPUT_CONFIG_LOCATIONS

        for config in configs:
            if not os.access(config, os.R_OK):
                logger.debug("Skipping file %s: Not accessible" % (
                    config
                ))
                continue
            try:
                logger.trace("Parsing %s" % (config))
                parser.readfp(open(config, 'r'))
            except IOError as e:
                logger.warning("Skipping file %s: %s" % (
                    config,
                    e
                ))
                continue
            except configparser.ParsingError as e:
                raise DputConfigurationError("Error parsing file %s: %s" % (
                    config,
                    e
                ))
        self.parser = parser
        self.configs = configs
        self.defaults = self._translate_strs(self.get_config("DEFAULT"))
        self.parser.remove_section("DEFAULT")
Exemple #5
0
    def _run_hook(self, hook):
        if hook in self._config and self._config[hook] != "":
            cmd = self._config[hook]
            (output, stderr, ret) = run_command(cmd)
            if ret == -1:
                if not os.path.exists(cmd):
                    logger.warning(
                        "Error: You've set a hook (%s) to run (`%s`), "
                        "but it can't be found (and doesn't appear to exist)."
                        " Please verify the path and correct it." % (
                            hook,
                            self._config[hook]
                        )
                    )
                    return

            sys.stdout.write(output)  # XXX: Fixme
            if ret != 0:
                raise DputError(
                    "Command `%s' returned an error: %s [err=%d]" % (
                        self._config[hook],
                        stderr,
                        ret
                    )
                )
Exemple #6
0
    def initialize(self, **kwargs):
        """
        See :meth:`dput.uploader.AbstractUploader.initialize`
        """
        fqdn = self._config['fqdn']
        incoming = self._config['incoming']

        ssh_options = []
        if "ssh_options" in self._config:
            ssh_options.extend(self._config['ssh_options'])
        if 'port' in self._config:
            ssh_options.append("-oPort=%d" % self._config['port'])
        username = None
        if 'login' in self._config and self._config['login'] != "*":
            username = self._config['login']

        if incoming.startswith('~/'):
            logger.warning("SFTP does not support ~/path, continuing with"
                           "relative directory name instead.")
            incoming = incoming[2:]

        if username:
            logger.info("Logging into host %s as %s" % (fqdn, username))
        else:
            logger.info("Logging into host %s" % fqdn)
        self._sftp = Sftp(servername=fqdn, username=username,
                               ssh_options=ssh_options)
        self.incoming = incoming
Exemple #7
0
def lintian(changes, profile, interface):
    """
    The ``lintian`` checker is a stock dput checker that checks packages
    intended for upload for common mistakes, using the static checking
    tool, `lintian <http://lintian.debian.org/>`.

    Profile key: ``lintian``

    Example profile::

        {
            "run_lintian": true
            "lintian": {
            }
        }

    No keys are current supported, but there are plans to set custom
    ignore lists, etc.
    """
    if "run_lintian" in profile:
        logger.warning("Setting 'run_lintian' is deprecated. "
                       "Please configure the lintian checker instead.")
        if not profile['run_lintian']:  # XXX: Broken. Fixme.
            logger.info("skipping lintian checking, enable with "
                        "run_lintian = 1 in your dput.cf")
            return

    tags = lint(
        changes._absfile,
        pedantic=True,
        info=True,
        experimental=True
    )

    sorted_tags = {}

    for tag in tags:
        if not tag['severity'] in sorted_tags:
            sorted_tags[tag['severity']] = {}
        if tag['tag'] not in sorted_tags[tag['severity']]:
            sorted_tags[tag['severity']][tag['tag']] = tag
    tags = sorted_tags

    # XXX: Make this configurable
    if not "E" in tags:
        return
    for tag in set(tags["E"]):
        print "  - %s: %s" % (tags["E"][tag]['severity'], tag)

    inp = interface.boolean('Lintian Checker',
                            'Upload despite of Lintian finding issues?',
                            default=BUTTON_NO)
    if not inp:
        raise LintianHookException(
            "User didn't own up to the "
            "Lintian issues"
        )
    else:
        logger.warning("Uploading with outstanding Lintian issues.")
Exemple #8
0
def run_pre_hooks(changes, profile):
    for name, hook in get_hooks(profile):
        if 'pre' in hook and hook['pre']:
            run_hook(name, hook, changes, profile)
        if 'pre' not in hook and 'post' not in hook:
            logger.warning("Hook: %s has no pre/post ordering. Assuming "
                           "pre.")
            run_hook(name, hook, changes, profile)
Exemple #9
0
 def initialize(self, **kwargs):
     """
     See :meth:`dput.uploader.AbstractUploader.initialize`
     """
     login = find_username(self._config)
     self._scp_base = ["scp", "-p", "-C"]
     # XXX: Timeout?
     if 'port' in self._config:
         self._scp_base += ("-P", "%s" % self._config['port'])
     self._scp_host = "%s@%s" % (login, self._config['fqdn'])
     logger.debug("Using scp to upload to %s" % (self._scp_host))
     logger.warning("SCP is deprecated. Please consider upgrading to SFTP.")
Exemple #10
0
    def upload_write_error(self, e):
        """
        .. warning::
           don't call this.

        please don't call this
        """
        # XXX: Refactor this, please god, refactor this.
        logger.warning("""Upload permissions error

You either don't have the rights to upload a file, or, if this is on
ftp-master, you may have tried to overwrite a file already on the server.

Continuing anyway in case you want to recover from an incomplete upload.
No file was uploaded, however.""")
Exemple #11
0
def lint(path, pedantic=False, info=False, experimental=False):
    args = ["lintian", "--show-overrides"]

    if pedantic:
        args.append("--pedantic")
    if info:
        args.append("-I")
    if experimental:
        args.append("-E")

    args.append(path)

    try:
        output = subprocess.check_output(args)
    except subprocess.CalledProcessError as e:
        output = e.output
    except OSError as e:
        logger.warning("Failed to execute lintian: %s" % (e))
        raise DputConfigurationError("lintian: %s" % (e))

    return process(output)
Exemple #12
0
def get_obj(cls, checker_method):  # checker_method is a bad name.
    """
    Get an object by plugin def (``checker_method``) in class ``cls`` (such
    as ``hooks``).
    """
    logger.trace("Attempting to resolve %s %s" % (cls, checker_method))
    try:
        config = load_config(cls, checker_method)
        validate_object('plugin', config, "%s/%s" % (cls, checker_method))

        if config is None or config == {}:
            raise NoSuchConfigError("No such config")
    except NoSuchConfigError:
        logger.debug("failed to resolve config %s" % (checker_method))
        return None
    path = config['path']
    logger.trace("loading %s %s" % (cls, path))
    try:
        return load_obj(path)
    except ImportError as e:
        logger.warning("failed to resolve path %s: %s" % (path, e))
        return None
Exemple #13
0
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

from dput.core import logger
from dput.exceptions import HookException

try:
    from distro_info import (DebianDistroInfo, UbuntuDistroInfo,
                             DistroDataOutdated)
except ImportError:
    logger.warning('Uploading to Ubuntu requires python-distro-info to be '
                   'installed')
    raise


class UnknownDistribution(HookException):
    """
    Subclass of the :class:`dput.exceptions.HookException`.

    Thrown if the ``supported-distribution`` checker encounters an issue.
    """
    pass


class UnsupportedDistribution(HookException):
    """
    Subclass of the :class:`dput.exceptions.HookException`.
Exemple #14
0
    def validate(self, args):
        if args.force:
            return

        if not os.path.exists(DM_KEYRING):
            raise DmCommandError(
                "To manage DM permissions, the `debian-keyring' "
                "keyring package must be installed. "
                "File %s does not exist" % (DM_KEYRING)
            )
            return

        # I HATE embedded functions. But OTOH this function is not usable
        # somewhere else, so...
        def pretty_print_list(tuples):
            fingerprints = ""
            for entry in tuples:
                fingerprints += "\n- %s (%s)" % entry
            return fingerprints

        # TODO: Validate input. Packages must exist (i.e. be not NEW)
        (out, err, exit_status) = run_command([
            "gpg", "--no-options",
            "--no-auto-check-trustdb", "--no-default-keyring",
            "--list-key", "--with-colons", "--fingerprint",
            "--keyring", DM_KEYRING, args.dm
        ])
        if exit_status != 0:
            logger.warning("")
            logger.warning("There was an error looking up the DM's key")
            logger.warning("")
            logger.warning(" dput-ng uses the DM keyring in /usr/share/keyrings/")
            logger.warning(" as the keyring to pull full fingerprints from.")
            logger.warning("")
            logger.warning(" Please ensure your keyring is up to date:")
            logger.warning("")
            logger.warning("   sudo apt-get install debian-keyring")
            logger.warning("")
            logger.warning(" Or, if you can not get the keyring, you may use their")
            logger.warning(" full fingerprint (without spaces) and pass the --force")
            logger.warning(" argument in. This goes to dak directly, so try to")
            logger.warning(" pay attention to formatting.")
            logger.warning("")
            logger.warning("")
            raise DmCommandError("DM fingerprint lookup "
                                 "for argument %s failed. "
                                 "GnuPG returned error: %s" %
                                 (args.dm, err))
        possible_fingerprints = []
        current_uid = None
        next_line_contains_fpr = False
        gpg_out = out.split("\n")
        for line in gpg_out:
            if next_line_contains_fpr:
                assert(line.startswith("fpr"))
                parsed_fingerprint = line.split(":")
                # fpr:::::::::CACE80AE01512F9AE8AB80D61C01F443C9C93C5A:
                possible_fingerprints.append((current_uid,
                                              parsed_fingerprint[9],))
                next_line_contains_fpr = False
                continue

            elif not line.startswith("pub"):
                continue

            else:
                # will give a line like:
                # pub:-:4096:1:7B585B30807C2A87:2011-08-18:::-:
                # Paul Tagliamonte <*****@*****.**>::scESC:
                # without the newline
                parsed_fingerprint = line.split(":")

                current_uid = parsed_fingerprint[9]
                next_line_contains_fpr = True

        if len(possible_fingerprints) > 1:
            raise DmCommandError("DM argument `%s' is ambiguous. "
                                 "Possible choices:\n%s" %
                                 (args.dm,
                                  pretty_print_list(possible_fingerprints)))

        possible_fingerprints = possible_fingerprints[0]
        logger.info("Picking DM %s with fingerprint %s" %
                    possible_fingerprints)
        args.dm = possible_fingerprints[1]
Exemple #15
0
def invoke_dput(changes, args):
    """
    .. warning::
       This method may change names. Please use it via :func:`dput.upload`.
       also, please don't depend on args, that's likely to change shortly.

    Given a changes file ``changes``, and arguments to dput ``args``,
    upload a package to the archive that makes sense.

    """
    profile = dput.profile.load_profile(args.host)
    check_modules(profile)

    fqdn = None
    if "fqdn" in profile:
        fqdn = profile['fqdn']
    else:
        fqdn = profile['name']

    logfile = determine_logfile(changes, profile, args)
    tmp_logfile = tempfile.NamedTemporaryFile()
    if should_write_logfile(args):
        full_upload_log = profile["full_upload_log"]
        if args.full_upload_log:
            full_upload_log = args.full_upload_log
        _write_upload_log(tmp_logfile.name, full_upload_log)

    if args.delayed:
        make_delayed_upload(profile, args.delayed)

    if args.simulate:
        logger.warning("Not uploading for real - dry run")

    if args.passive:
        force_passive_ftp_upload(profile)

    logger.info("Uploading %s using %s to %s (host: %s; directory: %s)" % (
        changes.get_package_name(),
        profile['method'],
        profile['name'],
        fqdn,
        profile['incoming']
    ))

    if 'hooks' in profile:
        run_pre_hooks(changes, profile)
    else:
        logger.trace(profile)
        logger.warning("No hooks defined in the profile. "
                       "Not checking upload.")

    # check only is a special case of -s
    if args.check_only:
        args.simulate = 1

    with uploader(profile['method'], profile,
                  simulate=args.simulate) as obj:

        if args.check_only:
            logger.info("Package %s passes all checks" % (
                changes.get_package_name()
            ))
            return

        if args.no_upload_log:
            logger.info("Not writing upload log upon request")

        files = changes.get_files() + [changes.get_changes_file()]
        for path in files:
            logger.info("Uploading %s%s" % (
                os.path.basename(path),
                " (simulation)" if args.simulate else ""
            ))

            if not args.simulate:
                obj.upload_file(path)

        if args.simulate:
            return

        if 'hooks' in profile:
            run_post_hooks(changes, profile)
        else:
            logger.trace(profile)
            logger.warning("No hooks defined in the profile. "
                           "Not post-processing upload.")
    if should_write_logfile(args):
        tmp_logfile.flush()
        shutil.copy(tmp_logfile.name, logfile)
        #print(tmp_logfile.name)
        tmp_logfile.close()
Exemple #16
0
def invoke_dcut(args):
    profile = dput.profile.load_profile(args.host)

    fqdn = None
    if "fqdn" in profile:
        fqdn = profile["fqdn"]

    if not "allow_dcut" in profile or not profile["allow_dcut"]:
        raise UploadException(
            "Profile %s does not allow command file uploads" "Please set allow_dcut=1 to allow such uploads"
        )

    logger.info("Uploading commands file to %s (incoming: %s)" % (fqdn or profile["name"], profile["incoming"]))

    if args.simulate:
        logger.warning("Not uploading for real - dry run")

    command = args.command
    assert issubclass(type(command), AbstractCommand)
    command.validate(args)

    if args.passive:
        force_passive_ftp_upload(profile)

    upload_path = None
    fh = None
    upload_filename = command.generate_commands_name(profile)
    try:
        if command.cmd_name == "upload":
            logger.debug("Uploading file %s as is to %s" % (args.upload_file, profile["name"]))
            if not os.access(args.upload_file, os.R_OK):
                raise DcutError("Cannot access %s: No such file" % (args.upload_file))
            upload_path = args.upload_file
        else:
            fh = tempfile.NamedTemporaryFile(mode="w+r", delete=False)
            (name, email) = write_header(fh, profile, args)
            command.produce(fh, args)
            fh.flush()
            # print fh.name
            fh.close()

            signing_key = None
            if "default_keyid" in profile:
                signing_key = profile["default_keyid"]
            if args.keyid:
                signing_key = args.keyid

            sign_file(fh.name, signing_key, profile, name, email)
            upload_path = fh.name

        if not args.simulate and not args.output:
            upload_commands_file(upload_path, upload_filename, profile, args)
        elif args.output and not args.simulate:
            if os.access(args.output, os.R_OK):
                logger.error("Not writing %s: File already exists" % (args.output))
                # ... but intentionally do nothing
                # TODO: or raise exception?
                return
            shutil.move(fh.name, args.output)
        elif args.simulate:
            pass
        else:
            # we should *never* come here
            assert False

    finally:
        if fh and os.access(fh.name, os.R_OK):
            os.unlink(fh.name)
Exemple #17
0
    def initialize(self, **kwargs):
        """
        See :meth:`dput.uploader.AbstractUploader.initialize`
        """
        fqdn = self._config['fqdn']
        incoming = self._config['incoming']

        self.sftp_config = {}
        if "sftp" in self._config:
            self.sftp_config = self._config['sftp']

        self.putargs = {'confirm': False}

        if "confirm_upload" in self.sftp_config:
            self.putargs['confirm'] = self.sftp_config['confirm_upload']

        if incoming.startswith('~/'):
            logger.warning("SFTP does not support ~/path, continuing with"
                           "relative directory name instead.")
            incoming = incoming[2:]
        # elif incoming.startswith('~') and not self.host_is_launchpad:
        #    raise SftpUploadException("SFTP doesn't support ~path. "
        #                              "if you need $HOME paths, use SCP.")
        #  XXX: What to do here?? - PRT

        ssh_kwargs = {
            "port": 22,
            "compress": True
        }

        # XXX: Timeout override

        if 'port' in self._config:
            ssh_kwargs['port'] = self._config['port']

        if 'scp_compress' in self._config:
            ssh_kwargs['compress'] = self._config['scp_compress']

        config = paramiko.SSHConfig()
        if os.path.exists('/etc/ssh/ssh_config'):
            config.parse(open('/etc/ssh/ssh_config'))
        if os.path.exists(os.path.expanduser('~/.ssh/config')):
            config.parse(open(os.path.expanduser('~/.ssh/config')))
        o = config.lookup(fqdn)

        user = find_username(self._config)
        if "user" in o:
            user = o['user']

        ssh_kwargs['username'] = user

        if 'identityfile' in o:
            pkey = os.path.expanduser(o['identityfile'])
            ssh_kwargs['key_filename'] = pkey

        logger.info("Logging into host %s as %s" % (fqdn, user))
        self._sshclient = paramiko.SSHClient()
        if 'globalknownhostsfile' in o:
            for gkhf in o['globalknownhostsfile'].split():
                if os.path.isfile(gkhf):
                    self._sshclient.load_system_host_keys(gkhf)
        else:
            files = [
                "/etc/ssh/ssh_known_hosts",
                "/etc/ssh/ssh_known_hosts2"
            ]
            for fpath in files:
                if os.path.isfile(fpath):
                    self._sshclient.load_system_host_keys(fpath)

        if 'userknownhostsfile' in o:
            for u in o['userknownhostsfile'].split():
                # actually, ssh supports a bit more than ~/,
                # but that would be a task for paramiko...
                ukhf = os.path.expanduser(u)
                if os.path.isfile(ukhf):
                    self._sshclient.load_host_keys(ukhf)
        else:
            for u in ['~/.ssh/known_hosts2', '~/.ssh/known_hosts']:
                ukhf = os.path.expanduser(u)
                if os.path.isfile(ukhf):
                    # Ideally, that should be load_host_keys,
                    # so that the known_hosts file can be written
                    # again. But paramiko can destroy the contents
                    # or parts of it, so no writing by using
                    # load_system_host_keys here, too:
                    self._sshclient.load_system_host_keys(ukhf)
        self._sshclient.set_missing_host_key_policy(AskToAccept(self))
        self._auth(fqdn, ssh_kwargs)
        try:
            self._sftp = self._sshclient.open_sftp()
        except paramiko.SSHException, e:
            raise SftpUploadException(
                "Error opening SFTP channel to %s (perhaps sftp is "
                "disabled there?): %s" % (
                    fqdn,
                    repr(e)
                )
            )
Exemple #18
0
        # self._sftp.chdir(incoming)
        self.incoming = incoming

    def _auth(self, fqdn, ssh_kwargs, _first=0):
        if _first == 3:
            raise SftpUploadException("Failed to authenticate")
        try:
            self._sshclient.connect(fqdn, **ssh_kwargs)
            logger.debug("Logged in!")
        except socket.error, e:
            raise SftpUploadException("SFTP error uploading to %s: %s" % (
                fqdn,
                repr(e)
            ))
        except paramiko.AuthenticationException:
            logger.warning("Failed to auth. Prompting for a login pair.")
            # XXX: Ask for pw only
            user = self.interface.question('please login', 'Username')
            # 4 first error
            pw = self.interface.password(None, "Password")

            if user is not None:
                ssh_kwargs['username'] = user
            ssh_kwargs['password'] = pw
            self._auth(fqdn, ssh_kwargs, _first=_first + 1)
        except paramiko.SSHException, e:
            raise SftpUploadException("SFTP error uploading to %s: %s" % (
                fqdn,
                repr(e)
            ))