def check_debs_in_upload(changes, profile, interface):
"""
The ``check-debs`` checker is a stock dput checker that checks packages
intended for upload for .deb packages.
Profile key: ``foo``
Example profile::
{
"skip": false,
"enforce": "debs"
}
``skip`` controls if the checker should drop out without checking
for anything at all.
``enforce`` This controls what we check for. Valid values are
"debs" or "source". Nonsense values will cause
an abort.
"""
debs = {}
if 'check-debs' in profile:
debs = profile['check-debs']
if 'skip' in debs and debs['skip']:
logger.debug("Skipping deb checker.")
return
enforce_debs = True
if 'enforce' in debs:
model = debs['enforce']
if model == 'debs':
enforce_debs = True
elif model == 'source':
enforce_debs = False
else:
logger.warning("Garbage value for check-debs/enforce - is %s,"
" valid values are `debs` and `source`. Skipping"
" checks." % (model))
return
else:
logger.warning("No `enforce` key in check-debs. Skipping checks.")
return
has_debs = False
for fil in changes.get_files():
xtns = ['.deb', '.udeb']
for xtn in xtns:
if fil.endswith(xtn):
has_debs = True
if enforce_debs and not has_debs:
raise BinaryUploadError(
"There are no .debs in this upload, and we're enforcing them."
)
if not enforce_debs and has_debs:
raise BinaryUploadError(
"There are .debs in this upload, and enforcing they don't exist."
)
def _auth(self, fqdn, ssh_kwargs, _first=0):
if _first == 3:
raise SftpUploadException("Failed to authenticate")
try:
self._sshclient.connect(fqdn, **ssh_kwargs)
logger.debug("Logged in!")
except socket.error as e:
raise SftpUploadException("SFTP error uploading to %s: %s" % (
fqdn,
repr(e)
))
except paramiko.AuthenticationException:
logger.warning("Failed to auth. Prompting for a login pair.")
# XXX: Ask for pw only
user = self.interface.question('please login', 'Username')
# 4 first error
pw = self.interface.password(None, "Password")
if user is not None:
ssh_kwargs['username'] = user
ssh_kwargs['password'] = pw
self._auth(fqdn, ssh_kwargs, _first=_first + 1)
except paramiko.SSHException as e:
raise SftpUploadException("SFTP error uploading to %s: %s" % (
fqdn,
repr(e)
))
def check_protected_distributions(changes, profile, interface):
"""
The ``protected distributions`` checker is a stock dput checker that makes
sure, users intending an upload for a special care archive (
testing-proposed-updates, stable-security, etc.) did really follow the
archive policies for that.
Profile key: none
"""
# XXX: This check does not contain code names yet. We need a global way
# to retrieve and share current code names.
suite = changes["Distribution"]
query_user = False
release_team_suites = ["testing-proposed-updates", "proposed-updates", "stable", "testing"]
if suite in release_team_suites:
msg = "Are you sure to upload to %s? Did you coordinate with the " "Release Team before your upload?" % (suite)
error_msg = "Aborting upload to Release Team managed suite upon " "request"
query_user = True
security_team_suites = ["stable-security", "oldstable-security", "testing-security"]
if suite in security_team_suites:
msg = "Are you sure to upload to %s? Did you coordinate with the " "Security Team before your upload?" % (suite)
error_msg = "Aborting upload to Security Team managed suite upon " "request"
query_user = True
if query_user:
logger.trace("Querying the user for input. The upload targets a " "protected distribution")
if not interface.boolean("Protected Checker", msg, default=BUTTON_NO):
raise BadDistributionError(error_msg)
else:
logger.warning("Uploading with explicit confirmation by the user")
else:
logger.trace("Nothing to do for checker protected_distributions")
def preload(self, configs):
"""
See :meth:`dput.config.AbstractConfig.preload`
"""
parser = configparser.ConfigParser()
if configs is None:
configs = dput.core.DPUT_CONFIG_LOCATIONS
for config in configs:
if not os.access(config, os.R_OK):
logger.debug("Skipping file %s: Not accessible" % (
config
))
continue
try:
logger.trace("Parsing %s" % (config))
parser.readfp(open(config, 'r'))
except IOError as e:
logger.warning("Skipping file %s: %s" % (
config,
e
))
continue
except configparser.ParsingError as e:
raise DputConfigurationError("Error parsing file %s: %s" % (
config,
e
))
self.parser = parser
self.configs = configs
self.defaults = self._translate_strs(self.get_config("DEFAULT"))
self.parser.remove_section("DEFAULT")
def _run_hook(self, hook):
if hook in self._config and self._config[hook] != "":
cmd = self._config[hook]
(output, stderr, ret) = run_command(cmd)
if ret == -1:
if not os.path.exists(cmd):
logger.warning(
"Error: You've set a hook (%s) to run (`%s`), "
"but it can't be found (and doesn't appear to exist)."
" Please verify the path and correct it." % (
hook,
self._config[hook]
)
)
return
sys.stdout.write(output) # XXX: Fixme
if ret != 0:
raise DputError(
"Command `%s' returned an error: %s [err=%d]" % (
self._config[hook],
stderr,
ret
)
)
def initialize(self, **kwargs):
"""
See :meth:`dput.uploader.AbstractUploader.initialize`
"""
fqdn = self._config['fqdn']
incoming = self._config['incoming']
ssh_options = []
if "ssh_options" in self._config:
ssh_options.extend(self._config['ssh_options'])
if 'port' in self._config:
ssh_options.append("-oPort=%d" % self._config['port'])
username = None
if 'login' in self._config and self._config['login'] != "*":
username = self._config['login']
if incoming.startswith('~/'):
logger.warning("SFTP does not support ~/path, continuing with"
"relative directory name instead.")
incoming = incoming[2:]
if username:
logger.info("Logging into host %s as %s" % (fqdn, username))
else:
logger.info("Logging into host %s" % fqdn)
self._sftp = Sftp(servername=fqdn, username=username,
ssh_options=ssh_options)
self.incoming = incoming
def lintian(changes, profile, interface):
"""
The ``lintian`` checker is a stock dput checker that checks packages
intended for upload for common mistakes, using the static checking
tool, `lintian <http://lintian.debian.org/>`.
Profile key: ``lintian``
Example profile::
{
"run_lintian": true
"lintian": {
}
}
No keys are current supported, but there are plans to set custom
ignore lists, etc.
"""
if "run_lintian" in profile:
logger.warning("Setting 'run_lintian' is deprecated. "
"Please configure the lintian checker instead.")
if not profile['run_lintian']: # XXX: Broken. Fixme.
logger.info("skipping lintian checking, enable with "
"run_lintian = 1 in your dput.cf")
return
tags = lint(
changes._absfile,
pedantic=True,
info=True,
experimental=True
)
sorted_tags = {}
for tag in tags:
if not tag['severity'] in sorted_tags:
sorted_tags[tag['severity']] = {}
if tag['tag'] not in sorted_tags[tag['severity']]:
sorted_tags[tag['severity']][tag['tag']] = tag
tags = sorted_tags
# XXX: Make this configurable
if not "E" in tags:
return
for tag in set(tags["E"]):
print " - %s: %s" % (tags["E"][tag]['severity'], tag)
inp = interface.boolean('Lintian Checker',
'Upload despite of Lintian finding issues?',
default=BUTTON_NO)
if not inp:
raise LintianHookException(
"User didn't own up to the "
"Lintian issues"
)
else:
logger.warning("Uploading with outstanding Lintian issues.")
def run_pre_hooks(changes, profile):
for name, hook in get_hooks(profile):
if 'pre' in hook and hook['pre']:
run_hook(name, hook, changes, profile)
if 'pre' not in hook and 'post' not in hook:
logger.warning("Hook: %s has no pre/post ordering. Assuming "
"pre.")
run_hook(name, hook, changes, profile)
def initialize(self, **kwargs):
"""
See :meth:`dput.uploader.AbstractUploader.initialize`
"""
login = find_username(self._config)
self._scp_base = ["scp", "-p", "-C"]
# XXX: Timeout?
if 'port' in self._config:
self._scp_base += ("-P", "%s" % self._config['port'])
self._scp_host = "%s@%s" % (login, self._config['fqdn'])
logger.debug("Using scp to upload to %s" % (self._scp_host))
logger.warning("SCP is deprecated. Please consider upgrading to SFTP.")
def upload_write_error(self, e):
"""
.. warning::
don't call this.
please don't call this
"""
# XXX: Refactor this, please god, refactor this.
logger.warning("""Upload permissions error
You either don't have the rights to upload a file, or, if this is on
ftp-master, you may have tried to overwrite a file already on the server.
Continuing anyway in case you want to recover from an incomplete upload.
No file was uploaded, however.""")
def lint(path, pedantic=False, info=False, experimental=False):
args = ["lintian", "--show-overrides"]
if pedantic:
args.append("--pedantic")
if info:
args.append("-I")
if experimental:
args.append("-E")
args.append(path)
try:
output = subprocess.check_output(args)
except subprocess.CalledProcessError as e:
output = e.output
except OSError as e:
logger.warning("Failed to execute lintian: %s" % (e))
raise DputConfigurationError("lintian: %s" % (e))
return process(output)
def get_obj(cls, checker_method): # checker_method is a bad name.
"""
Get an object by plugin def (``checker_method``) in class ``cls`` (such
as ``hooks``).
"""
logger.trace("Attempting to resolve %s %s" % (cls, checker_method))
try:
config = load_config(cls, checker_method)
validate_object('plugin', config, "%s/%s" % (cls, checker_method))
if config is None or config == {}:
raise NoSuchConfigError("No such config")
except NoSuchConfigError:
logger.debug("failed to resolve config %s" % (checker_method))
return None
path = config['path']
logger.trace("loading %s %s" % (cls, path))
try:
return load_obj(path)
except ImportError as e:
logger.warning("failed to resolve path %s: %s" % (path, e))
return None
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.
from dput.core import logger
from dput.exceptions import HookException
try:
from distro_info import (DebianDistroInfo, UbuntuDistroInfo,
DistroDataOutdated)
except ImportError:
logger.warning('Uploading to Ubuntu requires python-distro-info to be '
'installed')
raise
class UnknownDistribution(HookException):
"""
Subclass of the :class:`dput.exceptions.HookException`.
Thrown if the ``supported-distribution`` checker encounters an issue.
"""
pass
class UnsupportedDistribution(HookException):
"""
Subclass of the :class:`dput.exceptions.HookException`.
def validate(self, args):
if args.force:
return
if not os.path.exists(DM_KEYRING):
raise DmCommandError(
"To manage DM permissions, the `debian-keyring' "
"keyring package must be installed. "
"File %s does not exist" % (DM_KEYRING)
)
return
# I HATE embedded functions. But OTOH this function is not usable
# somewhere else, so...
def pretty_print_list(tuples):
fingerprints = ""
for entry in tuples:
fingerprints += "\n- %s (%s)" % entry
return fingerprints
# TODO: Validate input. Packages must exist (i.e. be not NEW)
(out, err, exit_status) = run_command([
"gpg", "--no-options",
"--no-auto-check-trustdb", "--no-default-keyring",
"--list-key", "--with-colons", "--fingerprint",
"--keyring", DM_KEYRING, args.dm
])
if exit_status != 0:
logger.warning("")
logger.warning("There was an error looking up the DM's key")
logger.warning("")
logger.warning(" dput-ng uses the DM keyring in /usr/share/keyrings/")
logger.warning(" as the keyring to pull full fingerprints from.")
logger.warning("")
logger.warning(" Please ensure your keyring is up to date:")
logger.warning("")
logger.warning(" sudo apt-get install debian-keyring")
logger.warning("")
logger.warning(" Or, if you can not get the keyring, you may use their")
logger.warning(" full fingerprint (without spaces) and pass the --force")
logger.warning(" argument in. This goes to dak directly, so try to")
logger.warning(" pay attention to formatting.")
logger.warning("")
logger.warning("")
raise DmCommandError("DM fingerprint lookup "
"for argument %s failed. "
"GnuPG returned error: %s" %
(args.dm, err))
possible_fingerprints = []
current_uid = None
next_line_contains_fpr = False
gpg_out = out.split("\n")
for line in gpg_out:
if next_line_contains_fpr:
assert(line.startswith("fpr"))
parsed_fingerprint = line.split(":")
# fpr:::::::::CACE80AE01512F9AE8AB80D61C01F443C9C93C5A:
possible_fingerprints.append((current_uid,
parsed_fingerprint[9],))
next_line_contains_fpr = False
continue
elif not line.startswith("pub"):
continue
else:
# will give a line like:
# pub:-:4096:1:7B585B30807C2A87:2011-08-18:::-:
# Paul Tagliamonte <*****@*****.**>::scESC:
# without the newline
parsed_fingerprint = line.split(":")
current_uid = parsed_fingerprint[9]
next_line_contains_fpr = True
if len(possible_fingerprints) > 1:
raise DmCommandError("DM argument `%s' is ambiguous. "
"Possible choices:\n%s" %
(args.dm,
pretty_print_list(possible_fingerprints)))
possible_fingerprints = possible_fingerprints[0]
logger.info("Picking DM %s with fingerprint %s" %
possible_fingerprints)
args.dm = possible_fingerprints[1]
def invoke_dput(changes, args):
"""
.. warning::
This method may change names. Please use it via :func:`dput.upload`.
also, please don't depend on args, that's likely to change shortly.
Given a changes file ``changes``, and arguments to dput ``args``,
upload a package to the archive that makes sense.
"""
profile = dput.profile.load_profile(args.host)
check_modules(profile)
fqdn = None
if "fqdn" in profile:
fqdn = profile['fqdn']
else:
fqdn = profile['name']
logfile = determine_logfile(changes, profile, args)
tmp_logfile = tempfile.NamedTemporaryFile()
if should_write_logfile(args):
full_upload_log = profile["full_upload_log"]
if args.full_upload_log:
full_upload_log = args.full_upload_log
_write_upload_log(tmp_logfile.name, full_upload_log)
if args.delayed:
make_delayed_upload(profile, args.delayed)
if args.simulate:
logger.warning("Not uploading for real - dry run")
if args.passive:
force_passive_ftp_upload(profile)
logger.info("Uploading %s using %s to %s (host: %s; directory: %s)" % (
changes.get_package_name(),
profile['method'],
profile['name'],
fqdn,
profile['incoming']
))
if 'hooks' in profile:
run_pre_hooks(changes, profile)
else:
logger.trace(profile)
logger.warning("No hooks defined in the profile. "
"Not checking upload.")
# check only is a special case of -s
if args.check_only:
args.simulate = 1
with uploader(profile['method'], profile,
simulate=args.simulate) as obj:
if args.check_only:
logger.info("Package %s passes all checks" % (
changes.get_package_name()
))
return
if args.no_upload_log:
logger.info("Not writing upload log upon request")
files = changes.get_files() + [changes.get_changes_file()]
for path in files:
logger.info("Uploading %s%s" % (
os.path.basename(path),
" (simulation)" if args.simulate else ""
))
if not args.simulate:
obj.upload_file(path)
if args.simulate:
return
if 'hooks' in profile:
run_post_hooks(changes, profile)
else:
logger.trace(profile)
logger.warning("No hooks defined in the profile. "
"Not post-processing upload.")
if should_write_logfile(args):
tmp_logfile.flush()
shutil.copy(tmp_logfile.name, logfile)
#print(tmp_logfile.name)
tmp_logfile.close()
def invoke_dcut(args):
profile = dput.profile.load_profile(args.host)
fqdn = None
if "fqdn" in profile:
fqdn = profile["fqdn"]
if not "allow_dcut" in profile or not profile["allow_dcut"]:
raise UploadException(
"Profile %s does not allow command file uploads" "Please set allow_dcut=1 to allow such uploads"
)
logger.info("Uploading commands file to %s (incoming: %s)" % (fqdn or profile["name"], profile["incoming"]))
if args.simulate:
logger.warning("Not uploading for real - dry run")
command = args.command
assert issubclass(type(command), AbstractCommand)
command.validate(args)
if args.passive:
force_passive_ftp_upload(profile)
upload_path = None
fh = None
upload_filename = command.generate_commands_name(profile)
try:
if command.cmd_name == "upload":
logger.debug("Uploading file %s as is to %s" % (args.upload_file, profile["name"]))
if not os.access(args.upload_file, os.R_OK):
raise DcutError("Cannot access %s: No such file" % (args.upload_file))
upload_path = args.upload_file
else:
fh = tempfile.NamedTemporaryFile(mode="w+r", delete=False)
(name, email) = write_header(fh, profile, args)
command.produce(fh, args)
fh.flush()
# print fh.name
fh.close()
signing_key = None
if "default_keyid" in profile:
signing_key = profile["default_keyid"]
if args.keyid:
signing_key = args.keyid
sign_file(fh.name, signing_key, profile, name, email)
upload_path = fh.name
if not args.simulate and not args.output:
upload_commands_file(upload_path, upload_filename, profile, args)
elif args.output and not args.simulate:
if os.access(args.output, os.R_OK):
logger.error("Not writing %s: File already exists" % (args.output))
# ... but intentionally do nothing
# TODO: or raise exception?
return
shutil.move(fh.name, args.output)
elif args.simulate:
pass
else:
# we should *never* come here
assert False
finally:
if fh and os.access(fh.name, os.R_OK):
os.unlink(fh.name)
def initialize(self, **kwargs):
"""
See :meth:`dput.uploader.AbstractUploader.initialize`
"""
fqdn = self._config['fqdn']
incoming = self._config['incoming']
self.sftp_config = {}
if "sftp" in self._config:
self.sftp_config = self._config['sftp']
self.putargs = {'confirm': False}
if "confirm_upload" in self.sftp_config:
self.putargs['confirm'] = self.sftp_config['confirm_upload']
if incoming.startswith('~/'):
logger.warning("SFTP does not support ~/path, continuing with"
"relative directory name instead.")
incoming = incoming[2:]
# elif incoming.startswith('~') and not self.host_is_launchpad:
# raise SftpUploadException("SFTP doesn't support ~path. "
# "if you need $HOME paths, use SCP.")
# XXX: What to do here?? - PRT
ssh_kwargs = {
"port": 22,
"compress": True
}
# XXX: Timeout override
if 'port' in self._config:
ssh_kwargs['port'] = self._config['port']
if 'scp_compress' in self._config:
ssh_kwargs['compress'] = self._config['scp_compress']
config = paramiko.SSHConfig()
if os.path.exists('/etc/ssh/ssh_config'):
config.parse(open('/etc/ssh/ssh_config'))
if os.path.exists(os.path.expanduser('~/.ssh/config')):
config.parse(open(os.path.expanduser('~/.ssh/config')))
o = config.lookup(fqdn)
user = find_username(self._config)
if "user" in o:
user = o['user']
ssh_kwargs['username'] = user
if 'identityfile' in o:
pkey = os.path.expanduser(o['identityfile'])
ssh_kwargs['key_filename'] = pkey
logger.info("Logging into host %s as %s" % (fqdn, user))
self._sshclient = paramiko.SSHClient()
if 'globalknownhostsfile' in o:
for gkhf in o['globalknownhostsfile'].split():
if os.path.isfile(gkhf):
self._sshclient.load_system_host_keys(gkhf)
else:
files = [
"/etc/ssh/ssh_known_hosts",
"/etc/ssh/ssh_known_hosts2"
]
for fpath in files:
if os.path.isfile(fpath):
self._sshclient.load_system_host_keys(fpath)
if 'userknownhostsfile' in o:
for u in o['userknownhostsfile'].split():
# actually, ssh supports a bit more than ~/,
# but that would be a task for paramiko...
ukhf = os.path.expanduser(u)
if os.path.isfile(ukhf):
self._sshclient.load_host_keys(ukhf)
else:
for u in ['~/.ssh/known_hosts2', '~/.ssh/known_hosts']:
ukhf = os.path.expanduser(u)
if os.path.isfile(ukhf):
# Ideally, that should be load_host_keys,
# so that the known_hosts file can be written
# again. But paramiko can destroy the contents
# or parts of it, so no writing by using
# load_system_host_keys here, too:
self._sshclient.load_system_host_keys(ukhf)
self._sshclient.set_missing_host_key_policy(AskToAccept(self))
self._auth(fqdn, ssh_kwargs)
try:
self._sftp = self._sshclient.open_sftp()
except paramiko.SSHException, e:
raise SftpUploadException(
"Error opening SFTP channel to %s (perhaps sftp is "
"disabled there?): %s" % (
fqdn,
repr(e)
)
)
# self._sftp.chdir(incoming)
self.incoming = incoming
def _auth(self, fqdn, ssh_kwargs, _first=0):
if _first == 3:
raise SftpUploadException("Failed to authenticate")
try:
self._sshclient.connect(fqdn, **ssh_kwargs)
logger.debug("Logged in!")
except socket.error, e:
raise SftpUploadException("SFTP error uploading to %s: %s" % (
fqdn,
repr(e)
))
except paramiko.AuthenticationException:
logger.warning("Failed to auth. Prompting for a login pair.")
# XXX: Ask for pw only
user = self.interface.question('please login', 'Username')
# 4 first error
pw = self.interface.password(None, "Password")
if user is not None:
ssh_kwargs['username'] = user
ssh_kwargs['password'] = pw
self._auth(fqdn, ssh_kwargs, _first=_first + 1)
except paramiko.SSHException, e:
raise SftpUploadException("SFTP error uploading to %s: %s" % (
fqdn,
repr(e)
))