def sign_release_dir(suite, dirname): cnf = Config() if 'Dinstall::SigningKeyring' in cnf or 'Dinstall::SigningHomedir' in cnf: args = { 'keyids': suite.signingkeys or [], 'pubring': cnf.get('Dinstall::SigningPubKeyring') or None, 'secring': cnf.get('Dinstall::SigningKeyring') or None, 'homedir': cnf.get('Dinstall::SigningHomedir') or None, 'passphrase_file': cnf.get('Dinstall::SigningPassphraseFile') or None, } relname = os.path.join(dirname, 'Release') dest = os.path.join(dirname, 'Release.gpg') if os.path.exists(dest): os.unlink(dest) inlinedest = os.path.join(dirname, 'InRelease') if os.path.exists(inlinedest): os.unlink(inlinedest) with open(relname, 'r') as stdin: with open(dest, 'w') as stdout: daklib.gpg.sign(stdin, stdout, inline=False, **args) stdin.seek(0) with open(inlinedest, 'w') as stdout: daklib.gpg.sign(stdin, stdout, inline=True, **args)
def prepare(self): """prepare upload for further processing This copies the files involved to a temporary directory. If you use this method directly, you have to remove the directory given by the C{directory} attribute later on your own. Instead of using the method directly, you can also use a with-statement:: with ArchiveUpload(...) as upload: ... This will automatically handle any required cleanup. """ assert self.directory is None assert self.original_changes.valid_signature cnf = Config() session = self.transaction.session group = cnf.get('Dinstall::UnprivGroup') or None self.directory = utils.temp_dirname(parent=cnf.get('Dir::TempPath'), mode=0o2750, group=group) with FilesystemTransaction() as fs: src = os.path.join(self.original_directory, self.original_changes.filename) dst = os.path.join(self.directory, self.original_changes.filename) fs.copy(src, dst, mode=0o640) self.changes = upload.Changes(self.directory, self.original_changes.filename, self.keyrings) for f in self.changes.files.itervalues(): src = os.path.join(self.original_directory, f.filename) dst = os.path.join(self.directory, f.filename) if not os.path.exists(src): continue fs.copy(src, dst, mode=0o640) source = None try: source = self.changes.source except Exception: # Do not raise an exception here if the .dsc is invalid. pass if source is not None: for f in source.files.itervalues(): src = os.path.join(self.original_directory, f.filename) dst = os.path.join(self.directory, f.filename) if not os.path.exists(dst): try: db_file = self.transaction.get_file(f, source.dsc['Source'], check_hashes=False) db_archive_file = session.query(ArchiveFile).filter_by(file=db_file).first() fs.copy(db_archive_file.path, dst, mode=0o640) except KeyError: # Ignore if get_file could not find it. Upload will # probably be rejected later. pass
def _subst_for_upload(upload): """ Prepare substitutions used for announce mails. @type upload: L{daklib.upload.Source} or L{daklib.upload.Binary} @param upload: upload to handle @rtype: dict @returns: A dict of substition values for use by L{daklib.utils.TemplateSubst} """ cnf = Config() maintainer = upload.maintainer or cnf['Dinstall::MyEmailAddress'] changed_by = upload.changed_by or maintainer if upload.sourceful: maintainer_to = mail_addresses_for_upload(maintainer, changed_by, upload.fingerprint) else: maintainer_to = mail_addresses_for_upload(maintainer, maintainer, upload.fingerprint) bcc = 'X-DAK: dak {0}'.format(upload.program) if 'Dinstall::Bcc' in cnf: bcc = '{0}\nBcc: {1}'.format(bcc, cnf['Dinstall::Bcc']) subst = { '__DISTRO__': cnf['Dinstall::MyDistribution'], '__BUG_SERVER__': cnf.get('Dinstall::BugServer'), '__ADMIN_ADDRESS__': cnf['Dinstall::MyAdminAddress'], '__DAK_ADDRESS__': cnf['Dinstall::MyEmailAddress'], '__REJECTOR_ADDRESS__': cnf['Dinstall::MyEmailAddress'], '__MANUAL_REJECT_MESSAGE__': '', '__BCC__': bcc, '__MAINTAINER__': changed_by, '__MAINTAINER_FROM__': fix_maintainer(changed_by)[1], '__MAINTAINER_TO__': ', '.join(maintainer_to), '__CHANGES_FILENAME__': upload.changes_filename, '__FILE_CONTENTS__': upload.changes, '__SOURCE__': upload.source, '__VERSION__': upload.version, '__ARCHITECTURE__': upload.architecture, '__WARNINGS__': '\n'.join(upload.warnings), } override_maintainer = cnf.get('Dinstall::OverrideMaintainer') if override_maintainer: subst['__MAINTAINER_FROM__'] = subst['__MAINTAINER_TO__'] = override_maintainer return subst
def main(): global Options, Logger, Sections, Priorities cnf = Config() session = DBConn().session() Arguments = [('a', "automatic", "Process-New::Options::Automatic"), ('b', "no-binaries", "Process-New::Options::No-Binaries"), ('c', "comments", "Process-New::Options::Comments"), ('h', "help", "Process-New::Options::Help"), ('m', "manual-reject", "Process-New::Options::Manual-Reject", "HasArg"), ('t', "trainee", "Process-New::Options::Trainee"), ('q', 'queue', 'Process-New::Options::Queue', 'HasArg'), ('n', "no-action", "Process-New::Options::No-Action")] changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) for i in [ "automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee" ]: if not cnf.has_key("Process-New::Options::%s" % (i)): cnf["Process-New::Options::%s" % (i)] = "" queue_name = cnf.get('Process-New::Options::Queue', 'new') new_queue = session.query(PolicyQueue).filter_by( queue_name=queue_name).one() if len(changes_files) == 0: uploads = new_queue.uploads else: uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \ .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all() Options = cnf.subtree("Process-New::Options") if Options["Help"]: usage() if not Options["No-Action"]: try: Logger = daklog.Logger("process-new") except CantOpenError as e: Options["Trainee"] = "True" Sections = Section_Completer(session) Priorities = Priority_Completer(session) readline.parse_and_bind("tab: complete") if len(uploads) > 1: sys.stderr.write("Sorting changes...\n") uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"]) if Options["Comments"]: show_new_comments(uploads, session) else: for upload in uploads: do_pkg(upload, session) end()
def action_dm_migrate(self, fingerprint, section, session): self._action_dm_admin_common(fingerprint, section, session) cnf = Config() acl_name = cnf.get('Command::DM::ACL', 'dm') acl = session.query(ACL).filter_by(name=acl_name).one() fpr_hash_from = section['From'].translate(None, ' ') fpr_from = session.query(Fingerprint).filter_by(fingerprint=fpr_hash_from).first() if fpr_from is None: self.result.append('Unknown fingerprint (From): {0}\nNo action taken.'.format(fpr_hash_from)) return fpr_hash_to = section['To'].translate(None, ' ') fpr_to = session.query(Fingerprint).filter_by(fingerprint=fpr_hash_to).first() if fpr_to is None: self.result.append('Unknown fingerprint (To): {0}\nNo action taken.'.format(fpr_hash_to)) return if fpr_to.keyring is None or fpr_to.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'): self.result.append('Key (To) {0} is not in DM keyring.\nNo action taken.'.format(fpr_to.fingerprint)) return self.log.log(['dm-migrate', 'from={0}'.format(fpr_hash_from), 'to={0}'.format(fpr_hash_to)]) sources = [] for entry in session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr_from): self.log.log(['dm-migrate', 'from={0}'.format(fpr_hash_from), 'to={0}'.format(fpr_hash_to), 'source={0}'.format(entry.source)]) entry.fingerprint = fpr_to sources.append(entry.source) self.result.append('Migrated {0} to {1}.\n{2} acl entries changed: {3}'.format(fpr_hash_from, fpr_hash_to, len(sources), ", ".join(sources))) session.commit()
def do_pkg(upload_id): cnf = Config() session = DBConn().session() upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one() queue = upload.policy_queue changes = upload.changes origchanges = os.path.join(queue.path, changes.changesname) print origchanges htmlname = "{0}_{1}.html".format(changes.source, changes.version) htmlfile = os.path.join(cnf['Show-New::HTMLPath'], htmlname) # Have we already processed this? if os.path.exists(htmlfile) and \ os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()): with open(htmlfile, "r") as fd: if fd.read() != timeout_str: sources.append(htmlname) return (PROC_STATUS_SUCCESS, '%s already up-to-date' % htmlfile) # Go, process it... Now! htmlfiles_to_process.append(htmlfile) sources.append(htmlname) group = cnf.get('Dinstall::UnprivGroup') or None with open(htmlfile, 'w') as outfile: with policy.UploadCopy(upload, group=group) as upload_copy: handler = policy.PolicyQueueUploadHandler(upload, session) missing = [(o['type'], o['package']) for o in handler.missing_overrides()] distribution = changes.distribution print >> outfile, html_header(changes.source, missing) print >> outfile, examine_package.display_changes( distribution, origchanges) if upload.source is not None and ('dsc', upload.source.source) in missing: fn = os.path.join(upload_copy.directory, upload.source.poolfile.basename) print >> outfile, examine_package.check_dsc( distribution, fn, session) for binary in upload.binaries: if (binary.binarytype, binary.package) not in missing: continue fn = os.path.join(upload_copy.directory, binary.poolfile.basename) print >> outfile, examine_package.check_deb( distribution, fn, session) print >> outfile, html_footer() session.close() htmlfiles_to_process.remove(htmlfile) return (PROC_STATUS_SUCCESS, '{0} already updated'.format(htmlfile))
def _do_bts_versiontracking(self): cnf = Config() fs = self.transaction.fs btsdir = cnf.get('Dir::BTSVersionTrack') if btsdir is None or btsdir == '': return base = os.path.join(btsdir, self.changes.filename[:-8]) # version history sourcedir = self.unpacked_source() if sourcedir is not None: fh = open(os.path.join(sourcedir, 'debian', 'changelog'), 'r') versions = fs.create("{0}.versions".format(base), mode=0o644) for line in fh.readlines(): if re_changelog_versions.match(line): versions.write(line) fh.close() versions.close() # binary -> source mapping debinfo = fs.create("{0}.debinfo".format(base), mode=0o644) for binary in self.changes.binaries: control = binary.control source_package, source_version = binary.source line = " ".join([control['Package'], control['Version'], control['Architecture'], source_package, source_version]) print >>debinfo, line debinfo.close()
def action_dm_remove(self, fingerprint, section, session): self._action_dm_admin_common(fingerprint, section, session) cnf = Config() acl_name = cnf.get('Command::DM::ACL', 'dm') acl = session.query(ACL).filter_by(name=acl_name).one() fpr_hash = section['Fingerprint'].translate(None, ' ') fpr = session.query(Fingerprint).filter_by( fingerprint=fpr_hash).first() if fpr is None: self.result.append( 'Unknown fingerprint: {0}\nNo action taken.'.format(fpr_hash)) return self.log.log(['dm-remove', fpr.fingerprint]) count = 0 for entry in session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr): self.log.log([ 'dm-remove', fpr.fingerprint, 'source={0}'.format(entry.source) ]) count += 1 session.delete(entry) self.result.append('Removed: {0}.\n{1} acl entries removed.'.format( fpr.fingerprint, count)) session.commit()
def lock_package(package): """ Lock C{package} so that noone else jumps in processing it. @type package: string @param package: source package name to lock """ cnf = Config() path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package) try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) except OSError as e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: user = pwd.getpwuid( os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') raise AlreadyLockedError(user) try: yield fd finally: os.unlink(path)
def main(): global Options, Logger cnf = Config() for i in ["Help"]: key = "Manage-External-Signature-Requests::Options::{}".format(i) if key not in cnf: cnf[key] = "" Arguments = [('h', "help", "Manage-External-Signature-Requests::Options::Help")] apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Manage-External-Signature-Requests::Options") if Options["Help"]: usage() Logger = daklog.Logger('manage-external-signature-requests') if 'External-Signature-Requests' not in cnf: print("DAK not configured to handle external signature requests") return config = cnf.subtree('External-Signature-Requests') session = DBConn().session() export_external_signature_requests(session, config['Export']) if 'ExportSigningKeys' in config: args = { 'pubring': cnf.get('Dinstall::SigningPubKeyring') or None, 'secring': cnf.get('Dinstall::SigningKeyring') or None, 'homedir': cnf.get('Dinstall::SigningHomedir') or None, 'passphrase_file': cnf.get('Dinstall::SigningPassphraseFile') or None, } sign_external_signature_requests( session, config['Export'], config.value_list('ExportSigningKeys'), args) session.close() Logger.close()
def check(self, upload): changes = upload.changes # Only check sourceful uploads. if changes.source is None: return True # Only check uploads to unstable or experimental. if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions: return True cnf = Config() if 'Dinstall::LintianTags' not in cnf: return True tagfile = cnf['Dinstall::LintianTags'] with open(tagfile, 'r') as sourcefile: sourcecontent = sourcefile.read() try: lintiantags = yaml.safe_load(sourcecontent)['lintian'] except yaml.YAMLError as msg: raise Exception('Could not read lintian tags file {0}, YAML error: {1}'.format(tagfile, msg)) fd, temp_filename = utils.temp_filename(mode=0o644) temptagfile = os.fdopen(fd, 'w') for tags in lintiantags.itervalues(): for tag in tags: print >>temptagfile, tag temptagfile.close() changespath = os.path.join(upload.directory, changes.filename) try: cmd = [] result = 0 user = cnf.get('Dinstall::UnprivUser') or None if user is not None: cmd.extend(['sudo', '-H', '-u', user]) cmd.extend(['/usr/bin/lintian', '--show-overrides', '--tags-from-file', temp_filename, changespath]) output = daklib.daksubprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: result = e.returncode output = e.output finally: os.unlink(temp_filename) if result == 2: utils.warn("lintian failed for %s [return code: %s]." % \ (changespath, result)) utils.warn(utils.prefix_multi_line_string(output, \ " [possible output:] ")) parsed_tags = lintian.parse_lintian_output(output) rejects = list(lintian.generate_reject_messages(parsed_tags, lintiantags)) if len(rejects) != 0: raise Reject('\n'.join(rejects)) return True
def announce_accept(upload): """ Announce an upload. @type upload: L{daklib.upload.Source} or L{daklib.upload.Binary} @param upload: upload to handle """ cnf = Config() subst = _subst_for_upload(upload) whitelists = _whitelists(upload) accepted_to_real_suite = any(suite.policy_queue is None or suite in upload.from_policy_suites for suite in upload.suites) suite_names = [] for suite in upload.suites: if suite.policy_queue: suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name)) else: suite_names.append(suite.suite_name) suite_names.extend(suite.suite_name for suite in upload.from_policy_suites) subst['__SUITE__'] = ', '.join(suite_names) or '(none)' message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted')) send_mail(message, whitelists=whitelists) if accepted_to_real_suite and upload.sourceful: # send mail to announce lists and tracking server announce = set() for suite in upload.suites: if suite.policy_queue is None or suite in upload.from_policy_suites: announce.update(suite.announce or []) announce_list_address = ", ".join(announce) # according to #890944 this email shall be sent to dispatch@<TrackingServer> to avoid # bouncing emails # the package email alias is not yet created shortly after accepting the package tracker = cnf.get('Dinstall::TrackingServer') if tracker: announce_list_address = "{0}\nBcc: dispatch@{1}".format(announce_list_address, tracker) if len(announce_list_address) != 0: my_subst = subst.copy() my_subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_list_address message = TemplateSubst(my_subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce')) send_mail(message, whitelists=whitelists) close_bugs_default = cnf.find_b('Dinstall::CloseBugs') close_bugs = any(s.close_bugs if s.close_bugs is not None else close_bugs_default for s in upload.suites) if accepted_to_real_suite and upload.sourceful and close_bugs: for bug in upload.bugs: my_subst = subst.copy() my_subst['__BUG_NUMBER__'] = str(bug) message = TemplateSubst(my_subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close')) send_mail(message, whitelists=whitelists)
def main(): global Options cnf = Config() Arguments = [ ('h', "help", "Auto-Decruft::Options::Help"), ('n', "dry-run", "Auto-Decruft::Options::Dry-Run"), ('d', "debug", "Auto-Decruft::Options::Debug"), ('s', "suite", "Auto-Decruft::Options::Suite", "HasArg"), # The "\0" seems to be the only way to disable short options. ("\0", 'if-newer-version-in', "Auto-Decruft::Options::OtherSuite", "HasArg"), ("\0", 'if-newer-version-in-rm-msg', "Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg") ] for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg"]: if not cnf.has_key("Auto-Decruft::Options::%s" % (i)): cnf["Auto-Decruft::Options::%s" % (i)] = "" cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Auto-Decruft::Options") if Options["Help"]: usage() debug = False dryrun = False if Options["Dry-Run"]: dryrun = True if Options["Debug"]: debug = True if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: utils.fubar( "--if-newer-version-in requires --if-newer-version-in-rm-msg") session = DBConn().session() suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() auto_decruft_suite(suite_name, suite_id, session, dryrun, debug) if Options["OtherSuite"]: osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun) if not dryrun: session.commit()
def main(): global Options cnf = Config() Arguments = [('h', "help", "Auto-Decruft::Options::Help"), ('n', "dry-run", "Auto-Decruft::Options::Dry-Run"), ('d', "debug", "Auto-Decruft::Options::Debug"), ('s', "suite", "Auto-Decruft::Options::Suite", "HasArg"), # The "\0" seems to be the only way to disable short options. ("\0", 'if-newer-version-in', "Auto-Decruft::Options::OtherSuite", "HasArg"), ("\0", 'if-newer-version-in-rm-msg', "Auto-Decruft::Options::OtherSuiteRMMsg", "HasArg"), ("\0", 'decruft-equal-versions', "Auto-Decruft::Options::OtherSuiteDecruftEqual") ] for i in ["help", "Dry-Run", "Debug", "OtherSuite", "OtherSuiteRMMsg", "OtherSuiteDecruftEqual"]: key = "Auto-Decruft::Options::%s" % i if key not in cnf: cnf[key] = "" cnf["Auto-Decruft::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Auto-Decruft::Options") if Options["Help"]: usage() debug = False dryrun = False decruft_equal_versions = False if Options["Dry-Run"]: dryrun = True if Options["Debug"]: debug = True if Options["OtherSuiteDecruftEqual"]: decruft_equal_versions = True if Options["OtherSuite"] and not Options["OtherSuiteRMMsg"]: utils.fubar("--if-newer-version-in requires --if-newer-version-in-rm-msg") session = DBConn().session() suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() auto_decruft_suite(suite_name, suite_id, session, dryrun, debug) if Options["OtherSuite"]: osuite = get_suite(Options["OtherSuite"].lower(), session).suite_name decruft_newer_version_in(osuite, suite_name, suite_id, Options["OtherSuiteRMMsg"], session, dryrun, decruft_equal_versions) if not dryrun: session.commit()
def main(argv=None): if argv is None: argv = sys.argv arguments = [('h', 'help', 'Process-Commands::Options::Help'), ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')] cnf = Config() cnf['Process-Commands::Options::Dummy'] = '' filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) options = cnf.subtree('Process-Commands::Options') if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options): usage() sys.exit(0) log = Logger('command') now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) rejectdir = cnf['Dir::Reject'] if len(filenames) == 0: filenames = [ fn for fn in os.listdir(options['Directory']) if fn.endswith('.dak-commands') ] for fn in filenames: basename = os.path.basename(fn) if not fn.endswith('.dak-commands'): log.log(['unexpected filename', basename]) continue with open(fn, 'r') as fh: data = fh.read() try: command = CommandFile(basename, data, log) command.evaluate() except: created = os.stat(fn).st_mtime now = time.time() too_new = (now - created < int(cnf.get('Dinstall::SkipTime', '60'))) if too_new: log.log(['skipped (too new)']) continue log.log(['reject', basename]) dst = find_next_free(os.path.join(rejectdir, basename)) else: log.log(['done', basename]) dst = find_next_free(os.path.join(donedir, basename)) with FilesystemTransaction() as fs: fs.unlink(fn) fs.create(dst, mode=0o644).write(data) fs.commit() log.close()
def main(): global Options, Logger, Sections, Priorities cnf = Config() session = DBConn().session() Arguments = [('a', "automatic", "Process-New::Options::Automatic"), ('b', "no-binaries", "Process-New::Options::No-Binaries"), ('c', "comments", "Process-New::Options::Comments"), ('h', "help", "Process-New::Options::Help"), ('m', "manual-reject", "Process-New::Options::Manual-Reject", "HasArg"), ('t', "trainee", "Process-New::Options::Trainee"), ('q', 'queue', 'Process-New::Options::Queue', 'HasArg'), ('n', "no-action", "Process-New::Options::No-Action")] changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) for i in ["automatic", "no-binaries", "comments", "help", "manual-reject", "no-action", "version", "trainee"]: key = "Process-New::Options::%s" % i if key not in cnf: cnf[key] = "" queue_name = cnf.get('Process-New::Options::Queue', 'new') new_queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).one() if len(changes_files) == 0: uploads = new_queue.uploads else: uploads = session.query(PolicyQueueUpload).filter_by(policy_queue=new_queue) \ .join(DBChange).filter(DBChange.changesname.in_(changes_files)).all() Options = cnf.subtree("Process-New::Options") if Options["Help"]: usage() if not Options["No-Action"]: try: Logger = daklog.Logger("process-new") except CantOpenError as e: Options["Trainee"] = "True" Sections = Section_Completer(session) Priorities = Priority_Completer(session) readline.parse_and_bind("tab: complete") if len(uploads) > 1: print("Sorting changes...", file=sys.stderr) uploads = sort_uploads(new_queue, uploads, session, Options["No-Binaries"]) if Options["Comments"]: show_new_comments(uploads, session) else: for upload in uploads: do_pkg(upload, session) end()
def main(): global Options, Logger cnf = Config() for i in ["Help"]: key = "Manage-External-Signature-Requests::Options::{}".format(i) if key not in cnf: cnf[key] = "" Arguments = [('h', "help", "Manage-External-Signature-Requests::Options::Help")] apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Manage-External-Signature-Requests::Options") if Options["Help"]: usage() Logger = daklog.Logger('manage-external-signature-requests') if 'External-Signature-Requests' not in cnf: print("DAK not configured to handle external signature requests") return config = cnf.subtree('External-Signature-Requests') session = DBConn().session() export_external_signature_requests(session, config['Export']) if 'ExportSigningKeys' in config: args = { 'pubring': cnf.get('Dinstall::SigningPubKeyring') or None, 'secring': cnf.get('Dinstall::SigningKeyring') or None, 'homedir': cnf.get('Dinstall::SigningHomedir') or None, 'passphrase_file': cnf.get('Dinstall::SigningPassphraseFile') or None, } sign_external_signature_requests(session, config['Export'], config.value_list('ExportSigningKeys'), args) session.close() Logger.close()
def do_update(self): """ Add suite options for overrides and control-suite to DB """ print(__doc__) try: cnf = Config() c = self.db.cursor() c.execute("ALTER TABLE suite ADD COLUMN overrideprocess BOOLEAN NOT NULL DEFAULT FALSE") c.execute("COMMENT ON COLUMN suite.overrideprocess IS %s", ['If true, check-overrides will process the suite by default']) c.execute("ALTER TABLE suite ADD COLUMN overrideorigin TEXT DEFAULT NULL") c.execute("COMMENT ON COLUMN suite.overrideprocess IS %s", ['If NOT NULL, check-overrides will take missing overrides from the named suite']) # Migrate config file values into database if "Check-Overrides::OverrideSuites" in cnf: for suitename in cnf.subtree("Check-Overrides::OverrideSuites").list(): if cnf.get("Check-Overrides::OverrideSuites::%s::Process" % suitename, "0") == "1": print("Marking %s to have overrides processed automatically" % suitename.lower()) c.execute("UPDATE suite SET overrideprocess = TRUE WHERE suite_name = %s", [suitename.lower()]) originsuite = cnf.get("Check-Overrides::OverrideSuites::%s::OriginSuite" % suitename, '') if originsuite != '': print("Setting %s to use %s as origin for overrides" % (suitename.lower(), originsuite.lower())) c.execute("UPDATE suite SET overrideorigin = %s WHERE suite_name = %s", [originsuite.lower(), suitename.lower()]) c.execute("ALTER TABLE suite ADD COLUMN allowcsset BOOLEAN NOT NULL DEFAULT FALSE") c.execute("COMMENT ON COLUMN suite.allowcsset IS %s", ['Allow control-suite to be used with the --set option without forcing']) # Import historical hard-coded values c.execute("UPDATE suite SET allowcsset = TRUE WHERE suite_name IN ('testing', 'squeeze-updates')") c.execute("UPDATE config SET value = '70' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError('Unable to apply sick update 70, rollback issued. Error message : %s' % (str(msg)))
def main(): global Options, Logger cnf = Config() for i in ["Help", "No-Action", "Maximum" ]: if not cnf.has_key("Clean-Suites::Options::%s" % (i)): cnf["Clean-Suites::Options::%s" % (i)] = "" Arguments = [('h',"help","Clean-Suites::Options::Help"), ('n',"no-action","Clean-Suites::Options::No-Action"), ('m',"maximum","Clean-Suites::Options::Maximum", "HasArg")] apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Clean-Suites::Options") if cnf["Clean-Suites::Options::Maximum"] != "": try: # Only use Maximum if it's an integer max_delete = int(cnf["Clean-Suites::Options::Maximum"]) if max_delete < 1: utils.fubar("If given, Maximum must be at least 1") except ValueError as e: utils.fubar("If given, Maximum must be an integer") else: max_delete = None if Options["Help"]: usage() Logger = daklog.Logger("clean-suites", debug=Options["No-Action"]) session = DBConn().session() now_date = datetime.now() # Stay of execution; default to 1.5 days soe = int(cnf.get('Clean-Suites::StayOfExecution', '129600')) delete_date = now_date - timedelta(seconds=soe) check_binaries(now_date, delete_date, max_delete, session) clean_binaries(now_date, delete_date, max_delete, session) check_sources(now_date, delete_date, max_delete, session) check_files(now_date, delete_date, max_delete, session) clean(now_date, delete_date, max_delete, session) clean_maintainers(now_date, delete_date, max_delete, session) clean_fingerprints(now_date, delete_date, max_delete, session) clean_empty_directories(session) Logger.close()
def do_pkg(upload_id): cnf = Config() session = DBConn().session() upload = session.query(PolicyQueueUpload).filter_by(id=upload_id).one() queue = upload.policy_queue changes = upload.changes origchanges = os.path.join(queue.path, changes.changesname) print origchanges htmlname = "{0}_{1}.html".format(changes.source, changes.version) htmlfile = os.path.join(cnf["Show-New::HTMLPath"], htmlname) # Have we already processed this? if os.path.exists(htmlfile) and os.stat(htmlfile).st_mtime > time.mktime(changes.created.timetuple()): with open(htmlfile, "r") as fd: if fd.read() != timeout_str: sources.append(htmlname) return (PROC_STATUS_SUCCESS, "%s already up-to-date" % htmlfile) # Go, process it... Now! htmlfiles_to_process.append(htmlfile) sources.append(htmlname) group = cnf.get("Dinstall::UnprivGroup") or None with open(htmlfile, "w") as outfile: with policy.UploadCopy(upload, group=group) as upload_copy: handler = policy.PolicyQueueUploadHandler(upload, session) missing = [(o["type"], o["package"]) for o in handler.missing_overrides()] distribution = changes.distribution print >> outfile, html_header(changes.source, missing) print >> outfile, examine_package.display_changes(distribution, origchanges) if upload.source is not None and ("dsc", upload.source.source) in missing: fn = os.path.join(upload_copy.directory, upload.source.poolfile.basename) print >> outfile, examine_package.check_dsc(distribution, fn, session) for binary in upload.binaries: if (binary.binarytype, binary.package) not in missing: continue fn = os.path.join(upload_copy.directory, binary.poolfile.basename) print >> outfile, examine_package.check_deb(distribution, fn, session) print >> outfile, html_footer() session.close() htmlfiles_to_process.remove(htmlfile) return (PROC_STATUS_SUCCESS, "{0} already updated".format(htmlfile))
def get_transitions(self): cnf = Config() path = cnf.get('Dinstall::ReleaseTransitions', '') if path == '' or not os.path.exists(path): return None contents = file(path, 'r').read() try: transitions = yaml.safe_load(contents) return transitions except yaml.YAMLError as msg: utils.warn('Not checking transitions, the transitions file is broken: {0}'.format(msg)) return None
def do_lintian (filename): cnf = Config() cmd = [] user = cnf.get('Dinstall::UnprivUser') or None if user is not None: cmd.extend(['sudo', '-H', '-u', user]) color = 'always' if use_html: color = 'html' cmd.extend(['lintian', '--show-overrides', '--color', color, "--", filename]) return do_command(cmd, escaped=True)
def do_lintian(filename): cnf = Config() cmd = [] user = cnf.get("Dinstall::UnprivUser") or None if user is not None: cmd.extend(["sudo", "-H", "-u", user]) color = "always" if use_html: color = "html" cmd.extend(["lintian", "--show-overrides", "--color", color, "--", filename]) return do_command(cmd, escaped=True)
def announce_accept(upload): cnf = Config() subst = _subst_for_upload(upload) whitelists = _whitelists(upload) accepted_to_real_suite = any(suite.policy_queue is None or suite in upload.from_policy_suites for suite in upload.suites) suite_names = [] for suite in upload.suites: if suite.policy_queue: suite_names.append("{0}->{1}".format(suite.suite_name, suite.policy_queue.queue_name)) else: suite_names.append(suite.suite_name) suite_names.extend(suite.suite_name for suite in upload.from_policy_suites) subst['__SUITE__'] = ', '.join(suite_names) or '(none)' message = TemplateSubst(subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.accepted')) send_mail(message, whitelists=whitelists) if accepted_to_real_suite and upload.sourceful: # senf mail to announce lists and tracking server announce = set() for suite in upload.suites: if suite.policy_queue is None or suite in upload.from_policy_suites: announce.update(suite.announce or []) announce_list_address = ", ".join(announce) tracking = cnf.get('Dinstall::TrackingServer') if tracking: announce_list_address = "{0}\nBcc: {1}@{2}".format(announce_list_address, upload.source, tracking) if len(announce_list_address) != 0: my_subst = subst.copy() my_subst['__ANNOUNCE_LIST_ADDRESS__'] = announce_list_address message = TemplateSubst(my_subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.announce')) send_mail(message, whitelists=whitelists) close_bugs_default = cnf.find_b('Dinstall::CloseBugs') close_bugs = any(s.close_bugs if s.close_bugs is not None else close_bugs_default for s in upload.suites) if accepted_to_real_suite and upload.sourceful and close_bugs: for bug in upload.bugs: my_subst = subst.copy() my_subst['__BUG_NUMBER__'] = str(bug) message = TemplateSubst(my_subst, os.path.join(cnf['Dir::Templates'], 'process-unchecked.bug-close')) send_mail(message, whitelists=whitelists)
def do_update(self): print __doc__ try: cnf = Config() c = self.db.cursor() stayofexecution = cnf.get('Clean-Suites::StayOfExecution', '129600') c.execute("ALTER TABLE archive ADD COLUMN stayofexecution INTERVAL NOT NULL DEFAULT %s", (stayofexecution,)) c.execute("UPDATE archive SET stayofexecution='0' WHERE name IN ('new', 'policy', 'build-queues')") c.execute("UPDATE config SET value = '77' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError('Unable to apply sick update 77, rollback issued. Error message: {0}'.format(msg))
def check_daily_lock(): """ Raises CantGetLockError if the dinstall daily.lock exists. """ cnf = Config() try: lockfile = cnf.get("Process-New::DinstallLockFile", os.path.join(cnf['Dir::Lock'], 'processnew.lock')) os.open(lockfile, os.O_RDONLY | os.O_CREAT | os.O_EXCL) except OSError as e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: raise CantGetLockError os.unlink(lockfile)
def do_pkg(upload, session): # Try to get an included dsc dsc = upload.source cnf = Config() group = cnf.get('Dinstall::UnprivGroup') or None try: with lock_package(upload.changes.source), \ UploadCopy(upload, group=group) as upload_copy: handler = PolicyQueueUploadHandler(upload, session) if handler.get_action() is not None: print("PENDING %s\n" % handler.get_action()) return do_new(upload, upload_copy, handler, session) except AlreadyLockedError as e: print("Seems to be locked by %s already, skipping..." % (e))
def do_lintian(filename): cnf = Config() cmd = [] user = cnf.get('Dinstall::UnprivUser') or None if user is not None: cmd.extend(['sudo', '-H', '-u', user]) color = 'always' if use_html: color = 'html' cmd.extend( ['lintian', '--show-overrides', '--color', color, "--", filename]) try: return do_command(cmd, escaped=True) except OSError as e: return (colour_output("Running lintian failed: %s" % (e), "error"))
def lock_package(package): """ Lock C{package} so that noone else jumps in processing it. @type package: string @param package: source package name to lock """ cnf = Config() path = os.path.join(cnf.get("Process-New::LockDir", cnf['Dir::Lock']), package) try: fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDONLY) except OSError as e: if e.errno == errno.EEXIST or e.errno == errno.EACCES: user = pwd.getpwuid(os.stat(path)[stat.ST_UID])[4].split(',')[0].replace('.', '') raise AlreadyLockedError(user) try: yield fd finally: os.unlink(path)
def do_pkg(upload, session): # Try to get an included dsc dsc = upload.source cnf = Config() group = cnf.get('Dinstall::UnprivGroup') or None #bcc = "X-DAK: dak process-new" #if cnf.has_key("Dinstall::Bcc"): # u.Subst["__BCC__"] = bcc + "\nBcc: %s" % (cnf["Dinstall::Bcc"]) #else: # u.Subst["__BCC__"] = bcc try: with lock_package(upload.changes.source): with UploadCopy(upload, group=group) as upload_copy: handler = PolicyQueueUploadHandler(upload, session) if handler.get_action() is not None: print "PENDING %s\n" % handler.get_action() return do_new(upload, upload_copy, handler, session) except AlreadyLockedError as e: print "Seems to be locked by %s already, skipping..." % (e)
def action_dm_remove(self, fingerprint, section, session): self._action_dm_admin_common(fingerprint, section, session) cnf = Config() acl_name = cnf.get('Command::DM::ACL', 'dm') acl = session.query(ACL).filter_by(name=acl_name).one() fpr_hash = section['Fingerprint'].translate(None, ' ') fpr = session.query(Fingerprint).filter_by(fingerprint=fpr_hash).first() if fpr is None: self.result.append('Unknown fingerprint: {0}\nNo action taken.'.format(fpr_hash)) return self.log.log(['dm-remove', fpr.fingerprint]) count = 0 for entry in session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr): self.log.log(['dm-remove', fpr.fingerprint, 'source={0}'.format(entry.source)]) count += 1 session.delete(entry) self.result.append('Removed: {0}.\n{1} acl entries removed.'.format(fpr.fingerprint, count)) session.commit()
def check(self, upload): changes = upload.changes # Only check sourceful uploads. if changes.source is None: return True # Only check uploads to unstable or experimental. if 'unstable' not in changes.distributions and 'experimental' not in changes.distributions: return True cnf = Config() if 'Dinstall::LintianTags' not in cnf: return True tagfile = cnf['Dinstall::LintianTags'] with open(tagfile, 'r') as sourcefile: sourcecontent = sourcefile.read() try: lintiantags = yaml.safe_load(sourcecontent)['lintian'] except yaml.YAMLError as msg: raise Exception( 'Could not read lintian tags file {0}, YAML error: {1}'.format( tagfile, msg)) with tempfile.NamedTemporaryFile(mode="w+t") as temptagfile: os.fchmod(temptagfile.fileno(), 0o644) for tags in six.itervalues(lintiantags): for tag in tags: print(tag, file=temptagfile) temptagfile.flush() changespath = os.path.join(upload.directory, changes.filename) cmd = [] user = cnf.get('Dinstall::UnprivUser') or None if user is not None: cmd.extend(['sudo', '-H', '-u', user]) cmd.extend([ '/usr/bin/lintian', '--show-overrides', '--tags-from-file', temptagfile.name, changespath ]) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output_raw = process.communicate()[0] output = six.ensure_text(output_raw) result = process.returncode if result == 2: utils.warn("lintian failed for %s [return code: %s]." % (changespath, result)) utils.warn( utils.prefix_multi_line_string(output, " [possible output:] ")) parsed_tags = lintian.parse_lintian_output(output) rejects = list( lintian.generate_reject_messages(parsed_tags, lintiantags)) if len(rejects) != 0: raise Reject('\n'.join(rejects)) return True
def clean(now_date, archives, max_delete, session): cnf = Config() count = 0 size = 0 Logger.log(["Cleaning out packages..."]) morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool') # Build directory as morguedir/morguesubdir/year/month/day dest = os.path.join(morguedir, morguesubdir, str(now_date.year), '%.2d' % now_date.month, '%.2d' % now_date.day) if not Options["No-Action"] and not os.path.exists(dest): os.makedirs(dest) # Delete from source Logger.log(["Deleting from source table..."]) q = session.execute(""" WITH deleted_sources AS ( DELETE FROM source USING files f WHERE source.file = f.id AND NOT EXISTS (SELECT 1 FROM files_archive_map af JOIN archive_delete_date ad ON af.archive_id = ad.archive_id WHERE af.file_id = source.file AND (af.last_used IS NULL OR af.last_used > ad.delete_date)) RETURNING source.id AS id, f.filename AS filename ), deleted_dsc_files AS ( DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources) RETURNING df.file AS file_id ), now_unused_source_files AS ( UPDATE files_archive_map af SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc. WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files) AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id) ) SELECT filename FROM deleted_sources""") for s in q: Logger.log(["delete source", s[0]]) if not Options["No-Action"]: session.commit() # Delete files from the pool old_files = session.query(ArchiveFile).filter( sql.text( 'files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)' )).join(Archive) if max_delete is not None: old_files = old_files.limit(max_delete) Logger.log(["Limiting removals to %d" % max_delete]) if archives is not None: archive_ids = [a.archive_id for a in archives] old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids)) for af in old_files: filename = af.path try: st = os.lstat(filename) except FileNotFoundError: Logger.log(["database referred to non-existing file", filename]) session.delete(af) continue Logger.log(["delete archive file", filename]) if stat.S_ISLNK(st.st_mode): count += 1 Logger.log(["delete symlink", filename]) if not Options["No-Action"]: os.unlink(filename) session.delete(af) elif stat.S_ISREG(st.st_mode): size += st.st_size count += 1 dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.lexists(dest_filename): dest_filename = utils.find_next_free(dest_filename) if not Options["No-Action"]: if af.archive.use_morgue: Logger.log(["move to morgue", filename, dest_filename]) utils.move(filename, dest_filename) else: Logger.log(["removed file", filename]) os.unlink(filename) session.delete(af) else: utils.fubar("%s is neither symlink nor file?!" % (filename)) if count > 0: Logger.log(["total", count, utils.size_type(size)]) # Delete entries in files no longer referenced by any archive query = """ DELETE FROM files f WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id) """ session.execute(query) if not Options["No-Action"]: session.commit()
def update_db(self): # Ok, try and find the configuration table print("Determining dak database revision ...") cnf = Config() logger = Logger('update-db') modules = [] try: # Build a connect string if "DB::Service" in cnf: connect_str = "service=%s" % cnf["DB::Service"] else: connect_str = "dbname=%s" % (cnf["DB::Name"]) if "DB::Host" in cnf and cnf["DB::Host"] != '': connect_str += " host=%s" % (cnf["DB::Host"]) if "DB::Port" in cnf and cnf["DB::Port"] != '-1': connect_str += " port=%d" % (int(cnf["DB::Port"])) self.db = psycopg2.connect(connect_str) db_role = cnf.get("DB::Role") if db_role: self.db.cursor().execute('SET ROLE "{}"'.format(db_role)) except Exception as e: print("FATAL: Failed connect to database (%s)" % str(e)) sys.exit(1) database_revision = int(self.get_db_rev()) logger.log( ['transaction id before update: %s' % self.get_transaction_id()]) if database_revision == -1: print("dak database schema predates update-db.") print("") print( "This script will attempt to upgrade it to the lastest, but may fail." ) print( "Please make sure you have a database backup handy. If you don't, press Ctrl-C now!" ) print("") print("Continuing in five seconds ...") time.sleep(5) print("") print("Attempting to upgrade pre-zero database to zero") self.update_db_to_zero() database_revision = 0 dbfiles = glob( os.path.join(os.path.dirname(__file__), 'dakdb/update*.py')) required_database_schema = max( map(int, findall('update(\d+).py', " ".join(dbfiles)))) print("dak database schema at %d" % database_revision) print("dak version requires schema %d" % required_database_schema) if database_revision < required_database_schema: print("\nUpdates to be applied:") for i in range(database_revision, required_database_schema): i += 1 dakdb = __import__("dakdb", globals(), locals(), ['update' + str(i)]) update_module = getattr(dakdb, "update" + str(i)) print( "Update %d: %s" % (i, next(s for s in update_module.__doc__.split("\n") if s))) modules.append((update_module, i)) if not Config().find_b("Update-DB::Options::Yes", False): prompt = "\nUpdate database? (y/N) " answer = utils.our_raw_input(prompt) if answer.upper() != 'Y': sys.exit(0) else: print("no updates required") logger.log(["no updates required"]) sys.exit(0) for module in modules: (update_module, i) = module try: update_module.do_update(self) message = "updated database schema from %d to %d" % ( database_revision, i) print(message) logger.log([message]) except DBUpdateError as e: # Seems the update did not work. print("Was unable to update database schema from %d to %d." % (database_revision, i)) print("The error message received was %s" % (e)) logger.log(["DB Schema upgrade failed"]) logger.close() utils.fubar("DB Schema upgrade failed") database_revision += 1 logger.close()
def main (): global suite, suite_id, source_binaries, source_versions cnf = Config() Arguments = [('h',"help","Cruft-Report::Options::Help"), ('m',"mode","Cruft-Report::Options::Mode", "HasArg"), ('R',"rdep-check", "Cruft-Report::Options::Rdep-Check"), ('s',"suite","Cruft-Report::Options::Suite","HasArg"), ('w',"wanna-build-dump","Cruft-Report::Options::Wanna-Build-Dump","HasArg")] for i in [ "help", "Rdep-Check" ]: if not cnf.has_key("Cruft-Report::Options::%s" % (i)): cnf["Cruft-Report::Options::%s" % (i)] = "" cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") if not cnf.has_key("Cruft-Report::Options::Mode"): cnf["Cruft-Report::Options::Mode"] = "daily" if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"): cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Cruft-Report::Options") if Options["Help"]: usage() if Options["Rdep-Check"]: rdeps = True else: rdeps = False # Set up checks based on mode if Options["Mode"] == "daily": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu" ] elif Options["Mode"] == "full": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais" ] elif Options["Mode"] == "bdo": checks = [ "nbs", "obsolete source" ] else: utils.warn("%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) usage(1) session = DBConn().session() bin_pkgs = {} src_pkgs = {} bin2source = {} bins_in_suite = {} nbs = {} source_versions = {} anais_output = "" nfu_packages = {} suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() if "obsolete source" in checks: report_obsolete_source(suite_name, session) if "nbs" in checks: reportAllNBS(suite_name, suite_id, session, rdeps) if "outdated non-free" in checks: report_outdated_nonfree(suite_name, session, rdeps) bin_not_built = {} if "bnb" in checks: bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files components = get_component_names(session) for component in components: filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path, suite_name, component) # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance... (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) sources = utils.open_file(temp_filename) Sources = apt_pkg.TagFile(sources) while Sources.step(): source = Sources.section.find('Package') source_version = Sources.section.find('Version') architecture = Sources.section.find('Architecture') binaries = Sources.section.find('Binary') binaries_list = [ i.strip() for i in binaries.split(',') ] if "bnb" in checks: # Check for binaries not built on any architecture. for binary in binaries_list: if not bins_in_suite.has_key(binary): bin_not_built.setdefault(source, {}) bin_not_built[source][binary] = "" if "anais" in checks: anais_output += do_anais(architecture, binaries_list, source, session) # build indices for checking "no source" later source_index = component + '/' + source src_pkgs[source] = source_index for binary in binaries_list: bin_pkgs[binary] = source source_binaries[source] = binaries source_versions[source] = source_version sources.close() os.unlink(temp_filename) # Checks based on the Packages files check_components = components[:] if suite_name != "experimental": check_components.append('main/debian-installer'); for component in check_components: architectures = [ a.arch_string for a in get_suite_architectures(suite_name, skipsrc=True, skipall=True, session=session) ] for architecture in architectures: if component == 'main/debian-installer' and re.match("kfreebsd", architecture): continue filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % (suite.archive.path, suite_name, component, architecture) # apt_pkg.TagFile needs a real file handle (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) if "nfu" in checks: nfu_packages.setdefault(architecture,[]) nfu_entries = parse_nfu(architecture) packages = utils.open_file(temp_filename) Packages = apt_pkg.TagFile(packages) while Packages.step(): package = Packages.section.find('Package') source = Packages.section.find('Source', "") version = Packages.section.find('Version') if source == "": source = package if bin2source.has_key(package) and \ apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version bin2source[package]["source"] = source else: bin2source[package] = {} bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if not bin_pkgs.has_key(package): nbs.setdefault(source,{}) nbs[source].setdefault(package, {}) nbs[source][package][version] = "" else: if "nfu" in checks: if package in nfu_entries and \ version != source_versions[source]: # only suggest to remove out-of-date packages nfu_packages[architecture].append((package,version,source_versions[source])) packages.close() os.unlink(temp_filename) # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} for source in nbs.keys(): for package in nbs[source].keys(): versions = nbs[source][package].keys() versions.sort(apt_pkg.version_compare) latest_version = versions.pop() source_version = source_versions.get(source,"0") if apt_pkg.version_compare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: do_newer_version('unstable', 'experimental', 'NVIU', session) if "nvit" in checks: do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) ### if Options["Mode"] == "full": print "="*75 print if "nfu" in checks: do_nfu(nfu_packages) if "bnb" in checks: print "Unbuilt binary packages" print "-----------------------" print keys = bin_not_built.keys() keys.sort() for source in keys: binaries = bin_not_built[source].keys() binaries.sort() print " o %s: %s" % (source, ", ".join(binaries)) print if "bms" in checks: report_multiple_source(suite) if "anais" in checks: print "Architecture Not Allowed In Source" print "----------------------------------" print anais_output print if "dubious nbs" in checks: do_dubious_nbs(dubious_nbs)
def action_dm(self, fingerprint, section, session): cnf = Config() if ( "Command::DM::AdminKeyrings" not in cnf or "Command::DM::ACL" not in cnf or "Command::DM::Keyrings" not in cnf ): raise CommandError("DM command is not configured for this archive.") allowed_keyrings = cnf.value_list("Command::DM::AdminKeyrings") if fingerprint.keyring.keyring_name not in allowed_keyrings: raise CommandError("Key {0} is not allowed to set DM".format(fingerprint.fingerprint)) acl_name = cnf.get("Command::DM::ACL", "dm") acl = session.query(ACL).filter_by(name=acl_name).one() fpr_hash = section["Fingerprint"].translate(None, " ") fpr = session.query(Fingerprint).filter_by(fingerprint=fpr_hash).first() if fpr is None: raise CommandError("Unknown fingerprint {0}".format(fpr_hash)) if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list("Command::DM::Keyrings"): raise CommandError("Key {0} is not in DM keyring.".format(fpr.fingerprint)) addresses = gpg_get_key_addresses(fpr.fingerprint) if len(addresses) > 0: self.cc.append(addresses[0]) self.log.log(["dm", "fingerprint", fpr.fingerprint]) self.result.append("Fingerprint: {0}".format(fpr.fingerprint)) if len(addresses) > 0: self.log.log(["dm", "uid", addresses[0]]) self.result.append("Uid: {0}".format(addresses[0])) for source in self._split_packages(section.get("Allow", "")): # Check for existance of source package to catch typos if session.query(DBSource).filter_by(source=source).first() is None: raise CommandError("Tried to grant permissions for unknown source package: {0}".format(source)) if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None: aps = ACLPerSource() aps.acl = acl aps.fingerprint = fpr aps.source = source aps.created_by = fingerprint aps.reason = section.get("Reason") session.add(aps) self.log.log(["dm", "allow", fpr.fingerprint, source]) self.result.append("Allowed: {0}".format(source)) else: self.result.append("Already-Allowed: {0}".format(source)) session.flush() for source in self._split_packages(section.get("Deny", "")): count = session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete() if count == 0: raise CommandError( "Tried to remove upload permissions for package {0}, " "but no upload permissions were granted before.".format(source) ) self.log.log(["dm", "deny", fpr.fingerprint, source]) self.result.append("Denied: {0}".format(source)) session.commit()
def main(): cnf = Config() Arguments = [ ('n', "no-action", "Import-Users-From-Passwd::Options::No-Action"), ('q', "quiet", "Import-Users-From-Passwd::Options::Quiet"), ('v', "verbose", "Import-Users-From-Passwd::Options::Verbose"), ('h', "help", "Import-Users-From-Passwd::Options::Help") ] for i in ["no-action", "quiet", "verbose", "help"]: key = "Import-Users-From-Passwd::Options::%s" % i if key not in cnf: cnf[key] = "" arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Import-Users-From-Passwd::Options") if Options["Help"]: usage() elif arguments: utils.warn( "dak import-users-from-passwd takes no non-option arguments.") usage(1) session = DBConn().session() valid_gid = cnf.get("Import-Users-From-Passwd::ValidGID", "") if valid_gid: debiangrp = grp.getgrnam(valid_gid).gr_mem else: debiangrp = [] passwd_unames = {} for entry in pwd.getpwall(): uname = entry[0] if uname not in debiangrp: if Options["Verbose"]: print("Skipping %s (Not in group %s)." % (uname, valid_gid)) continue passwd_unames[uname] = "" postgres_unames = {} q = session.execute("SELECT usename FROM pg_user") for i in q.fetchall(): uname = i[0] postgres_unames[uname] = "" known_postgres_unames = {} for i in cnf.get("Import-Users-From-Passwd::KnownPostgres", "").split(","): uname = i.strip() known_postgres_unames[uname] = "" for uname in sorted(postgres_unames): if uname not in passwd_unames and uname not in known_postgres_unames: print( "I: Deleting %s from Postgres, no longer in passwd or list of known Postgres users" % (uname)) q = session.execute('DROP USER "%s"' % (uname)) safe_name = re.compile('^[A-Za-z0-9]+$') for uname in sorted(passwd_unames): if uname not in postgres_unames: if not Options["Quiet"]: print("Creating %s user in Postgres." % (uname)) if not Options["No-Action"]: if safe_name.match(uname): # NB: I never figured out how to use a bind parameter for this query # XXX: Fix this as it looks like a potential SQL injection attack to me # (hence the safe_name match we do) try: q = session.execute('CREATE USER "%s"' % (uname)) session.commit() except Exception as e: utils.warn("Could not create user %s (%s)" % (uname, str(e))) session.rollback() else: print("NOT CREATING USER %s. Doesn't match safety regex" % uname) session.commit()
def clean(now_date, archives, max_delete, session): cnf = Config() count = 0 size = 0 Logger.log(["Cleaning out packages..."]) morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool') # Build directory as morguedir/morguesubdir/year/month/day dest = os.path.join(morguedir, morguesubdir, str(now_date.year), '%.2d' % now_date.month, '%.2d' % now_date.day) if not Options["No-Action"] and not os.path.exists(dest): os.makedirs(dest) # Delete from source Logger.log(["Deleting from source table..."]) q = session.execute(""" WITH deleted_sources AS ( DELETE FROM source USING files f WHERE source.file = f.id AND NOT EXISTS (SELECT 1 FROM files_archive_map af JOIN archive_delete_date ad ON af.archive_id = ad.archive_id WHERE af.file_id = source.file AND (af.last_used IS NULL OR af.last_used > ad.delete_date)) RETURNING source.id AS id, f.filename AS filename ), deleted_dsc_files AS ( DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources) RETURNING df.file AS file_id ), now_unused_source_files AS ( UPDATE files_archive_map af SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc. WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files) AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id) ) SELECT filename FROM deleted_sources""") for s in q: Logger.log(["delete source", s[0]]) if not Options["No-Action"]: session.commit() # Delete files from the pool old_files = session.query(ArchiveFile).filter('files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)').join(Archive) if max_delete is not None: old_files = old_files.limit(max_delete) Logger.log(["Limiting removals to %d" % max_delete]) if archives is not None: archive_ids = [ a.archive_id for a in archives ] old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids)) for af in old_files: filename = af.path if not os.path.exists(filename): Logger.log(["database referred to non-existing file", af.path]) session.delete(af) continue Logger.log(["delete archive file", filename]) if os.path.isfile(filename): if os.path.islink(filename): count += 1 Logger.log(["delete symlink", filename]) if not Options["No-Action"]: os.unlink(filename) else: size += os.stat(filename)[stat.ST_SIZE] count += 1 dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.lexists(dest_filename): dest_filename = utils.find_next_free(dest_filename) if not Options["No-Action"]: if af.archive.use_morgue: Logger.log(["move to morgue", filename, dest_filename]) utils.move(filename, dest_filename) else: Logger.log(["removed file", filename]) os.unlink(filename) if not Options["No-Action"]: session.delete(af) session.commit() else: utils.fubar("%s is neither symlink nor file?!" % (filename)) if count > 0: Logger.log(["total", count, utils.size_type(size)]) # Delete entries in files no longer referenced by any archive query = """ DELETE FROM files f WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id) """ session.execute(query) if not Options["No-Action"]: session.commit()
def comment_accept(upload, srcqueue, comments, transaction): for byhand in upload.byhand: path = os.path.join(srcqueue.path, byhand.filename) if os.path.exists(path): raise Exception( 'E: cannot ACCEPT upload with unprocessed byhand file {0}'. format(byhand.filename)) cnf = Config() fs = transaction.fs session = transaction.session changesname = upload.changes.changesname allow_tainted = srcqueue.suite.archive.tainted # We need overrides to get the target component overridesuite = upload.target_suite if overridesuite.overridesuite is not None: overridesuite = session.query(Suite).filter_by( suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \ .join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \ .join(Component).one() return override.component def source_component_func(db_source): override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ .join(Component).one() return override.component all_target_suites = [upload.target_suite] all_target_suites.extend( [q.suite for q in upload.target_suite.copy_queues]) for suite in all_target_suites: if upload.source is not None: transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted) for db_binary in upload.binaries: # build queues may miss the source package if this is a binary-only upload if suite != upload.target_suite: transaction.copy_source(db_binary.source, suite, source_component_func( db_binary.source), allow_tainted=allow_tainted) transaction.copy_binary( db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive]) # Copy .changes if needed if upload.target_suite.copychanges: src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) # Copy upload to Process-Policy::CopyDir # Used on security.d.o to sync accepted packages to ftp-master, but this # should eventually be replaced by something else. copydir = cnf.get('Process-Policy::CopyDir') or None if copydir is not None: mode = upload.target_suite.archive.mode if upload.source is not None: for f in [df.poolfile for df in upload.source.srcfiles]: dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for db_binary in upload.binaries: f = db_binary.poolfile dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(copydir, upload.changes.changesname) if not os.path.exists(dst): fs.copy(src, dst, mode=mode) if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(upload.source.source, upload.source.version, urgency) print " ACCEPT" if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done src = os.path.join(upload.policy_queue.path, upload.changes.changesname) now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) dst = os.path.join(donedir, upload.changes.changesname) dst = utils.find_next_free(dst) fs.copy(src, dst, mode=0o644) remove_upload(upload, transaction)
def generate_release_files(self): """ Generate Release files for the given suite @type suite: string @param suite: Suite name """ suite = self.suite session = object_session(suite) architectures = get_suite_architectures(suite.suite_name, skipall=True, skipsrc=True, session=session) # Attribs contains a tuple of field names and the database names to use to # fill them in attribs = (('Origin', 'origin'), ('Label', 'label'), ('Suite', 'release_suite_output'), ('Version', 'version'), ('Codename', 'codename'), ('Changelogs', 'changelog_url'), ) # A "Sub" Release file has slightly different fields subattribs = (('Archive', 'suite_name'), ('Origin', 'origin'), ('Label', 'label'), ('Version', 'version')) # Boolean stuff. If we find it true in database, write out "yes" into the release file boolattrs = (('NotAutomatic', 'notautomatic'), ('ButAutomaticUpgrades', 'butautomaticupgrades'), ('Acquire-By-Hash', 'byhash'), ) cnf = Config() cnf_suite_suffix = cnf.get("Dinstall::SuiteSuffix", "").rstrip("/") suite_suffix = utils.suite_suffix(suite.suite_name) self.create_output_directories() self.create_release_symlinks() outfile = os.path.join(self.suite_release_path(), "Release") out = open(outfile + ".new", "w") for key, dbfield in attribs: # Hack to skip NULL Version fields as we used to do this # We should probably just always ignore anything which is None if key in ("Version", "Changelogs") and getattr(suite, dbfield) is None: continue out.write("%s: %s\n" % (key, getattr(suite, dbfield))) out.write("Date: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time())))) if suite.validtime: validtime = float(suite.validtime) out.write("Valid-Until: %s\n" % (time.strftime("%a, %d %b %Y %H:%M:%S UTC", time.gmtime(time.time() + validtime)))) for key, dbfield in boolattrs: if getattr(suite, dbfield, False): out.write("%s: yes\n" % (key)) out.write("Architectures: %s\n" % (" ".join([a.arch_string for a in architectures]))) components = [c.component_name for c in suite.components] out.write("Components: %s\n" % (" ".join(components))) # For exact compatibility with old g-r, write out Description here instead # of with the rest of the DB fields above if getattr(suite, 'description') is not None: out.write("Description: %s\n" % suite.description) for comp in components: for dirpath, dirnames, filenames in os.walk(os.path.join(self.suite_path(), comp), topdown=True): if not re_gensubrelease.match(dirpath): continue subfile = os.path.join(dirpath, "Release") subrel = open(subfile + '.new', "w") for key, dbfield in subattribs: if getattr(suite, dbfield) is not None: subrel.write("%s: %s\n" % (key, getattr(suite, dbfield))) for key, dbfield in boolattrs: if getattr(suite, dbfield, False): subrel.write("%s: yes\n" % (key)) subrel.write("Component: %s%s\n" % (suite_suffix, comp)) # Urgh, but until we have all the suite/component/arch stuff in the DB, # this'll have to do arch = os.path.split(dirpath)[-1] if arch.startswith('binary-'): arch = arch[7:] subrel.write("Architecture: %s\n" % (arch)) subrel.close() os.rename(subfile + '.new', subfile) # Now that we have done the groundwork, we want to get off and add the files with # their checksums to the main Release file oldcwd = os.getcwd() os.chdir(self.suite_path()) hashes = [x for x in RELEASE_HASHES if x.db_name in suite.checksums] fileinfo = {} fileinfo_byhash = {} uncompnotseen = {} for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): # SuiteSuffix deprecation: # components on security-master are updates/{main,contrib,non-free}, but # we want dists/${suite}/main. Until we can rename the components, # we cheat by having an updates -> . symlink. This should not be visited. if cnf_suite_suffix: path = os.path.join(dirpath, cnf_suite_suffix) try: target = os.readlink(path) if target == ".": dirnames.remove(cnf_suite_suffix) except (OSError, ValueError): pass for entry in filenames: if dirpath == '.' and entry in ["Release", "Release.gpg", "InRelease"]: continue filename = os.path.join(dirpath.lstrip('./'), entry) if re_includeinrelease_byhash.match(entry): fileinfo[filename] = fileinfo_byhash[filename] = {} elif re_includeinrelease_plain.match(entry): fileinfo[filename] = {} # Skip things we don't want to include else: continue contents = open(filename, 'r').read() # If we find a file for which we have a compressed version and # haven't yet seen the uncompressed one, store the possibility # for future use if entry.endswith(".gz") and filename[:-3] not in uncompnotseen: uncompnotseen[filename[:-3]] = (gzip.GzipFile, filename) elif entry.endswith(".bz2") and filename[:-4] not in uncompnotseen: uncompnotseen[filename[:-4]] = (bz2.BZ2File, filename) elif entry.endswith(".xz") and filename[:-3] not in uncompnotseen: uncompnotseen[filename[:-3]] = (XzFile, filename) fileinfo[filename]['len'] = len(contents) for hf in hashes: fileinfo[filename][hf.release_field] = hf.func(contents) for filename, comp in uncompnotseen.items(): # If we've already seen the uncompressed file, we don't # need to do anything again if filename in fileinfo: continue fileinfo[filename] = {} # File handler is comp[0], filename of compressed file is comp[1] contents = comp[0](comp[1], 'r').read() fileinfo[filename]['len'] = len(contents) for hf in hashes: fileinfo[filename][hf.release_field] = hf.func(contents) for field in sorted(h.release_field for h in hashes): out.write('%s:\n' % field) for filename in sorted(fileinfo.keys()): out.write(" %s %8d %s\n" % (fileinfo[filename][field], fileinfo[filename]['len'], filename)) out.close() os.rename(outfile + '.new', outfile) self._update_hashfile_table(session, fileinfo_byhash, hashes) self._make_byhash_links(fileinfo_byhash, hashes) self._make_byhash_base_symlink(fileinfo_byhash, hashes) sign_release_dir(suite, os.path.dirname(outfile)) os.chdir(oldcwd) return
def main(): global suite, suite_id, source_binaries, source_versions cnf = Config() Arguments = [('h', "help", "Cruft-Report::Options::Help"), ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"), ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"), ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"), ('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg")] for i in ["help", "Rdep-Check"]: if not cnf.has_key("Cruft-Report::Options::%s" % (i)): cnf["Cruft-Report::Options::%s" % (i)] = "" cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") if not cnf.has_key("Cruft-Report::Options::Mode"): cnf["Cruft-Report::Options::Mode"] = "daily" if not cnf.has_key("Cruft-Report::Options::Wanna-Build-Dump"): cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Cruft-Report::Options") if Options["Help"]: usage() if Options["Rdep-Check"]: rdeps = True else: rdeps = False # Set up checks based on mode if Options["Mode"] == "daily": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu" ] elif Options["Mode"] == "full": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "dubious nbs", "bnb", "bms", "anais" ] elif Options["Mode"] == "bdo": checks = ["nbs", "obsolete source"] else: utils.warn( "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) usage(1) session = DBConn().session() bin_pkgs = {} src_pkgs = {} bin2source = {} bins_in_suite = {} nbs = {} source_versions = {} anais_output = "" nfu_packages = {} suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() if "obsolete source" in checks: report_obsolete_source(suite_name, session) if "nbs" in checks: reportAllNBS(suite_name, suite_id, session, rdeps) if "outdated non-free" in checks: report_outdated_nonfree(suite_name, session, rdeps) bin_not_built = {} if "bnb" in checks: bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files components = get_component_names(session) for component in components: filename = "%s/dists/%s/%s/source/Sources.gz" % (suite.archive.path, suite_name, component) # apt_pkg.TagFile needs a real file handle and can't handle a GzipFile instance... (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput("gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) sources = utils.open_file(temp_filename) Sources = apt_pkg.TagFile(sources) while Sources.step(): source = Sources.section.find('Package') source_version = Sources.section.find('Version') architecture = Sources.section.find('Architecture') binaries = Sources.section.find('Binary') binaries_list = [i.strip() for i in binaries.split(',')] if "bnb" in checks: # Check for binaries not built on any architecture. for binary in binaries_list: if not bins_in_suite.has_key(binary): bin_not_built.setdefault(source, {}) bin_not_built[source][binary] = "" if "anais" in checks: anais_output += do_anais(architecture, binaries_list, source, session) # build indices for checking "no source" later source_index = component + '/' + source src_pkgs[source] = source_index for binary in binaries_list: bin_pkgs[binary] = source source_binaries[source] = binaries source_versions[source] = source_version sources.close() os.unlink(temp_filename) # Checks based on the Packages files check_components = components[:] if suite_name != "staging": check_components.append('main/debian-installer') for component in check_components: architectures = [ a.arch_string for a in get_suite_architectures( suite_name, skipsrc=True, skipall=True, session=session) ] for architecture in architectures: if component == 'main/debian-installer' and re.match( "kfreebsd", architecture): continue filename = "%s/dists/%s/%s/binary-%s/Packages.gz" % ( suite.archive.path, suite_name, component, architecture) # apt_pkg.TagFile needs a real file handle (fd, temp_filename) = utils.temp_filename() (result, output) = commands.getstatusoutput( "gunzip -c %s > %s" % (filename, temp_filename)) if (result != 0): sys.stderr.write("Gunzip invocation failed!\n%s\n" % (output)) sys.exit(result) if "nfu" in checks: nfu_packages.setdefault(architecture, []) nfu_entries = parse_nfu(architecture) packages = utils.open_file(temp_filename) Packages = apt_pkg.TagFile(packages) while Packages.step(): package = Packages.section.find('Package') source = Packages.section.find('Source', "") version = Packages.section.find('Version') if source == "": source = package if bin2source.has_key(package) and \ apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version bin2source[package]["source"] = source else: bin2source[package] = {} bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if not bin_pkgs.has_key(package): nbs.setdefault(source, {}) nbs[source].setdefault(package, {}) nbs[source][package][version] = "" else: if "nfu" in checks: if package in nfu_entries and \ version != source_versions[source]: # only suggest to remove out-of-date packages nfu_packages[architecture].append( (package, version, source_versions[source])) packages.close() os.unlink(temp_filename) # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} for source in nbs.keys(): for package in nbs[source].keys(): versions = nbs[source][package].keys() versions.sort(apt_pkg.version_compare) latest_version = versions.pop() source_version = source_versions.get(source, "0") if apt_pkg.version_compare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: do_newer_version('chromodoris', 'staging', 'NVIU', session) # FIXME: Not used in Tanglu #if "nvit" in checks: # do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) ### if Options["Mode"] == "full": print "=" * 75 print if "nfu" in checks: do_nfu(nfu_packages) if "bnb" in checks: print "Unbuilt binary packages" print "-----------------------" print keys = bin_not_built.keys() keys.sort() for source in keys: binaries = bin_not_built[source].keys() binaries.sort() print " o %s: %s" % (source, ", ".join(binaries)) print if "bms" in checks: report_multiple_source(suite) if "anais" in checks: print "Architecture Not Allowed In Source" print "----------------------------------" print anais_output print if "dubious nbs" in checks: do_dubious_nbs(dubious_nbs)
def clean(now_date, delete_date, max_delete, session): cnf = Config() count = 0 size = 0 print "Cleaning out packages..." morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool') # Build directory as morguedir/morguesubdir/year/month/day dest = os.path.join(morguedir, morguesubdir, str(now_date.year), '%.2d' % now_date.month, '%.2d' % now_date.day) if not Options["No-Action"] and not os.path.exists(dest): os.makedirs(dest) # Delete from source print "Deleting from source table... " q = session.execute(""" SELECT s.id, f.filename FROM source s, files f WHERE f.last_used <= :deletedate AND s.file = f.id AND s.id NOT IN (SELECT src_id FROM extra_src_references)""", {'deletedate': delete_date}) for s in q.fetchall(): Logger.log(["delete source", s[1], s[0]]) if not Options["No-Action"]: session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]}) session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]}) if not Options["No-Action"]: session.commit() # Delete files from the pool old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date) if max_delete is not None: old_files = old_files.limit(max_delete) print "Limiting removals to %d" % max_delete for pf in old_files: filename = os.path.join(pf.location.path, pf.filename) if not os.path.exists(filename): utils.warn("can not find '%s'." % (filename)) continue Logger.log(["delete pool file", filename]) if os.path.isfile(filename): if os.path.islink(filename): count += 1 Logger.log(["delete symlink", filename]) if not Options["No-Action"]: os.unlink(filename) else: size += os.stat(filename)[stat.ST_SIZE] count += 1 dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.exists(dest_filename): dest_filename = utils.find_next_free(dest_filename) Logger.log(["move to morgue", filename, dest_filename]) if not Options["No-Action"]: utils.move(filename, dest_filename) if not Options["No-Action"]: session.delete(pf) session.commit() else: utils.fubar("%s is neither symlink nor file?!" % (filename)) if count > 0: Logger.log(["total", count, utils.size_type(size)]) print "Cleaned %d files, %s." % (count, utils.size_type(size))
def comment_accept(upload, srcqueue, comments, transaction): for byhand in upload.byhand: path = os.path.join(srcqueue.path, byhand.filename) if os.path.exists(path): raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename)) cnf = Config() fs = transaction.fs session = transaction.session changesname = upload.changes.changesname allow_tainted = srcqueue.suite.archive.tainted # We need overrides to get the target component overridesuite = upload.target_suite if overridesuite.overridesuite is not None: overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): section = db_binary.proxy['Section'] component_name = 'main' if section.find('/') != -1: component_name = section.split('/', 1)[0] return get_mapped_component(component_name, session=session) def source_component_func(db_source): package_list = PackageList(db_source.proxy) component = source_component_from_package_list(package_list, upload.target_suite) if component is not None: return get_mapped_component(component.component_name, session=session) # Fallback for packages without Package-List field query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ .join(Component) return query.one().component all_target_suites = [upload.target_suite] all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) for suite in all_target_suites: if upload.source is not None: transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted) for db_binary in upload.binaries: # build queues may miss the source package if this is a binary-only upload if suite != upload.target_suite: transaction.copy_source(db_binary.source, suite, source_component_func(db_binary.source), allow_tainted=allow_tainted) transaction.copy_binary(db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive]) # Copy .changes if needed if upload.target_suite.copychanges: src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) # Copy upload to Process-Policy::CopyDir # Used on security.d.o to sync accepted packages to ftp-master, but this # should eventually be replaced by something else. copydir = cnf.get('Process-Policy::CopyDir') or None if copydir is not None: mode = upload.target_suite.archive.mode if upload.source is not None: for f in [ df.poolfile for df in upload.source.srcfiles ]: dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for db_binary in upload.binaries: f = db_binary.poolfile dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(copydir, upload.changes.changesname) if not os.path.exists(dst): fs.copy(src, dst, mode=mode) if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(upload.source.source, upload.source.version, urgency) print " ACCEPT" if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done src = os.path.join(upload.policy_queue.path, upload.changes.changesname) now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) dst = os.path.join(donedir, upload.changes.changesname) dst = utils.find_next_free(dst) fs.copy(src, dst, mode=0o644) remove_upload(upload, transaction)
def main (): cnf = Config() Arguments = [('n', "no-action", "Import-Users-From-Passwd::Options::No-Action"), ('q', "quiet", "Import-Users-From-Passwd::Options::Quiet"), ('v', "verbose", "Import-Users-From-Passwd::Options::Verbose"), ('h', "help", "Import-Users-From-Passwd::Options::Help")] for i in [ "no-action", "quiet", "verbose", "help" ]: if not cnf.has_key("Import-Users-From-Passwd::Options::%s" % (i)): cnf["Import-Users-From-Passwd::Options::%s" % (i)] = "" arguments = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Import-Users-From-Passwd::Options") if Options["Help"]: usage() elif arguments: utils.warn("dak import-users-from-passwd takes no non-option arguments.") usage(1) session = DBConn().session() valid_gid = cnf.get("Import-Users-From-Passwd::ValidGID", "") if valid_gid: debiangrp = grp.getgrnam(valid_gid).gr_mem else: debiangrp = [] passwd_unames = {} for entry in pwd.getpwall(): uname = entry[0] if uname not in debiangrp: if Options["Verbose"]: print "Skipping %s (Not in group %s)." % (uname, valid_gid) continue passwd_unames[uname] = "" postgres_unames = {} q = session.execute("SELECT usename FROM pg_user") for i in q.fetchall(): uname = i[0] postgres_unames[uname] = "" known_postgres_unames = {} for i in cnf.get("Import-Users-From-Passwd::KnownPostgres","").split(","): uname = i.strip() known_postgres_unames[uname] = "" keys = postgres_unames.keys() keys.sort() for uname in keys: if not passwd_unames.has_key(uname) and not known_postgres_unames.has_key(uname): print "I: Deleting %s from Postgres, no longer in passwd or list of known Postgres users" % (uname) q = session.execute('DROP USER "%s"' % (uname)) keys = passwd_unames.keys() keys.sort() safe_name = re.compile('^[A-Za-z0-9]+$') for uname in keys: if not postgres_unames.has_key(uname): if not Options["Quiet"]: print "Creating %s user in Postgres." % (uname) if not Options["No-Action"]: if safe_name.match(uname): # NB: I never figured out how to use a bind parameter for this query # XXX: Fix this as it looks like a potential SQL injection attack to me # (hence the safe_name match we do) try: q = session.execute('CREATE USER "%s"' % (uname)) session.commit() except Exception as e: utils.warn("Could not create user %s (%s)" % (uname, str(e))) session.rollback() else: print "NOT CREATING USER %s. Doesn't match safety regex" % uname session.commit()
def action_dm(self, fingerprint, section, session): cnf = Config() if 'Command::DM::AdminKeyrings' not in cnf \ or 'Command::DM::ACL' not in cnf \ or 'Command::DM::Keyrings' not in cnf: raise CommandError('DM command is not configured for this archive.') allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings') if fingerprint.keyring.keyring_name not in allowed_keyrings: raise CommandError('Key {0} is not allowed to set DM'.format(fingerprint.fingerprint)) acl_name = cnf.get('Command::DM::ACL', 'dm') acl = session.query(ACL).filter_by(name=acl_name).one() fpr_hash = section['Fingerprint'].translate(None, ' ') fpr = session.query(Fingerprint).filter_by(fingerprint=fpr_hash).first() if fpr is None: raise CommandError('Unknown fingerprint {0}'.format(fpr_hash)) if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list('Command::DM::Keyrings'): raise CommandError('Key {0} is not in DM keyring.'.format(fpr.fingerprint)) addresses = gpg_get_key_addresses(fpr.fingerprint) if len(addresses) > 0: self.cc.append(addresses[0]) self.log.log(['dm', 'fingerprint', fpr.fingerprint]) self.result.append('Fingerprint: {0}'.format(fpr.fingerprint)) if len(addresses) > 0: self.log.log(['dm', 'uid', addresses[0]]) self.result.append('Uid: {0}'.format(addresses[0])) for source in self._split_packages(section.get('Allow', '')): # Check for existance of source package to catch typos if session.query(DBSource).filter_by(source=source).first() is None: raise CommandError('Tried to grant permissions for unknown source package: {0}'.format(source)) if session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).first() is None: aps = ACLPerSource() aps.acl = acl aps.fingerprint = fpr aps.source = source aps.created_by = fingerprint aps.reason = section.get('Reason') session.add(aps) self.log.log(['dm', 'allow', fpr.fingerprint, source]) self.result.append('Allowed: {0}'.format(source)) else: self.result.append('Already-Allowed: {0}'.format(source)) session.flush() for source in self._split_packages(section.get('Deny', '')): count = session.query(ACLPerSource).filter_by(acl=acl, fingerprint=fpr, source=source).delete() if count == 0: raise CommandError('Tried to remove upload permissions for package {0}, ' 'but no upload permissions were granted before.'.format(source)) self.log.log(['dm', 'deny', fpr.fingerprint, source]) self.result.append('Denied: {0}'.format(source)) session.commit()
def main(argv=None): if argv is None: argv = sys.argv arguments = [('h', 'help', 'Process-Commands::Options::Help'), ('d', 'directory', 'Process-Commands::Options::Directory', 'HasArg')] cnf = Config() cnf['Process-Commands::Options::Dummy'] = '' filenames = apt_pkg.parse_commandline(cnf.Cnf, arguments, argv) options = cnf.subtree('Process-Commands::Options') if 'Help' in options or (len(filenames) == 0 and 'Directory' not in options): usage() sys.exit(0) log = Logger('command') now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) rejectdir = cnf['Dir::Reject'] if len(filenames) == 0: cdir = options['Directory'] filenames = [ os.path.join(cdir, fn) for fn in os.listdir(cdir) if fn.endswith('.dak-commands') ] for fn in filenames: basename = os.path.basename(fn) if not fn.endswith('.dak-commands'): log.log(['unexpected filename', basename]) continue with open(fn, 'r') as fh: data = fh.read() try: command = CommandFile(basename, data, log) command.evaluate() except CommandError as e: created = os.stat(fn).st_mtime now = time.time() too_new = (now - created < int(cnf.get('Dinstall::SkipTime', '60'))) if too_new: log.log(['skipped (too new)']) continue log.log(['reject', basename, e]) except Exception as e: log.log_traceback('Exception while processing %s:' % (basename), e) dst = find_next_free(os.path.join(rejectdir, basename)) else: log.log(['done', basename]) dst = find_next_free(os.path.join(donedir, basename)) with FilesystemTransaction() as fs: fs.unlink(fn) fs.create(dst, mode=0o644).write(data) fs.commit() log.close()
def action_dm(self, fingerprint, section, session): cnf = Config() if 'Command::DM::AdminKeyrings' not in cnf \ or 'Command::DM::ACL' not in cnf \ or 'Command::DM::Keyrings' not in cnf: raise CommandError( 'DM command is not configured for this archive.') allowed_keyrings = cnf.value_list('Command::DM::AdminKeyrings') if fingerprint.keyring.keyring_name not in allowed_keyrings: raise CommandError('Key {0} is not allowed to set DM'.format( fingerprint.fingerprint)) acl_name = cnf.get('Command::DM::ACL', 'dm') acl = session.query(ACL).filter_by(name=acl_name).one() fpr_hash = section['Fingerprint'].translate(None, ' ') fpr = session.query(Fingerprint).filter_by( fingerprint=fpr_hash).first() if fpr is None: raise CommandError('Unknown fingerprint {0}'.format(fpr_hash)) if fpr.keyring is None or fpr.keyring.keyring_name not in cnf.value_list( 'Command::DM::Keyrings'): raise CommandError('Key {0} is not in DM keyring.'.format( fpr.fingerprint)) addresses = gpg_get_key_addresses(fpr.fingerprint) if len(addresses) > 0: self.cc.append(addresses[0]) self.log.log(['dm', 'fingerprint', fpr.fingerprint]) self.result.append('Fingerprint: {0}'.format(fpr.fingerprint)) if len(addresses) > 0: self.log.log(['dm', 'uid', addresses[0]]) self.result.append('Uid: {0}'.format(addresses[0])) for source in self._split_packages(section.get('Allow', '')): # Check for existance of source package to catch typos if session.query(DBSource).filter_by( source=source).first() is None: raise CommandError( 'Tried to grant permissions for unknown source package: {0}' .format(source)) if session.query(ACLPerSource).filter_by( acl=acl, fingerprint=fpr, source=source).first() is None: aps = ACLPerSource() aps.acl = acl aps.fingerprint = fpr aps.source = source aps.created_by = fingerprint aps.reason = section.get('Reason') session.add(aps) self.log.log(['dm', 'allow', fpr.fingerprint, source]) self.result.append('Allowed: {0}'.format(source)) else: self.result.append('Already-Allowed: {0}'.format(source)) session.flush() for source in self._split_packages(section.get('Deny', '')): count = session.query(ACLPerSource).filter_by( acl=acl, fingerprint=fpr, source=source).delete() if count == 0: raise CommandError( 'Tried to remove upload permissions for package {0}, ' 'but no upload permissions were granted before.'.format( source)) self.log.log(['dm', 'deny', fpr.fingerprint, source]) self.result.append('Denied: {0}'.format(source)) session.commit()
def comment_accept(upload, srcqueue, comments, transaction): for byhand in upload.byhand: path = os.path.join(srcqueue.path, byhand.filename) if os.path.exists(path): raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename)) cnf = Config() fs = transaction.fs session = transaction.session changesname = upload.changes.changesname allow_tainted = srcqueue.suite.archive.tainted # We need overrides to get the target component overridesuite = upload.target_suite if overridesuite.overridesuite is not None: overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): section = db_binary.proxy['Section'] component_name = 'main' if section.find('/') != -1: component_name = section.split('/', 1)[0] return get_mapped_component(component_name, session=session) def is_debug_binary(db_binary): return daklib.utils.is_in_debug_section(db_binary.proxy) def has_debug_binaries(upload): return any((is_debug_binary(x) for x in upload.binaries)) def source_component_func(db_source): package_list = PackageList(db_source.proxy) component = source_component_from_package_list(package_list, upload.target_suite) if component is not None: return get_mapped_component(component.component_name, session=session) # Fallback for packages without Package-List field query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ .join(Component) return query.one().component all_target_suites = [upload.target_suite] all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) for suite in all_target_suites: debug_suite = suite.debug_suite if upload.source is not None: # If we have Source in this upload, let's include it into # upload suite. transaction.copy_source( upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted, ) if debug_suite is not None and has_debug_binaries(upload): # If we're handing a debug package, we also need to include the # source in the debug suite as well. transaction.copy_source( upload.source, debug_suite, source_component_func(upload.source), allow_tainted=allow_tainted, ) for db_binary in upload.binaries: # Now, let's work out where to copy this guy to -- if it's # a debug binary, and the suite has a debug suite, let's go # ahead and target the debug suite rather then the stock # suite. copy_to_suite = suite if debug_suite is not None and is_debug_binary(db_binary): copy_to_suite = debug_suite # build queues and debug suites may miss the source package # if this is a binary-only upload. if copy_to_suite != upload.target_suite: transaction.copy_source( db_binary.source, copy_to_suite, source_component_func(db_binary.source), allow_tainted=allow_tainted, ) transaction.copy_binary( db_binary, copy_to_suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive], ) check_upload_for_external_signature_request(session, suite, copy_to_suite, db_binary) suite.update_last_changed() # Copy .changes if needed if upload.target_suite.copychanges: src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) # List of files in the queue directory queue_files = [changesname] chg = daklib.upload.Changes(upload.policy_queue.path, changesname, keyrings=[], require_signature=False) queue_files.extend(f.filename for f in chg.buildinfo_files) # Copy upload to Process-Policy::CopyDir # Used on security.d.o to sync accepted packages to ftp-master, but this # should eventually be replaced by something else. copydir = cnf.get('Process-Policy::CopyDir') or None if copydir is not None: mode = upload.target_suite.archive.mode if upload.source is not None: for f in [ df.poolfile for df in upload.source.srcfiles ]: dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for db_binary in upload.binaries: f = db_binary.poolfile dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for fn in queue_files: src = os.path.join(upload.policy_queue.path, fn) dst = os.path.join(copydir, fn) # We check for `src` to exist as old uploads in policy queues # might still miss the `.buildinfo` files. if os.path.exists(src) and not os.path.exists(dst): fs.copy(src, dst, mode=mode) if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency # As per policy 5.6.17, the urgency can be followed by a space and a # comment. Extract only the urgency from the string. if ' ' in urgency: (urgency, comment) = urgency.split(' ', 1) if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(upload.source.source, upload.source.version, urgency) print " ACCEPT" if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) for fn in queue_files: src = os.path.join(upload.policy_queue.path, fn) if os.path.exists(src): dst = os.path.join(donedir, fn) dst = utils.find_next_free(dst) fs.copy(src, dst, mode=0o644) remove_upload(upload, transaction)
def main(): global suite, suite_id, source_binaries, source_versions cnf = Config() Arguments = [('h', "help", "Cruft-Report::Options::Help"), ('m', "mode", "Cruft-Report::Options::Mode", "HasArg"), ('R', "rdep-check", "Cruft-Report::Options::Rdep-Check"), ('s', "suite", "Cruft-Report::Options::Suite", "HasArg"), ('w', "wanna-build-dump", "Cruft-Report::Options::Wanna-Build-Dump", "HasArg")] for i in ["help", "Rdep-Check"]: key = "Cruft-Report::Options::%s" % i if key not in cnf: cnf[key] = "" cnf["Cruft-Report::Options::Suite"] = cnf.get("Dinstall::DefaultSuite", "unstable") if "Cruft-Report::Options::Mode" not in cnf: cnf["Cruft-Report::Options::Mode"] = "daily" if "Cruft-Report::Options::Wanna-Build-Dump" not in cnf: cnf["Cruft-Report::Options::Wanna-Build-Dump"] = "/srv/ftp-master.debian.org/scripts/nfu" apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Cruft-Report::Options") if Options["Help"]: usage() if Options["Rdep-Check"]: rdeps = True else: rdeps = False # Set up checks based on mode if Options["Mode"] == "daily": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata" ] elif Options["Mode"] == "full": checks = [ "nbs", "nviu", "nvit", "obsolete source", "outdated non-free", "nfu", "nbs metadata", "dubious nbs", "bnb", "bms", "anais" ] elif Options["Mode"] == "bdo": checks = ["nbs", "obsolete source"] else: utils.warn( "%s is not a recognised mode - only 'full', 'daily' or 'bdo' are understood." % (Options["Mode"])) usage(1) session = DBConn().session() bin_pkgs = {} src_pkgs = {} bin2source = {} bins_in_suite = {} nbs = {} source_versions = {} anais_output = "" nfu_packages = {} suite = get_suite(Options["Suite"].lower(), session) if not suite: utils.fubar("Cannot find suite %s" % Options["Suite"].lower()) suite_id = suite.suite_id suite_name = suite.suite_name.lower() if "obsolete source" in checks: report_obsolete_source(suite_name, session) if "nbs" in checks: reportAllNBS(suite_name, suite_id, session, rdeps) if "nbs metadata" in checks: reportNBSMetadata(suite_name, suite_id, session, rdeps) if "outdated non-free" in checks: report_outdated_nonfree(suite_name, session, rdeps) bin_not_built = {} if "bnb" in checks: bins_in_suite = get_suite_binaries(suite, session) # Checks based on the Sources files components = get_component_names(session) for component in components: filename = "%s/dists/%s/%s/source/Sources" % (suite.archive.path, suite_name, component) filename = utils.find_possibly_compressed_file(filename) with apt_pkg.TagFile(filename) as Sources: while Sources.step(): source = Sources.section.find('Package') source_version = Sources.section.find('Version') architecture = Sources.section.find('Architecture') binaries = Sources.section.find('Binary') binaries_list = [i.strip() for i in binaries.split(',')] if "bnb" in checks: # Check for binaries not built on any architecture. for binary in binaries_list: if binary not in bins_in_suite: bin_not_built.setdefault(source, {}) bin_not_built[source][binary] = "" if "anais" in checks: anais_output += do_anais(architecture, binaries_list, source, session) # build indices for checking "no source" later source_index = component + '/' + source src_pkgs[source] = source_index for binary in binaries_list: bin_pkgs[binary] = source source_binaries[source] = binaries source_versions[source] = source_version # Checks based on the Packages files check_components = components[:] if suite_name != "experimental": check_components.append('main/debian-installer') for component in check_components: architectures = [ a.arch_string for a in get_suite_architectures( suite_name, skipsrc=True, skipall=True, session=session) ] for architecture in architectures: if component == 'main/debian-installer' and re.match( "kfreebsd", architecture): continue if "nfu" in checks: nfu_packages.setdefault(architecture, []) nfu_entries = parse_nfu(architecture) filename = "%s/dists/%s/%s/binary-%s/Packages" % ( suite.archive.path, suite_name, component, architecture) filename = utils.find_possibly_compressed_file(filename) with apt_pkg.TagFile(filename) as Packages: while Packages.step(): package = Packages.section.find('Package') source = Packages.section.find('Source', "") version = Packages.section.find('Version') if source == "": source = package if package in bin2source and \ apt_pkg.version_compare(version, bin2source[package]["version"]) > 0: bin2source[package]["version"] = version bin2source[package]["source"] = source else: bin2source[package] = {} bin2source[package]["version"] = version bin2source[package]["source"] = source if source.find("(") != -1: m = re_extract_src_version.match(source) source = m.group(1) version = m.group(2) if package not in bin_pkgs: nbs.setdefault(source, {}) nbs[source].setdefault(package, {}) nbs[source][package][version] = "" else: if "nfu" in checks: if package in nfu_entries and \ version != source_versions[source]: # only suggest to remove out-of-date packages nfu_packages[architecture].append( (package, version, source_versions[source])) # Distinguish dubious (version numbers match) and 'real' NBS (they don't) dubious_nbs = {} version_sort_key = functools.cmp_to_key(apt_pkg.version_compare) for source in nbs: for package in nbs[source]: latest_version = max(nbs[source][package], key=version_sort_key) source_version = source_versions.get(source, "0") if apt_pkg.version_compare(latest_version, source_version) == 0: add_nbs(dubious_nbs, source, latest_version, package, suite_id, session) if "nviu" in checks: do_newer_version('unstable', 'experimental', 'NVIU', session) if "nvit" in checks: do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session) ### if Options["Mode"] == "full": print("=" * 75) print() if "nfu" in checks: do_nfu(nfu_packages) if "bnb" in checks: print("Unbuilt binary packages") print("-----------------------") print() for source in sorted(bin_not_built): binaries = sorted(bin_not_built[source]) print(" o %s: %s" % (source, ", ".join(binaries))) print() if "bms" in checks: report_multiple_source(suite) if "anais" in checks: print("Architecture Not Allowed In Source") print("----------------------------------") print(anais_output) print() if "dubious nbs" in checks: do_dubious_nbs(dubious_nbs)