def main(): global Options, Logger cnf = Config() session = DBConn().session() Arguments = [('h', "help", "Process-Policy::Options::Help"), ('n', "no-action", "Process-Policy::Options::No-Action")] for i in ["help", "no-action"]: key = "Process-Policy::Options::%s" % i if key not in cnf: cnf[key] = "" queue_name = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) if len(queue_name) != 1: print("E: Specify exactly one policy queue") sys.exit(1) queue_name = queue_name[0] Options = cnf.subtree("Process-Policy::Options") if Options["Help"]: usage() Logger = daklog.Logger("process-policy") if not Options["No-Action"]: urgencylog = UrgencyLog() with ArchiveTransaction() as transaction: session = transaction.session try: pq = session.query(PolicyQueue).filter_by( queue_name=queue_name).one() except NoResultFound: print("E: Cannot find policy queue %s" % queue_name) sys.exit(1) commentsdir = os.path.join(pq.path, 'COMMENTS') # The comments stuff relies on being in the right directory os.chdir(pq.path) do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction) do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction) remove_unreferenced_binaries(pq, transaction) remove_unreferenced_sources(pq, transaction) if not Options['No-Action']: urgencylog.close()
def main(): global Options, Logger cnf = Config() session = DBConn().session() Arguments = [ ("h", "help", "Process-Policy::Options::Help"), ("n", "no-action", "Process-Policy::Options::No-Action"), ] for i in ["help", "no-action"]: if not cnf.has_key("Process-Policy::Options::%s" % (i)): cnf["Process-Policy::Options::%s" % (i)] = "" queue_name = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) if len(queue_name) != 1: print "E: Specify exactly one policy queue" sys.exit(1) queue_name = queue_name[0] Options = cnf.subtree("Process-Policy::Options") if Options["Help"]: usage() Logger = daklog.Logger("process-policy") if not Options["No-Action"]: urgencylog = UrgencyLog() with ArchiveTransaction() as transaction: session = transaction.session try: pq = session.query(PolicyQueue).filter_by(queue_name=queue_name).one() except NoResultFound: print "E: Cannot find policy queue %s" % queue_name sys.exit(1) commentsdir = os.path.join(pq.path, "COMMENTS") # The comments stuff relies on being in the right directory os.chdir(pq.path) do_comments(commentsdir, pq, "REJECT.", "REJECTED.", "NOTOK", comment_reject, transaction) do_comments(commentsdir, pq, "ACCEPT.", "ACCEPTED.", "OK", comment_accept, transaction) do_comments(commentsdir, pq, "ACCEPTED.", "ACCEPTED.", "OK", comment_accept, transaction) remove_unreferenced_binaries(pq, transaction) remove_unreferenced_sources(pq, transaction) if not Options["No-Action"]: urgencylog.close()
def accept(directory, upload): cnf = Config() Logger.log(['ACCEPT', upload.changes.filename]) print("ACCEPT") upload.install() utils.process_buildinfos(upload.directory, upload.changes.buildinfo_files, upload.transaction.fs, Logger) accepted_to_real_suite = any(suite.policy_queue is None for suite in upload.final_suites) sourceful_upload = upload.changes.sourceful control = upload.changes.changes if sourceful_upload and not Options['No-Action']: urgency = control.get('Urgency') # As per policy 5.6.17, the urgency can be followed by a space and a # comment. Extract only the urgency from the string. if ' ' in urgency: urgency, comment = urgency.split(' ', 1) if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(control['Source'], control['Version'], urgency) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # Move .changes to done, but only for uploads that were accepted to a # real suite. process-policy will handle this for uploads to queues. if accepted_to_real_suite: src = os.path.join(upload.directory, upload.changes.filename) now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) dst = os.path.join(donedir, upload.changes.filename) dst = utils.find_next_free(dst) upload.transaction.fs.copy(src, dst, mode=0o644) SummaryStats().accept_count += 1 SummaryStats().accept_bytes += upload.changes.bytes
def accept(directory, upload): cnf = Config() Logger.log(['ACCEPT', upload.changes.filename]) print "ACCEPT" upload.install() process_buildinfos(upload) accepted_to_real_suite = False for suite in upload.final_suites: accepted_to_real_suite = accepted_to_real_suite or suite.policy_queue is None sourceful_upload = 'source' in upload.changes.architectures control = upload.changes.changes if sourceful_upload and not Options['No-Action']: urgency = control.get('Urgency') if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(control['Source'], control['Version'], urgency) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # Move .changes to done, but only for uploads that were accepted to a # real suite. process-policy will handle this for uploads to queues. if accepted_to_real_suite: src = os.path.join(upload.directory, upload.changes.filename) now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) dst = os.path.join(donedir, upload.changes.filename) dst = utils.find_next_free(dst) upload.transaction.fs.copy(src, dst, mode=0o644) SummaryStats().accept_count += 1 SummaryStats().accept_bytes += upload.changes.bytes
def comment_accept(upload, srcqueue, comments, transaction): for byhand in upload.byhand: path = os.path.join(srcqueue.path, byhand.filename) if os.path.exists(path): raise Exception('E: cannot ACCEPT upload with unprocessed byhand file {0}'.format(byhand.filename)) cnf = Config() fs = transaction.fs session = transaction.session changesname = upload.changes.changesname allow_tainted = srcqueue.suite.archive.tainted # We need overrides to get the target component overridesuite = upload.target_suite if overridesuite.overridesuite is not None: overridesuite = session.query(Suite).filter_by(suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): section = db_binary.proxy['Section'] component_name = 'main' if section.find('/') != -1: component_name = section.split('/', 1)[0] return get_mapped_component(component_name, session=session) def is_debug_binary(db_binary): return daklib.utils.is_in_debug_section(db_binary.proxy) def has_debug_binaries(upload): return any((is_debug_binary(x) for x in upload.binaries)) def source_component_func(db_source): package_list = PackageList(db_source.proxy) component = source_component_from_package_list(package_list, upload.target_suite) if component is not None: return get_mapped_component(component.component_name, session=session) # Fallback for packages without Package-List field query = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ .join(Component) return query.one().component all_target_suites = [upload.target_suite] all_target_suites.extend([q.suite for q in upload.target_suite.copy_queues]) for suite in all_target_suites: debug_suite = suite.debug_suite if upload.source is not None: # If we have Source in this upload, let's include it into # upload suite. transaction.copy_source( upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted, ) if debug_suite is not None and has_debug_binaries(upload): # If we're handing a debug package, we also need to include the # source in the debug suite as well. transaction.copy_source( upload.source, debug_suite, source_component_func(upload.source), allow_tainted=allow_tainted, ) for db_binary in upload.binaries: # Now, let's work out where to copy this guy to -- if it's # a debug binary, and the suite has a debug suite, let's go # ahead and target the debug suite rather then the stock # suite. copy_to_suite = suite if debug_suite is not None and is_debug_binary(db_binary): copy_to_suite = debug_suite # build queues and debug suites may miss the source package # if this is a binary-only upload. if copy_to_suite != upload.target_suite: transaction.copy_source( db_binary.source, copy_to_suite, source_component_func(db_binary.source), allow_tainted=allow_tainted, ) transaction.copy_binary( db_binary, copy_to_suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive], ) check_upload_for_external_signature_request(session, suite, copy_to_suite, db_binary) suite.update_last_changed() # Copy .changes if needed if upload.target_suite.copychanges: src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) # List of files in the queue directory queue_files = [changesname] chg = daklib.upload.Changes(upload.policy_queue.path, changesname, keyrings=[], require_signature=False) queue_files.extend(f.filename for f in chg.buildinfo_files) # Copy upload to Process-Policy::CopyDir # Used on security.d.o to sync accepted packages to ftp-master, but this # should eventually be replaced by something else. copydir = cnf.get('Process-Policy::CopyDir') or None if copydir is not None: mode = upload.target_suite.archive.mode if upload.source is not None: for f in [ df.poolfile for df in upload.source.srcfiles ]: dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for db_binary in upload.binaries: f = db_binary.poolfile dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for fn in queue_files: src = os.path.join(upload.policy_queue.path, fn) dst = os.path.join(copydir, fn) # We check for `src` to exist as old uploads in policy queues # might still miss the `.buildinfo` files. if os.path.exists(src) and not os.path.exists(dst): fs.copy(src, dst, mode=mode) if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency # As per policy 5.6.17, the urgency can be followed by a space and a # comment. Extract only the urgency from the string. if ' ' in urgency: (urgency, comment) = urgency.split(' ', 1) if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(upload.source.source, upload.source.version, urgency) print " ACCEPT" if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) for fn in queue_files: src = os.path.join(upload.policy_queue.path, fn) if os.path.exists(src): dst = os.path.join(donedir, fn) dst = utils.find_next_free(dst) fs.copy(src, dst, mode=0o644) remove_upload(upload, transaction)
def main(): global Options, Logger cnf = Config() summarystats = SummaryStats() Arguments = [('a', "automatic", "Dinstall::Options::Automatic"), ('h', "help", "Dinstall::Options::Help"), ('n', "no-action", "Dinstall::Options::No-Action"), ('p', "no-lock", "Dinstall::Options::No-Lock"), ('s', "no-mail", "Dinstall::Options::No-Mail"), ('d', "directory", "Dinstall::Options::Directory", "HasArg")] for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version", "directory"]: key = "Dinstall::Options::%s" % i if key not in cnf: cnf[key] = "" changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Dinstall::Options") if Options["Help"]: usage() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: Options["Automatic"] = "" # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'process-upload.lock'), os.O_RDWR | os.O_CREAT) try: fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise # Initialise UrgencyLog() - it will deal with the case where we don't # want to log urgencies urgencylog = UrgencyLog() Logger = daklog.Logger("process-upload", Options["No-Action"]) # If we have a directory flag, use it to find our files if cnf["Dinstall::Options::Directory"] != "": # Note that we clobber the list of files we were given in this case # so warn if the user has done both if len(changes_files) > 0: utils.warn("Directory provided so ignoring files given on command line") changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)]) elif not len(changes_files) > 0: utils.fubar("No changes files given and no directory specified") else: Logger.log(["Using changes files from command-line", len(changes_files)]) process_changes(changes_files) if summarystats.accept_count: sets = "set" if summarystats.accept_count > 1: sets = "sets" print("Installed %d package %s, %s." % (summarystats.accept_count, sets, utils.size_type(int(summarystats.accept_bytes)))) Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) if summarystats.reject_count: sets = "set" if summarystats.reject_count > 1: sets = "sets" print("Rejected %d package %s." % (summarystats.reject_count, sets)) Logger.log(["rejected", summarystats.reject_count]) if not Options["No-Action"]: urgencylog.close() Logger.close()
def main(): global Options, Logger cnf = Config() summarystats = SummaryStats() DBConn() Arguments = [('a',"automatic","Dinstall::Options::Automatic"), ('h',"help","Dinstall::Options::Help"), ('n',"no-action","Dinstall::Options::No-Action"), ('p',"no-lock", "Dinstall::Options::No-Lock"), ('s',"no-mail", "Dinstall::Options::No-Mail"), ('d',"directory", "Dinstall::Options::Directory", "HasArg")] for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version", "directory"]: if not cnf.has_key("Dinstall::Options::%s" % (i)): cnf["Dinstall::Options::%s" % (i)] = "" changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Dinstall::Options") if Options["Help"]: usage() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: Options["Automatic"] = "" # Check that we aren't going to clash with the daily cron job if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later.") # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT) try: fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN': utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise # Initialise UrgencyLog() - it will deal with the case where we don't # want to log urgencies urgencylog = UrgencyLog() Logger = daklog.Logger("process-upload", Options["No-Action"]) # If we have a directory flag, use it to find our files if cnf["Dinstall::Options::Directory"] != "": # Note that we clobber the list of files we were given in this case # so warn if the user has done both if len(changes_files) > 0: utils.warn("Directory provided so ignoring files given on command line") changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)]) elif not len(changes_files) > 0: utils.fubar("No changes files given and no directory specified") else: Logger.log(["Using changes files from command-line", len(changes_files)]) # Sort the .changes files so that we process sourceful ones first changes_files.sort(utils.changes_compare) # Process the changes files for changes_file in changes_files: print "\n" + changes_file session = DBConn().session() process_it(changes_file, session) session.close() if summarystats.accept_count: sets = "set" if summarystats.accept_count > 1: sets = "sets" print "Installed %d package %s, %s." % (summarystats.accept_count, sets, utils.size_type(int(summarystats.accept_bytes))) Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) if summarystats.reject_count: sets = "set" if summarystats.reject_count > 1: sets = "sets" print "Rejected %d package %s." % (summarystats.reject_count, sets) Logger.log(["rejected", summarystats.reject_count]) byebye() if not Options["No-Action"]: urgencylog.close() Logger.close()
def comment_accept(upload, srcqueue, comments, transaction): for byhand in upload.byhand: path = os.path.join(srcqueue.path, byhand.filename) if os.path.exists(path): raise Exception( 'E: cannot ACCEPT upload with unprocessed byhand file {0}'. format(byhand.filename)) cnf = Config() fs = transaction.fs session = transaction.session changesname = upload.changes.changesname allow_tainted = srcqueue.suite.archive.tainted # We need overrides to get the target component overridesuite = upload.target_suite if overridesuite.overridesuite is not None: overridesuite = session.query(Suite).filter_by( suite_name=overridesuite.overridesuite).one() def binary_component_func(db_binary): override = session.query(Override).filter_by(suite=overridesuite, package=db_binary.package) \ .join(OverrideType).filter(OverrideType.overridetype == db_binary.binarytype) \ .join(Component).one() return override.component def source_component_func(db_source): override = session.query(Override).filter_by(suite=overridesuite, package=db_source.source) \ .join(OverrideType).filter(OverrideType.overridetype == 'dsc') \ .join(Component).one() return override.component all_target_suites = [upload.target_suite] all_target_suites.extend( [q.suite for q in upload.target_suite.copy_queues]) for suite in all_target_suites: if upload.source is not None: transaction.copy_source(upload.source, suite, source_component_func(upload.source), allow_tainted=allow_tainted) for db_binary in upload.binaries: # build queues may miss the source package if this is a binary-only upload if suite != upload.target_suite: transaction.copy_source(db_binary.source, suite, source_component_func( db_binary.source), allow_tainted=allow_tainted) transaction.copy_binary( db_binary, suite, binary_component_func(db_binary), allow_tainted=allow_tainted, extra_archives=[upload.target_suite.archive]) # Copy .changes if needed if upload.target_suite.copychanges: src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(upload.target_suite.path, upload.changes.changesname) fs.copy(src, dst, mode=upload.target_suite.archive.mode) # Copy upload to Process-Policy::CopyDir # Used on security.d.o to sync accepted packages to ftp-master, but this # should eventually be replaced by something else. copydir = cnf.get('Process-Policy::CopyDir') or None if copydir is not None: mode = upload.target_suite.archive.mode if upload.source is not None: for f in [df.poolfile for df in upload.source.srcfiles]: dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) for db_binary in upload.binaries: f = db_binary.poolfile dst = os.path.join(copydir, f.basename) if not os.path.exists(dst): fs.copy(f.fullpath, dst, mode=mode) src = os.path.join(upload.policy_queue.path, upload.changes.changesname) dst = os.path.join(copydir, upload.changes.changesname) if not os.path.exists(dst): fs.copy(src, dst, mode=mode) if upload.source is not None and not Options['No-Action']: urgency = upload.changes.urgency if urgency not in cnf.value_list('Urgency::Valid'): urgency = cnf['Urgency::Default'] UrgencyLog().log(upload.source.source, upload.source.version, urgency) print " ACCEPT" if not Options['No-Action']: Logger.log(["Policy Queue ACCEPT", srcqueue.queue_name, changesname]) pu = get_processed_upload(upload) daklib.announce.announce_accept(pu) # TODO: code duplication. Similar code is in process-upload. # Move .changes to done src = os.path.join(upload.policy_queue.path, upload.changes.changesname) now = datetime.datetime.now() donedir = os.path.join(cnf['Dir::Done'], now.strftime('%Y/%m/%d')) dst = os.path.join(donedir, upload.changes.changesname) dst = utils.find_next_free(dst) fs.copy(src, dst, mode=0o644) remove_upload(upload, transaction)