def parse_prod(logdate): global stats global users maildate = ''.join([x[-2:] for x in logdate.split('-')]) mailarchive = join(utils.get_conf()['Dir::Base'], 'mail/archive', 'mail-%s.xz' % maildate) if not isfile(mailarchive): return (fd, tmpfile) = utils.temp_filename(utils.get_conf()['Dir::TempPath']) system('xzcat %s > %s' % (mailarchive, tmpfile)) for message in mbox(tmpfile): if (message['subject'] and message['subject'].startswith('Comments regarding')): try: member = users[' '.join(message['From'].split()[:-1])] except KeyError: continue ts = mktime_tz(parsedate_tz(message['date'])) timestamp = datetime.fromtimestamp(ts).strftime("%Y%m%d%H%M%S") date = parse_timestamp(timestamp) if date not in stats: stats[date] = {'stats': {'NEW': 0, 'ACCEPT': 0, 'REJECT': 0, 'PROD': 0}, 'members': {}} if member not in stats[date]['members']: stats[date]['members'][member] = {'ACCEPT': 0, 'REJECT': 0, 'PROD': 0} if member not in stats['history']['members']: stats['history']['members'][member] = {'ACCEPT': 0, 'REJECT': 0, 'PROD': 0} stats[date]['stats']['PROD'] += 1 stats[date]['members'][member]['PROD'] += 1 stats['history']['stats']['PROD'] += 1 stats['history']['members'][member]['PROD'] += 1 unlink(tmpfile)
def init(): global Cnf, Options Cnf = utils.get_conf() Arguments = [('h', "help", "Show-Deferred::Options::Help"), ("p", "link-path", "Show-Deferred::LinkPath", "HasArg"), ("d", "deferred-queue", "Show-Deferred::DeferredQueue", "HasArg"), ('r', "rrd", "Show-Deferred::Options::Rrd", "HasArg")] args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) for i in ["help"]: key = "Show-Deferred::Options::%s" % i if key not in Cnf: Cnf[key] = "" for i, j in [("DeferredQueue", "--deferred-queue")]: key = "Show-Deferred::%s" % i if key not in Cnf: print("""%s is mandatory. set via config file or command-line option %s""" % (key, j), file=sys.stderr) Options = Cnf.subtree("Show-Deferred::Options") if Options["help"]: usage() # Initialise database connection DBConn() return args
def main (): global Cnf Cnf = utils.get_conf() Arguments = [('h',"help","Stats::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Stats::Options::%s" % (i)): Cnf["Stats::Options::%s" % (i)] = "" args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Stats::Options") if Options["Help"]: usage() if len(args) < 1: utils.warn("dak stats requires a MODE argument") usage(1) elif len(args) > 1: utils.warn("dak stats accepts only one MODE argument") usage(1) mode = args[0].lower() if mode == "arch-space": per_arch_space_use() elif mode == "pkg-nums": number_of_packages() elif mode == "daily-install": daily_install_stats() else: utils.warn("unknown mode '%s'" % (mode)) usage(1)
def main(): global Cnf Cnf = utils.get_conf() Arguments = [('h',"help","Queue-Report::Options::Help"), ('n',"new","Queue-Report::Options::New"), ('8','822',"Queue-Report::Options::822"), ('s',"sort","Queue-Report::Options::Sort", "HasArg"), ('a',"age","Queue-Report::Options::Age", "HasArg"), ('r',"rrd","Queue-Report::Options::Rrd", "HasArg"), ('d',"directories","Queue-Report::Options::Directories", "HasArg")] for i in [ "help" ]: if not Cnf.has_key("Queue-Report::Options::%s" % (i)): Cnf["Queue-Report::Options::%s" % (i)] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Queue-Report::Options") if Options["Help"]: usage() if Cnf.has_key("Queue-Report::Options::New"): header() queue_names = [] if Cnf.has_key("Queue-Report::Options::Directories"): for i in Cnf["Queue-Report::Options::Directories"].split(","): queue_names.append(i) elif Cnf.has_key("Queue-Report::Directories"): queue_names = Cnf.value_list("Queue-Report::Directories") else: queue_names = [ "byhand", "new" ] if Cnf.has_key("Queue-Report::Options::Rrd"): rrd_dir = Cnf["Queue-Report::Options::Rrd"] elif Cnf.has_key("Dir::Rrd"): rrd_dir = Cnf["Dir::Rrd"] else: rrd_dir = None f = None if Cnf.has_key("Queue-Report::Options::822"): # Open the report file f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w") session = DBConn().session() for queue_name in queue_names: queue = session.query(PolicyQueue).filter_by(queue_name=queue_name).first() if queue is not None: process_queue(queue, f, rrd_dir) else: utils.warn("Cannot find queue %s" % queue_name) if Cnf.has_key("Queue-Report::Options::822"): f.close() if Cnf.has_key("Queue-Report::Options::New"): footer()
def do_update(self): """ Execute the DB update """ print "Lets make suite_architecture table use sane values" Cnf = get_conf() query = "INSERT into suite_architectures (suite, architecture) VALUES (%s, %s)" #: Update query try: c = self.db.cursor() c.execute("DELETE FROM suite_architectures;") c.execute("SELECT id, arch_string FROM architecture;") a=c.fetchall() for arch in a: archs[arch[1]]=arch[0] c.execute("SELECT id,suite_name FROM suite") s=c.fetchall() for suite in s: suites[suite[1]]=suite[0] for suite in Cnf.subtree("Suite").list(): print "Processing suite %s" % (suite) architectures = Cnf.subtree("Suite::" + suite).value_list("Architectures") suite = suite.lower() for arch in architectures: c.execute(query, [suites[suite], archs[arch]]) c.execute("UPDATE config SET value = '4' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError("Unable to apply sanity to suite_architecture table, rollback issued. Error message : %s" % (str(msg)))
def show_config(command): args = [str(x) for x in command] cnf = utils.get_conf() die_arglen(args, 2, "E: config needs at least a command") mode = args[1].lower() if mode == 'db': connstr = "" if cnf.has_key("DB::Service"): # Service mode connstr = "postgresql://service=%s" % cnf["DB::Service"] elif cnf.has_key("DB::Host"): # TCP/IP connstr = "postgres://%s" % cnf["DB::Host"] if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1": connstr += ":%s" % cnf["DB::Port"] connstr += "/%s" % cnf["DB::Name"] else: # Unix Socket connstr = "postgres:///%s" % cnf["DB::Name"] if cnf["DB::Port"] and cnf["DB::Port"] != "-1": connstr += "?port=%s" % cnf["DB::Port"] print connstr elif mode == 'db-shell': e = [] if cnf.has_key("DB::Service"): e.append('PGSERVICE') print "PGSERVICE=%s" % cnf["DB::Service"] if cnf.has_key("DB::Name"): e.append('PGDATABASE') print "PGDATABASE=%s" % cnf["DB::Name"] if cnf.has_key("DB::Host"): print "PGHOST=%s" % cnf["DB::Host"] e.append('PGHOST') if cnf.has_key("DB::Port") and cnf["DB::Port"] != "-1": print "PGPORT=%s" % cnf["DB::Port"] e.append('PGPORT') print "export " + " ".join(e) elif mode == 'get': print cnf.get(args[2]) else: session = DBConn().session() try: o = session.query(DBConfig).filter_by(name = mode).one() print o.value except NoResultFound: print "W: option '%s' not set" % mode
def architecture(command): args = [str(x) for x in command] Cnf = utils.get_conf() d = DBConn() die_arglen(args, 2, "E: architecture needs at least a command") mode = args[1].lower() if mode == 'list': __architecture_list(d, args) elif mode == 'add': __architecture_add(d, args) elif mode == 'rm': __architecture_rm(d, args) else: die("E: architecture command unknown")
def main(): Cnf = utils.get_conf() Arguments = [('h','help','Make-Changelog::Options::Help'), ('a','archive','Make-Changelog::Options::Archive','HasArg'), ('s','suite','Make-Changelog::Options::Suite','HasArg'), ('b','base-suite','Make-Changelog::Options::Base-Suite','HasArg'), ('n','binnmu','Make-Changelog::Options::binNMU'), ('e','export','Make-Changelog::Options::export'), ('p','progress','Make-Changelog::Options::progress')] for i in ['help', 'suite', 'base-suite', 'binnmu', 'export', 'progress']: if not Cnf.has_key('Make-Changelog::Options::%s' % (i)): Cnf['Make-Changelog::Options::%s' % (i)] = '' apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree('Make-Changelog::Options') suite = Cnf['Make-Changelog::Options::Suite'] base_suite = Cnf['Make-Changelog::Options::Base-Suite'] binnmu = Cnf['Make-Changelog::Options::binNMU'] export = Cnf['Make-Changelog::Options::export'] progress = Cnf['Make-Changelog::Options::progress'] if Options['help'] or not (suite and base_suite) and not export: usage() for s in suite, base_suite: if not export and not get_suite(s): utils.fubar('Invalid suite "%s"' % s) session = DBConn().session() if export: archive = session.query(Archive).filter_by(archive_name=Options['Archive']).one() exportpath = archive.changelog if exportpath: export_files(session, archive, exportpath, progress) generate_export_filelist(exportpath) else: utils.fubar('No changelog export path defined') elif binnmu: display_changes(get_binary_uploads(suite, base_suite, session), 3) else: display_changes(get_source_uploads(suite, base_suite, session), 2) session.commit()
def do_update(self): print "Add constraints to src_uploaders" Cnf = get_conf() try: c = self.db.cursor() # Deal with out-of-date src_uploaders entries c.execute("DELETE FROM src_uploaders WHERE source NOT IN (SELECT id FROM source)") c.execute("DELETE FROM src_uploaders WHERE maintainer NOT IN (SELECT id FROM maintainer)") # Add constraints c.execute("ALTER TABLE src_uploaders ADD CONSTRAINT src_uploaders_maintainer FOREIGN KEY (maintainer) REFERENCES maintainer(id) ON DELETE CASCADE") c.execute("ALTER TABLE src_uploaders ADD CONSTRAINT src_uploaders_source FOREIGN KEY (source) REFERENCES source(id) ON DELETE CASCADE") c.execute("UPDATE config SET value = '10' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError("Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
def suite(command): args = [str(x) for x in command] Cnf = utils.get_conf() d = DBConn() die_arglen(args, 2, "E: suite needs at least a command") mode = args[1].lower() if mode == 'list': __suite_list(d, args) elif mode == 'show': __suite_show(d, args) elif mode == 'add': __suite_add(d, args, False) elif mode == 'add-all-arches': __suite_add(d, args, True) else: die("E: suite command unknown")
def main(): Cnf = utils.get_conf() cnf = Config() Arguments = [('h','help','Make-Changelog::Options::Help'), ('s','suite','Make-Changelog::Options::Suite','HasArg'), ('b','base-suite','Make-Changelog::Options::Base-Suite','HasArg'), ('n','binnmu','Make-Changelog::Options::binNMU'), ('e','export','Make-Changelog::Options::export')] for i in ['help', 'suite', 'base-suite', 'binnmu', 'export']: if not Cnf.has_key('Make-Changelog::Options::%s' % (i)): Cnf['Make-Changelog::Options::%s' % (i)] = '' apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree('Make-Changelog::Options') suite = Cnf['Make-Changelog::Options::Suite'] base_suite = Cnf['Make-Changelog::Options::Base-Suite'] binnmu = Cnf['Make-Changelog::Options::binNMU'] export = Cnf['Make-Changelog::Options::export'] if Options['help'] or not (suite and base_suite) and not export: usage() for s in suite, base_suite: if not export and not get_suite(s): utils.fubar('Invalid suite "%s"' % s) session = DBConn().session() if export: if cnf.exportpath: exportpath = os.path.join(Cnf['Dir::Export'], cnf.exportpath) export_files(session, Cnf['Dir::Pool'], exportpath) else: utils.fubar('No changelog export path defined') elif binnmu: display_changes(get_binary_uploads(suite, base_suite, session), 3) else: display_changes(get_source_uploads(suite, base_suite, session), 2) session.commit()
def main(): global Cnf global users Cnf = utils.get_conf() Arguments = [('h', "help", "Stats::Options::Help")] for i in ["help"]: key = "Stats::Options::%s" % i if key not in Cnf: Cnf[key] = "" args = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Stats::Options") if Options["Help"]: usage() if len(args) < 1: utils.warn("dak stats requires a MODE argument") usage(1) elif len(args) > 1: if args[0].lower() != "new": utils.warn("dak stats accepts only one MODE argument") usage(1) elif args[0].lower() == "new": utils.warn("new MODE requires an output file") usage(1) mode = args[0].lower() if mode == "arch-space": per_arch_space_use() elif mode == "pkg-nums": number_of_packages() elif mode == "daily-install": daily_install_stats() elif mode == "new": users = utils.get_users_from_ldap() new_stats(Cnf["Dir::Log"], args[1]) else: utils.warn("unknown mode '%s'" % (mode)) usage(1)
def init(): """ Initialize. Sets up database connection, parses commandline arguments. @attention: This function may run B{within sudo} """ global Cnf, Options apt_pkg.init() Cnf = utils.get_conf() Arguments = [ ("a", "automatic", "Edit-Transitions::Options::Automatic"), ("h", "help", "Edit-Transitions::Options::Help"), ("e", "edit", "Edit-Transitions::Options::Edit"), ("i", "import", "Edit-Transitions::Options::Import", "HasArg"), ("c", "check", "Edit-Transitions::Options::Check"), ("s", "sudo", "Edit-Transitions::Options::Sudo"), ("n", "no-action", "Edit-Transitions::Options::No-Action"), ] for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]: if not Cnf.has_key("Edit-Transitions::Options::%s" % (i)): Cnf["Edit-Transitions::Options::%s" % (i)] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Edit-Transitions::Options") if Options["help"]: usage() username = utils.getusername() if username != "dak": print "Non-dak user: %s" % username Options["sudo"] = "y" # Initialise DB connection DBConn()
def do_update(self): print "pending_contents should distinguish by arch" Cnf = get_conf() try: c = self.db.cursor() c.execute("DELETE FROM pending_content_associations") c.execute("""ALTER TABLE pending_content_associations ADD COLUMN architecture integer NOT NULL""") c.execute("""ALTER TABLE ONLY pending_content_associations ADD CONSTRAINT pending_content_assiciations_arch FOREIGN KEY (architecture) REFERENCES architecture(id) ON DELETE CASCADE""") c.execute("UPDATE config SET value = '9' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError("Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
def suite_component(command): args = [str(x) for x in command] Cnf = utils.get_conf() d = DBConn() die_arglen(args, 2, "E: suite-component needs at least a command") mode = args[1].lower() if mode == 'list': __suite_component_list(d, args) elif mode == 'list-component': __suite_component_listcomponent(d, args) elif mode == 'list-suite': __suite_component_listsuite(d, args) elif mode == 'add': __suite_component_add(d, args) # elif mode == 'rm': # __suite_architecture_rm(d, args) else: die("E: suite-component command unknown")
def main(): """ for now, we just dump a list of commands that could be sent for [email protected] """ global Cnf Cnf = utils.get_conf() for arg in arguments: opt = "BtsCategorize::Options::%s" % arg[1] if opt not in Cnf: Cnf[opt] = "" packages = apt_pkg.parse_commandline(Cnf, arguments, sys.argv) Options = Cnf.subtree('BtsCategorize::Options') if Options["Help"]: usage() sys.exit(0) if Options["Quiet"]: level = logging.ERROR elif Options["Verbose"]: level = logging.DEBUG else: level = logging.INFO logging.basicConfig(level=level, format='%(asctime)s %(levelname)s %(message)s', stream=sys.stderr) body = BugClassifier().email_text() if body: send_email(body, Options["Simulate"]) else: log.info("nothing to do")
def init(): """ Initialize. Sets up database connection, parses commandline arguments. @attention: This function may run B{within sudo} """ global Cnf, Options apt_pkg.init() Cnf = utils.get_conf() Arguments = [('a', "automatic", "Edit-Transitions::Options::Automatic"), ('h', "help", "Edit-Transitions::Options::Help"), ('e', "edit", "Edit-Transitions::Options::Edit"), ('i', "import", "Edit-Transitions::Options::Import", "HasArg"), ('c', "check", "Edit-Transitions::Options::Check"), ('s', "sudo", "Edit-Transitions::Options::Sudo"), ('n', "no-action", "Edit-Transitions::Options::No-Action")] for i in ["automatic", "help", "no-action", "edit", "import", "check", "sudo"]: key = "Edit-Transitions::Options::%s" % i if key not in Cnf: Cnf[key] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Edit-Transitions::Options") if Options["help"]: usage() username = utils.getusername() if username != "dak": print("Non-dak user: %s" % username) Options["sudo"] = "y" # Initialise DB connection DBConn()
def do_update(self): print "pending_contents should distinguish by arch" Cnf = get_conf() try: c = self.db.cursor() c.execute("DELETE FROM pending_content_associations") c.execute("""ALTER TABLE pending_content_associations ADD COLUMN architecture integer NOT NULL""") c.execute("""ALTER TABLE ONLY pending_content_associations ADD CONSTRAINT pending_content_assiciations_arch FOREIGN KEY (architecture) REFERENCES architecture(id) ON DELETE CASCADE""") c.execute("UPDATE config SET value = '9' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError( "Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
def suite(command): args = [str(x) for x in command] Cnf = utils.get_conf() d = DBConn() die_arglen(args, 2, "E: suite needs at least a command") mode = args[1].lower() if mode == 'list': __suite_list(d, args) elif mode == 'show': __suite_show(d, args) elif mode == 'rm': __suite_rm(d, args) elif mode == 'add': __suite_add(d, args, False) elif mode == 'add-all-arches': __suite_add(d, args, True) elif mode == 'add-build-queue': __suite_add_build_queue(d, args) else: die("E: suite command unknown")
def show_keyring(command): args = [str(x) for x in command] cnf = utils.get_conf() die_arglen(args, 2, "E: keyring needs at least a command") mode = args[1].lower() d = DBConn() q = d.session().query(Keyring).filter(Keyring.active == True) if mode == 'list-all': pass elif mode == 'list-binary': q = q.join(Keyring.acl).filter(ACL.allow_source == False) elif mode == 'list-source': q = q.join(Keyring.acl).filter(ACL.allow_source == True) else: die("E: keyring command unknown") for k in q.all(): print k.keyring_name
def show_keyring(command): args = [str(x) for x in command] cnf = utils.get_conf() die_arglen(args, 2, "E: keyring needs at least a command") mode = args[1].lower() d = DBConn() q = d.session().query(Keyring).filter(Keyring.active == True) # noqa:E712 if mode == 'list-all': pass elif mode == 'list-binary': q = q.join(Keyring.acl).filter(ACL.allow_source == False) # noqa:E712 elif mode == 'list-source': q = q.join(Keyring.acl).filter(ACL.allow_source == True) # noqa:E712 else: die("E: keyring command unknown") for k in q.all(): print(k.keyring_name)
def main(): """Initial setup of an archive.""" global Cnf Cnf = utils.get_conf() arguments = [('h', "help", "Init-Dirs::Options::Help")] for i in ["help"]: if not Cnf.has_key("Init-Dirs::Options::%s" % (i)): Cnf["Init-Dirs::Options::%s" % (i)] = "" d = DBConn() arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv) options = Cnf.subtree("Init-Dirs::Options") if options["Help"]: usage() elif arguments: utils.warn("dak init-dirs takes no arguments.") usage(exit_code=1) create_directories()
def main (): """Initial setup of an archive.""" global Cnf Cnf = utils.get_conf() arguments = [('h', "help", "Init-Dirs::Options::Help")] for i in [ "help" ]: if not Cnf.has_key("Init-Dirs::Options::%s" % (i)): Cnf["Init-Dirs::Options::%s" % (i)] = "" d = DBConn() arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv) options = Cnf.subtree("Init-Dirs::Options") if options["Help"]: usage() elif arguments: utils.warn("dak init-dirs takes no arguments.") usage(exit_code=1) create_directories()
def version_check(command): args = [str(x) for x in command] Cnf = utils.get_conf() d = DBConn() die_arglen(args, 2, "E: version-check needs at least a command") mode = args[1].lower() if mode == 'list': __version_check_list(d) elif mode == 'list-suite': if len(args) != 3: die("E: version-check list-suite needs a single parameter") __version_check_list_suite(d, args[2]) elif mode == 'add': if len(args) != 5: die("E: version-check add needs three parameters") __version_check_add(d, args[2], args[3], args[4]) elif mode == 'rm': if len(args) != 5: die("E: version-check rm needs three parameters") __version_check_rm(d, args[2], args[3], args[4]) else: die("E: version-check command unknown")
def main(): """Perform administrative work on the dak database""" global dryrun Cnf = utils.get_conf() arguments = [('h', "help", "Admin::Options::Help"), ('n', "dry-run", "Admin::Options::Dry-Run")] for i in ["help", "dry-run"]: if not Cnf.has_key("Admin::Options::%s" % (i)): Cnf["Admin::Options::%s" % (i)] = "" arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv) options = Cnf.subtree("Admin::Options") if options["Help"] or len(arguments) < 1: usage() if options["Dry-Run"]: dryrun = True subcommand = str(arguments[0]) if subcommand in dispatch.keys(): dispatch[subcommand](arguments) else: die("E: Unknown command")
def main(): Cnf = utils.get_conf() count = 0 move_date = int(time.time()) os.chdir(Cnf["Dir::Done"]) files = glob.glob("%s/*" % (Cnf["Dir::Done"])) for filename in files: if os.path.isfile(filename): filemtime = os.stat(filename)[stat.ST_MTIME] if filemtime > move_date: continue mtime = time.gmtime(filemtime) dirname = time.strftime("%Y/%m/%d", mtime) if not os.path.exists(dirname): print "Creating: %s" % (dirname) os.makedirs(dirname) dest = dirname + '/' + os.path.basename(filename) if os.path.exists(dest): utils.warn("%s already exists." % (dest)) continue print "Move: %s -> %s" % (filename, dest) os.rename(filename, dest) count = count + 1 print "Moved %d files." % (count)
def main(): """Perform administrative work on the dak database""" global dryrun Cnf = utils.get_conf() arguments = [('h', "help", "Admin::Options::Help"), ('n', "dry-run", "Admin::Options::Dry-Run")] for i in [ "help", "dry-run" ]: if not Cnf.has_key("Admin::Options::%s" % (i)): Cnf["Admin::Options::%s" % (i)] = "" arguments = apt_pkg.parse_commandline(Cnf, arguments, sys.argv) options = Cnf.subtree("Admin::Options") if options["Help"] or len(arguments) < 1: usage() if options["Dry-Run"]: dryrun = True subcommand = str(arguments[0]) if subcommand in dispatch.keys(): dispatch[subcommand](arguments) else: die("E: Unknown command")
def main(): global Cnf Cnf = utils.get_conf() Arguments = [('h',"help","Queue-Report::Options::Help"), ('n',"new","Queue-Report::Options::New"), ('8','822',"Queue-Report::Options::822"), ('s',"sort","Queue-Report::Options::Sort", "HasArg"), ('a',"age","Queue-Report::Options::Age", "HasArg"), ('r',"rrd","Queue-Report::Options::Rrd", "HasArg"), ('d',"directories","Queue-Report::Options::Directories", "HasArg")] for i in [ "help" ]: if not Cnf.has_key("Queue-Report::Options::%s" % (i)): Cnf["Queue-Report::Options::%s" % (i)] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Queue-Report::Options") if Options["Help"]: usage() if Cnf.has_key("Queue-Report::Options::New"): header() # Initialize db so we can get the NEW comments dbconn = DBConn() queue_names = [ ] if Cnf.has_key("Queue-Report::Options::Directories"): for i in Cnf["Queue-Report::Options::Directories"].split(","): queue_names.append(i) elif Cnf.has_key("Queue-Report::Directories"): queue_names = Cnf.value_list("Queue-Report::Directories") else: queue_names = [ "byhand", "new" ] if Cnf.has_key("Queue-Report::Options::Rrd"): rrd_dir = Cnf["Queue-Report::Options::Rrd"] elif Cnf.has_key("Dir::Rrd"): rrd_dir = Cnf["Dir::Rrd"] else: rrd_dir = None f = None if Cnf.has_key("Queue-Report::Options::822"): # Open the report file f = open(Cnf["Queue-Report::ReportLocations::822Location"], "w") session = dbconn.session() for queue_name in queue_names: queue = get_policy_queue(queue_name, session) if queue: directory = os.path.abspath(queue.path) changes_files = glob.glob("%s/*.changes" % (directory)) process_changes_files(changes_files, os.path.basename(directory), f, rrd_dir) else: utils.warn("Cannot find queue %s" % queue_name) if Cnf.has_key("Queue-Report::Options::822"): f.close() if Cnf.has_key("Queue-Report::Options::New"): footer()
def main(): global Cnf Cnf = utils.get_conf() Arguments = [('h', "help", "Graph::Options::Help"), ('x', "extra-rrd", "Graph::Options::Extra-Rrd", "HasArg"), ('r', "rrd", "Graph::Options::Rrd", "HasArg"), ('i', "images", "Graph::Options::Images", "HasArg"), ('n', "names", "Graph::Options::Names", "HasArg")] for i in ["help"]: key = "Graph::Options::%s" % i if key not in Cnf: Cnf[key] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Graph::Options") if Options["Help"]: usage() names = [] if "Graph::Options::Names" in Cnf: for i in Cnf["Graph::Options::Names"].split(","): names.append(i) elif "Graph::Names" in Cnf: names = Cnf.value_list("Graph::Names") else: names = default_names extra_rrdtool_args = [] if "Graph::Options::Extra-Rrd" in Cnf: for i in Cnf["Graph::Options::Extra-Rrd"].split(","): f = open(i) extra_rrdtool_args.extend(f.read().strip().split("\n")) f.close() elif "Graph::Extra-Rrd" in Cnf: for i in Cnf.value_list("Graph::Extra-Rrd"): f = open(i) extra_rrdtool_args.extend(f.read().strip().split("\n")) f.close() if "Graph::Options::Rrd" in Cnf: rrd_dir = Cnf["Graph::Options::Rrd"] elif "Dir::Rrd" in Cnf: rrd_dir = Cnf["Dir::Rrd"] else: print("No directory to read RRD files from\n", file=sys.stderr) sys.exit(1) if "Graph::Options::Images" in Cnf: image_dir = Cnf["Graph::Options::Images"] else: print("No directory to write graph images to\n", file=sys.stderr) sys.exit(1) for name in names: stdargs = [rrd_dir, image_dir, name, extra_rrdtool_args] graph(*(stdargs + ['day', 'day', 'now-1d'])) graph(*(stdargs + ['week', 'week', 'now-1w'])) graph(*(stdargs + ['month', 'month', 'now-1m'])) graph(*(stdargs + ['year', 'year', 'now-1y'])) graph(*(stdargs + ['5years', '5 years', 'now-5y', True])) graph(*(stdargs + ['10years', '10 years', 'now-10y', True]))
def main(): global Cnf keyrings = None Cnf = utils.get_conf() Arguments = [ ('h', "help", "Add-User::Options::Help"), ('k', "key", "Add-User::Options::Key", "HasArg"), ('u', "user", "Add-User::Options::User", "HasArg"), ] for i in ["help"]: key = "Add-User::Options::%s" % i if key not in Cnf: Cnf[key] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Add-User::Options") if Options["help"]: usage() session = DBConn().session() if not keyrings: keyrings = get_active_keyring_paths() cmd = [ "gpg", "--with-colons", "--no-secmem-warning", "--no-auto-check-trustdb", "--with-fingerprint", "--no-default-keyring" ] cmd.extend(utils.gpg_keyring_args(keyrings).split()) cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]]) output = subprocess.check_output(cmd).rstrip() m = re_gpg_fingerprint_colon.search(output) if not m: print(output) utils.fubar( "0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s" % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] "))) primary_key = m.group(1) primary_key = primary_key.replace(" ", "") uid = "" if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]: uid = Cnf["Add-User::Options::User"] name = Cnf["Add-User::Options::User"] else: u = re_user_address.search(output) if not u: print(output) utils.fubar( "0x%s: (2) No userid found in gpg output but it returned 0?\n%s" % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] "))) uid = u.group(1) n = re_user_name.search(output) name = n.group(1) # Look for all email addresses on the key. emails = [] for line in output.split('\n'): e = re_user_mails.search(line) if not e: continue emails.append(e.group(2)) print("0x%s -> %s <%s> -> %s -> %s" % (Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key)) prompt = "Add user %s with above data (y/N) ? " % (uid) yn = utils.our_raw_input(prompt).lower() if yn == "y": # Create an account for the user? summary = "" # Now add user to the database. # Note that we provide a session, so we're responsible for committing uidobj = get_or_set_uid(uid, session=session) uid_id = uidobj.uid_id session.commit() # Lets add user to the email-whitelist file if its configured. if "Dinstall::MailWhiteList" in Cnf and Cnf[ "Dinstall::MailWhiteList"] != "": f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a") for mail in emails: f.write(mail + '\n') f.close() print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, name, primary_key)) # Should we send mail to the newly added user? if Cnf.find_b("Add-User::SendEmail"): mail = name + "<" + emails[0] + ">" Subst = {} Subst["__NEW_MAINTAINER__"] = mail Subst["__UID__"] = uid Subst["__KEYID__"] = Cnf["Add-User::Options::Key"] Subst["__PRIMARY_KEY__"] = primary_key Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"] Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"] Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"] Subst["__SUMMARY__"] = summary new_add_message = utils.TemplateSubst( Subst, Cnf["Dir::Templates"] + "/add-user.added") utils.send_mail(new_add_message) else: uid = None
def do_update(self): print "Moving some of the suite config into the DB" Cnf = get_conf() try: c = self.db.cursor() c.execute("ALTER TABLE suite ADD COLUMN untouchable BOOLEAN NOT NULL DEFAULT FALSE;") query = "UPDATE suite SET untouchable = TRUE WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): untouchable = Cnf.find("Suite::%s::Untouchable" % (suite)) if not untouchable: continue print "[Untouchable] Processing suite %s" % (suite) suite = suite.lower() c.execute(query, [suite]) c.execute("ALTER TABLE suite ADD COLUMN announce text NOT NULL DEFAULT '*****@*****.**';") query = "UPDATE suite SET announce = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): announce_list = Cnf.find("Suite::%s::Announce" % (suite)) print "[Announce] Processing suite %s" % (suite) suite = suite.lower() c.execute(query, [announce_list, suite]) c.execute("ALTER TABLE suite ADD COLUMN codename text;") query = "UPDATE suite SET codename = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): codename = Cnf.find("Suite::%s::CodeName" % (suite)) print "[Codename] Processing suite %s" % (suite) suite = suite.lower() c.execute(query, [codename, suite]) c.execute("ALTER TABLE suite ADD COLUMN overridecodename text;") query = "UPDATE suite SET overridecodename = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): codename = Cnf.find("Suite::%s::OverrideCodeName" % (suite)) print "[OverrideCodeName] Processing suite %s" % (suite) suite = suite.lower() c.execute(query, [codename, suite]) c.execute("ALTER TABLE suite ADD COLUMN validtime integer NOT NULL DEFAULT 604800;") query = "UPDATE suite SET validtime = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): validtime = Cnf.find("Suite::%s::ValidTime" % (suite)) print "[ValidTime] Processing suite %s" % (suite) if not validtime: validtime = 0 suite = suite.lower() c.execute(query, [validtime, suite]) c.execute("ALTER TABLE suite ADD COLUMN priority integer NOT NULL DEFAULT 0;") query = "UPDATE suite SET priority = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): priority = Cnf.find("Suite::%s::Priority" % (suite)) print "[Priority] Processing suite %s" % (suite) if not priority: priority = 0 suite = suite.lower() c.execute(query, [priority, suite]) c.execute("ALTER TABLE suite ADD COLUMN notautomatic BOOLEAN NOT NULL DEFAULT FALSE;") query = "UPDATE suite SET notautomatic = TRUE WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): notautomatic = Cnf.find("Suite::%s::NotAutomatic" % (suite)) print "[NotAutomatic] Processing suite %s" % (suite) if not notautomatic: continue suite = suite.lower() c.execute(query, [suite]) c.execute("UPDATE config SET value = '7' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError("Unable to appy suite config updates, rollback issued. Error message : %s" % (str(msg)))
def do_update(self): print "Moving some more of the suite config into the DB" Cnf = get_conf() try: c = self.db.cursor() c.execute("ALTER TABLE suite ADD COLUMN copychanges TEXT;") query = "UPDATE suite SET copychanges = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): copychanges = Cnf.find("Suite::%s::CopyChanges" % (suite)) print "[CopyChanges] Processing suite %s" % (suite) if not copychanges: continue suite = suite.lower() c.execute(query, [copychanges, suite]) c.execute("ALTER TABLE suite ADD COLUMN copydotdak TEXT;") query = "UPDATE suite SET copydotdak = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): copydotdak = Cnf.find("Suite::%s::CopyDotDak" % (suite)) print "[CopyDotDak] Processing suite %s" % (suite) if not copydotdak: continue suite = suite.lower() c.execute(query, [copydotdak, suite]) c.execute("ALTER TABLE suite ADD COLUMN commentsdir TEXT;") query = "UPDATE suite SET commentsdir = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): commentsdir = Cnf.find("Suite::%s::CommentsDir" % (suite)) print "[CommentsDir] Processing suite %s" % (suite) if not commentsdir: continue suite = suite.lower() c.execute(query, [commentsdir, suite]) c.execute("ALTER TABLE suite ADD COLUMN overridesuite TEXT;") query = "UPDATE suite SET overridesuite = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): overridesuite = Cnf.find("Suite::%s::OverrideSuite" % (suite)) print "[OverrideSuite] Processing suite %s" % (suite) if not overridesuite: continue suite = suite.lower() c.execute(query, [overridesuite, suite]) c.execute("ALTER TABLE suite ADD COLUMN changelogbase TEXT;") query = "UPDATE suite SET changelogbase = %s WHERE suite_name = %s" #: Update query for suite in Cnf.subtree("Suite").list(): changelogbase = Cnf.find("Suite::%s::ChangeLogBase" % (suite)) print "[ChangeLogBase] Processing suite %s" % (suite) if not changelogbase: continue suite = suite.lower() c.execute(query, [changelogbase, suite]) c.execute("UPDATE config SET value = '8' WHERE name = 'db_revision'") self.db.commit() except psycopg2.ProgrammingError as msg: self.db.rollback() raise DBUpdateError("Unable to apply suite config updates, rollback issued. Error message : %s" % (str(msg)))
import threading from daklib import utils from daklib.config import Config from daklib.dbconn import DBConn, get_component_by_package_suite from daklib.gpg import SignedFile from daklib.regexes import html_escaping, re_html_escaping, re_version, re_spacestrip, \ re_contrib, re_nonfree, re_localhost, re_newlinespace, \ re_package, re_doc_directory from daklib.dak_exceptions import ChangesUnicodeError import daklib.daksubprocess ################################################################################ Cnf = None Cnf = utils.get_conf() printed = threading.local() printed.copyrights = {} package_relations = {} #: Store relations of packages for later output # default is to not output html. use_html = False ################################################################################ def usage (exit_code=0): print """Usage: dak examine-package [PACKAGE]... Check NEW package(s). -h, --help show this help and exit
def main(): global Cnf, Options, Logger os.umask(0o002) Cnf = utils.get_conf() Arguments = [ ('h', "help", "Generate-Index-Diffs::Options::Help"), ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'), ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"), ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"), ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"), ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), ('n', "n-act", "Generate-Index-Diffs::Options::NoAct"), ] suites = apt_pkg.parse_commandline(Cnf,Arguments,sys.argv) Options = Cnf.subtree("Generate-Index-Diffs::Options") if Options.has_key("Help"): usage() maxdiffs = Options.get("MaxDiffs::Default", "56") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) maxsources = Options.get("MaxDiffs::Sources", maxdiffs) if not Options.has_key("PatchName"): format = "%Y-%m-%d-%H%M.%S" Options["PatchName"] = time.strftime( format ) session = DBConn().session() if not suites: query = session.query(Suite.suite_name) if Options.get('Archive'): query = query.join(Suite.archive).filter(Archive.archive_name == Options['Archive']) suites = [ s.suite_name for s in query ] for suitename in suites: print "Processing: " + suitename suiteobj = get_suite(suitename.lower(), session=session) # Use the canonical version of the suite name suite = suiteobj.suite_name if suiteobj.untouchable: print "Skipping: " + suite + " (untouchable)" continue architectures = get_suite_architectures(suite, skipall=True, session=session) components = [ c.component_name for c in session.query(Component.component_name) ] suite_suffix = Cnf.find("Dinstall::SuiteSuffix") if components and suite_suffix: longsuite = suite + "/" + suite_suffix else: longsuite = suite tree = os.path.join(suiteobj.archive.path, 'dists', longsuite) # See if there are Translations which might need a new pdiff cwd = os.getcwd() for component in components: #print "DEBUG: Working on %s" % (component) workpath=os.path.join(tree, component, "i18n") if os.path.isdir(workpath): os.chdir(workpath) for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): for entry in filenames: if not re_includeinpdiff.match(entry): #print "EXCLUDING %s" % (entry) continue (fname, fext) = os.path.splitext(entry) processfile=os.path.join(workpath, fname) #print "Working: %s" % (processfile) storename="%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname) #print "Storefile: %s" % (storename) genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs) os.chdir(cwd) for archobj in architectures: architecture = archobj.arch_string for component in components: if architecture == "source": longarch = architecture packages = "Sources" maxsuite = maxsources else: longarch = "binary-%s"% (architecture) packages = "Packages" maxsuite = maxpackages # Process Contents file = "%s/%s/Contents-%s" % (tree, component, architecture) storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) genchanges(Options, file + ".diff", storename, file, maxcontents) file = "%s/%s/%s/%s" % (tree, component, longarch, packages) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture) genchanges(Options, file + ".diff", storename, file, maxsuite)
def britney_changelog(packages, suite, session): old = {} current = {} Cnf = utils.get_conf() try: q = session.execute("SELECT changelog FROM suite WHERE id = :suiteid", {'suiteid': suite.suite_id}) brit_file = q.fetchone()[0] except: brit_file = None if brit_file: brit_file = os.path.join(Cnf['Dir::Root'], brit_file) else: return q = session.execute("""SELECT s.source, s.version, sa.id FROM source s, src_associations sa WHERE sa.suite = :suiteid AND sa.source = s.id""", {'suiteid': suite.suite_id}) for p in q.fetchall(): current[p[0]] = p[1] for p in packages.keys(): if p[2] == "source": old[p[0]] = p[1] new = {} for p in current.keys(): if p in old: if apt_pkg.version_compare(current[p], old[p]) > 0: new[p] = [current[p], old[p]] else: new[p] = [current[p], 0] query = "SELECT source, changelog FROM changelogs WHERE" for p in new.keys(): query += " source = '%s' AND version > '%s' AND version <= '%s'" \ % (p, new[p][1], new[p][0]) query += " AND architecture LIKE '%source%' AND distribution in \ ('unstable', 'experimental', 'testing-proposed-updates') OR" query += " False ORDER BY source, version DESC" q = session.execute(query) pu = None brit = utils.open_file(brit_file, 'w') for u in q: if pu and pu != u[0]: brit.write("\n") brit.write("%s\n" % u[1]) pu = u[0] if q.rowcount: brit.write("\n\n\n") for p in list(set(old.keys()).difference(current.keys())): brit.write("REMOVED: %s %s\n" % (p, old[p])) brit.flush() brit.close()
def main(): global Cnf keyrings = None Cnf = utils.get_conf() Arguments = [('h', "help", "Add-User::Options::Help"), ('k', "key", "Add-User::Options::Key", "HasArg"), ('u', "user", "Add-User::Options::User", "HasArg"), ] for i in ["help"]: key = "Add-User::Options::%s" % i if key not in Cnf: Cnf[key] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Add-User::Options") if Options["help"]: usage() session = DBConn().session() if not keyrings: keyrings = get_active_keyring_paths() cmd = ["gpg", "--with-colons", "--no-secmem-warning", "--no-auto-check-trustdb", "--with-fingerprint", "--no-default-keyring"] cmd.extend(utils.gpg_keyring_args(keyrings).split()) cmd.extend(["--list-key", "--", Cnf["Add-User::Options::Key"]]) output = subprocess.check_output(cmd).rstrip() m = re_gpg_fingerprint_colon.search(output) if not m: print(output) utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s" % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] "))) primary_key = m.group(1) primary_key = primary_key.replace(" ", "") uid = "" if "Add-User::Options::User" in Cnf and Cnf["Add-User::Options::User"]: uid = Cnf["Add-User::Options::User"] name = Cnf["Add-User::Options::User"] else: u = re_user_address.search(output) if not u: print(output) utils.fubar("0x%s: (2) No userid found in gpg output but it returned 0?\n%s" % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, " [GPG output:] "))) uid = u.group(1) n = re_user_name.search(output) name = n.group(1) # Look for all email addresses on the key. emails = [] for line in output.split('\n'): e = re_user_mails.search(line) if not e: continue emails.append(e.group(2)) print("0x%s -> %s <%s> -> %s -> %s" % (Cnf["Add-User::Options::Key"], name, emails[0], uid, primary_key)) prompt = "Add user %s with above data (y/N) ? " % (uid) yn = utils.our_raw_input(prompt).lower() if yn == "y": # Create an account for the user? summary = "" # Now add user to the database. # Note that we provide a session, so we're responsible for committing uidobj = get_or_set_uid(uid, session=session) uid_id = uidobj.uid_id session.commit() # Lets add user to the email-whitelist file if its configured. if "Dinstall::MailWhiteList" in Cnf and Cnf["Dinstall::MailWhiteList"] != "": f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a") for mail in emails: f.write(mail + '\n') f.close() print("Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, name, primary_key)) # Should we send mail to the newly added user? if Cnf.find_b("Add-User::SendEmail"): mail = name + "<" + emails[0] + ">" Subst = {} Subst["__NEW_MAINTAINER__"] = mail Subst["__UID__"] = uid Subst["__KEYID__"] = Cnf["Add-User::Options::Key"] Subst["__PRIMARY_KEY__"] = primary_key Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"] Subst["__ADMIN_ADDRESS__"] = Cnf["Dinstall::MyAdminAddress"] Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"] Subst["__DISTRO__"] = Cnf["Dinstall::MyDistribution"] Subst["__SUMMARY__"] = summary new_add_message = utils.TemplateSubst(Subst, Cnf["Dir::Templates"] + "/add-user.added") utils.send_mail(new_add_message) else: uid = None
def main(): global Cnf Cnf = utils.get_conf() Arguments = [('h',"help","Graph::Options::Help"), ('x',"extra-rrd","Graph::Options::Extra-Rrd", "HasArg"), ('r',"rrd","Graph::Options::Rrd", "HasArg"), ('i',"images","Graph::Options::Images", "HasArg"), ('n',"names","Graph::Options::Names", "HasArg")] for i in [ "help" ]: if not Cnf.has_key("Graph::Options::%s" % (i)): Cnf["Graph::Options::%s" % (i)] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Graph::Options") if Options["Help"]: usage() names = [] if Cnf.has_key("Graph::Options::Names"): for i in Cnf["Graph::Options::Names"].split(","): names.append(i) elif Cnf.has_key("Graph::Names"): names = Cnf.value_list("Graph::Names") else: names = default_names extra_rrdtool_args = [] if Cnf.has_key("Graph::Options::Extra-Rrd"): for i in Cnf["Graph::Options::Extra-Rrd"].split(","): f = open(i) extra_rrdtool_args.extend(f.read().strip().split("\n")) f.close() elif Cnf.has_key("Graph::Extra-Rrd"): for i in Cnf.value_list("Graph::Extra-Rrd"): f = open(i) extra_rrdtool_args.extend(f.read().strip().split("\n")) f.close() if Cnf.has_key("Graph::Options::Rrd"): rrd_dir = Cnf["Graph::Options::Rrd"] elif Cnf.has_key("Dir::Rrd"): rrd_dir = Cnf["Dir::Rrd"] else: print >> sys.stderr, "No directory to read RRD files from\n" sys.exit(1) if Cnf.has_key("Graph::Options::Images"): image_dir = Cnf["Graph::Options::Images"] else: print >> sys.stderr, "No directory to write graph images to\n" sys.exit(1) for name in names: stdargs = [rrd_dir, image_dir, name, extra_rrdtool_args] graph(*(stdargs+['day', 'day', 'now-1d'])) graph(*(stdargs+['week', 'week', 'now-1w'])) graph(*(stdargs+['month', 'month', 'now-1m'])) graph(*(stdargs+['year', 'year', 'now-1y'])) graph(*(stdargs+['5years', '5 years', 'now-5y', True])) graph(*(stdargs+['10years', '10 years', 'now-10y', True]))
def remove(session, reason, suites, removals, whoami=None, partial=False, components=None, done_bugs=None, date=None, carbon_copy=None, close_related_bugs=False): """Batch remove a number of packages Verify that the files listed in the Files field of the .dsc are those expected given the announced Format. @type session: SQLA Session @param session: The database session in use @type reason: string @param reason: The reason for the removal (e.g. "[auto-cruft] NBS (no longer built by <source>)") @type suites: list @param suites: A list of the suite names in which the removal should occur @type removals: list @param removals: A list of the removals. Each element should be a tuple (or list) of at least the following for 4 items from the database (in order): package, version, architecture, (database) id. For source packages, the "architecture" should be set to "source". @type partial: bool @param partial: Whether the removal is "partial" (e.g. architecture specific). @type components: list @param components: List of components involved in a partial removal. Can be an empty list to not restrict the removal to any components. @type whoami: string @param whoami: The person (or entity) doing the removal. Defaults to utils.whoami() @type date: string @param date: The date of the removal. Defaults to commands.getoutput("date -R") @type done_bugs: list @param done_bugs: A list of bugs to be closed when doing this removal. @type close_related_bugs: bool @param done_bugs: Whether bugs related to the package being removed should be closed as well. NB: Not implemented for more than one suite. @type carbon_copy: list @param carbon_copy: A list of mail addresses to CC when doing removals. NB: all items are taken "as-is" unlike "dak rm". @rtype: None @return: Nothing """ # Generate the summary of what's to be removed d = {} summary = "" sources = [] binaries = [] whitelists = [] versions = [] suite_ids_list = [] suites_list = utils.join_with_commas_and(suites) cnf = utils.get_conf() con_components = '' ####################################################################################################### if not reason: raise ValueError("Empty removal reason not permitted") if not removals: raise ValueError("Nothing to remove!?") if not suites: raise ValueError("Removals without a suite!?") if whoami is None: whoami = utils.whoami() if date is None: date = commands.getoutput("date -R") if partial and components: component_ids_list = [] for componentname in components: component = get_component(componentname, session=session) if component is None: raise ValueError("component '%s' not recognised." % componentname) else: component_ids_list.append(component.component_id) if component_ids_list: con_components = "AND component IN (%s)" % ", ".join( [str(i) for i in component_ids_list]) for i in removals: package = i[0] version = i[1] architecture = i[2] if package not in d: d[package] = {} if version not in d[package]: d[package][version] = [] if architecture not in d[package][version]: d[package][version].append(architecture) for package in sorted(d): versions = sorted(d[package], cmp=apt_pkg.version_compare) for version in versions: d[package][version].sort(utils.arch_compare_sw) summary += "%10s | %10s | %s\n" % (package, version, ", ".join( d[package][version])) for package in summary.split("\n"): for row in package.split("\n"): element = row.split("|") if len(element) == 3: if element[2].find("source") > 0: sources.append( "%s_%s" % tuple(elem.strip(" ") for elem in element[:2])) element[2] = sub("source\s?,?", "", element[2]).strip(" ") if element[2]: binaries.append("%s_%s [%s]" % tuple(elem.strip(" ") for elem in element)) dsc_type_id = get_override_type('dsc', session).overridetype_id deb_type_id = get_override_type('deb', session).overridetype_id for suite in suites: s = get_suite(suite, session=session) if s is not None: suite_ids_list.append(s.suite_id) whitelists.append(s.mail_whitelist) ####################################################################################################### log_filename = cnf["Rm::LogFile"] log822_filename = cnf["Rm::LogFile822"] with utils.open_file(log_filename, "a") as logfile, utils.open_file( log822_filename, "a") as logfile822: fcntl.lockf(logfile, fcntl.LOCK_EX) fcntl.lockf(logfile822, fcntl.LOCK_EX) logfile.write( "=========================================================================\n" ) logfile.write("[Date: %s] [ftpmaster: %s]\n" % (date, whoami)) logfile.write("Removed the following packages from %s:\n\n%s" % (suites_list, summary)) if done_bugs: logfile.write("Closed bugs: %s\n" % (", ".join(done_bugs))) logfile.write( "\n------------------- Reason -------------------\n%s\n" % reason) logfile.write("----------------------------------------------\n") logfile822.write("Date: %s\n" % date) logfile822.write("Ftpmaster: %s\n" % whoami) logfile822.write("Suite: %s\n" % suites_list) if sources: logfile822.write("Sources:\n") for source in sources: logfile822.write(" %s\n" % source) if binaries: logfile822.write("Binaries:\n") for binary in binaries: logfile822.write(" %s\n" % binary) logfile822.write("Reason: %s\n" % reason.replace('\n', '\n ')) if done_bugs: logfile822.write("Bug: %s\n" % (", ".join(done_bugs))) for i in removals: package = i[0] architecture = i[2] package_id = i[3] for suite_id in suite_ids_list: if architecture == "source": session.execute( "DELETE FROM src_associations WHERE source = :packageid AND suite = :suiteid", { 'packageid': package_id, 'suiteid': suite_id }) else: session.execute( "DELETE FROM bin_associations WHERE bin = :packageid AND suite = :suiteid", { 'packageid': package_id, 'suiteid': suite_id }) # Delete from the override file if not partial: if architecture == "source": type_id = dsc_type_id else: type_id = deb_type_id # TODO: Fix this properly to remove the remaining non-bind argument session.execute( "DELETE FROM override WHERE package = :package AND type = :typeid AND suite = :suiteid %s" % (con_components), { 'package': package, 'typeid': type_id, 'suiteid': suite_id }) session.commit() # ### REMOVAL COMPLETE - send mail time ### # # If we don't have a Bug server configured, we're done if "Dinstall::BugServer" not in cnf: if done_bugs or close_related_bugs: utils.warn( "Cannot send mail to BugServer as Dinstall::BugServer is not configured" ) logfile.write( "=========================================================================\n" ) logfile822.write("\n") return # read common subst variables for all bug closure mails Subst_common = {} Subst_common["__RM_ADDRESS__"] = cnf["Dinstall::MyEmailAddress"] Subst_common["__BUG_SERVER__"] = cnf["Dinstall::BugServer"] Subst_common["__CC__"] = "X-DAK: dak rm" if carbon_copy: Subst_common["__CC__"] += "\nCc: " + ", ".join(carbon_copy) Subst_common["__SUITE_LIST__"] = suites_list Subst_common["__SUBJECT__"] = "Removed package(s) from %s" % ( suites_list) Subst_common["__ADMIN_ADDRESS__"] = cnf["Dinstall::MyAdminAddress"] Subst_common["__DISTRO__"] = cnf["Dinstall::MyDistribution"] Subst_common["__WHOAMI__"] = whoami # Send the bug closing messages if done_bugs: Subst_close_rm = Subst_common bcc = [] if cnf.find("Dinstall::Bcc") != "": bcc.append(cnf["Dinstall::Bcc"]) if cnf.find("Rm::Bcc") != "": bcc.append(cnf["Rm::Bcc"]) if bcc: Subst_close_rm["__BCC__"] = "Bcc: " + ", ".join(bcc) else: Subst_close_rm["__BCC__"] = "X-Filler: 42" summarymail = "%s\n------------------- Reason -------------------\n%s\n" % ( summary, reason) summarymail += "----------------------------------------------\n" Subst_close_rm["__SUMMARY__"] = summarymail for bug in done_bugs: Subst_close_rm["__BUG_NUMBER__"] = bug if close_related_bugs: mail_message = utils.TemplateSubst( Subst_close_rm, cnf["Dir::Templates"] + "/rm.bug-close-with-related") else: mail_message = utils.TemplateSubst( Subst_close_rm, cnf["Dir::Templates"] + "/rm.bug-close") utils.send_mail(mail_message, whitelists=whitelists) # close associated bug reports if close_related_bugs: Subst_close_other = Subst_common bcc = [] wnpp = utils.parse_wnpp_bug_file() versions = list(set([re_bin_only_nmu.sub('', v) for v in versions])) if len(versions) == 1: Subst_close_other["__VERSION__"] = versions[0] else: logfile.write( "=========================================================================\n" ) logfile822.write("\n") raise ValueError( "Closing bugs with multiple package versions is not supported. Do it yourself." ) if bcc: Subst_close_other["__BCC__"] = "Bcc: " + ", ".join(bcc) else: Subst_close_other["__BCC__"] = "X-Filler: 42" # at this point, I just assume, that the first closed bug gives # some useful information on why the package got removed Subst_close_other["__BUG_NUMBER__"] = done_bugs[0] if len(sources) == 1: source_pkg = source.split("_", 1)[0] else: logfile.write( "=========================================================================\n" ) logfile822.write("\n") raise ValueError( "Closing bugs for multiple source packages is not supported. Please do it yourself." ) Subst_close_other["__BUG_NUMBER_ALSO__"] = "" Subst_close_other["__SOURCE__"] = source_pkg merged_bugs = set() other_bugs = bts.get_bugs('src', source_pkg, 'status', 'open', 'status', 'forwarded') if other_bugs: for bugno in other_bugs: if bugno not in merged_bugs: for bug in bts.get_status(bugno): for merged in bug.mergedwith: other_bugs.remove(merged) merged_bugs.add(merged) logfile.write("Also closing bug(s):") logfile822.write("Also-Bugs:") for bug in other_bugs: Subst_close_other["__BUG_NUMBER_ALSO__"] += str( bug) + "-done@" + cnf["Dinstall::BugServer"] + "," logfile.write(" " + str(bug)) logfile822.write(" " + str(bug)) logfile.write("\n") logfile822.write("\n") if source_pkg in wnpp: logfile.write("Also closing WNPP bug(s):") logfile822.write("Also-WNPP:") for bug in wnpp[source_pkg]: # the wnpp-rm file we parse also contains our removal # bugs, filtering that out if bug != Subst_close_other["__BUG_NUMBER__"]: Subst_close_other["__BUG_NUMBER_ALSO__"] += str( bug) + "-done@" + cnf["Dinstall::BugServer"] + "," logfile.write(" " + str(bug)) logfile822.write(" " + str(bug)) logfile.write("\n") logfile822.write("\n") mail_message = utils.TemplateSubst( Subst_close_other, cnf["Dir::Templates"] + "/rm.bug-close-related") if Subst_close_other["__BUG_NUMBER_ALSO__"]: utils.send_mail(mail_message) logfile.write( "=========================================================================\n" ) logfile822.write("\n")
def main(): global Cnf, Options, Logger os.umask(0o002) Cnf = utils.get_conf() Arguments = [('h', "help", "Generate-Index-Diffs::Options::Help"), ('a', 'archive', 'Generate-Index-Diffs::Options::Archive', 'hasArg'), ('c', None, "Generate-Index-Diffs::Options::CanonicalPath", "hasArg"), ('p', "patchname", "Generate-Index-Diffs::Options::PatchName", "hasArg"), ('d', "tmpdir", "Generate-Index-Diffs::Options::TempDir", "hasArg"), ('m', "maxdiffs", "Generate-Index-Diffs::Options::MaxDiffs", "hasArg"), ('n', "no-act", "Generate-Index-Diffs::Options::NoAct"), ('v', "verbose", "Generate-Index-Diffs::Options::Verbose"), ] suites = apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Generate-Index-Diffs::Options") if "Help" in Options: usage() maxdiffs = Options.get("MaxDiffs::Default", "56") maxpackages = Options.get("MaxDiffs::Packages", maxdiffs) maxcontents = Options.get("MaxDiffs::Contents", maxdiffs) maxsources = Options.get("MaxDiffs::Sources", maxdiffs) # can only be set via config at the moment max_parallel = int(Options.get("MaxParallel", "8")) if "PatchName" not in Options: format = "%Y-%m-%d-%H%M.%S" Options["PatchName"] = time.strftime(format) session = DBConn().session() pending_tasks = [] if not suites: query = session.query(Suite.suite_name) if Options.get('Archive'): archives = utils.split_args(Options['Archive']) query = query.join(Suite.archive).filter(Archive.archive_name.in_(archives)) suites = [s.suite_name for s in query] for suitename in suites: print("Processing: " + suitename) suiteobj = get_suite(suitename.lower(), session=session) # Use the canonical version of the suite name suite = suiteobj.suite_name if suiteobj.untouchable: print("Skipping: " + suite + " (untouchable)") continue skip_all = True if suiteobj.separate_contents_architecture_all or suiteobj.separate_packages_architecture_all: skip_all = False architectures = get_suite_architectures(suite, skipall=skip_all, session=session) components = [c.component_name for c in session.query(Component.component_name)] suite_suffix = utils.suite_suffix(suitename) if components and suite_suffix: longsuite = suite + "/" + suite_suffix else: longsuite = suite merged_pdiffs = suiteobj.merged_pdiffs tree = os.path.join(suiteobj.archive.path, 'dists', longsuite) # See if there are Translations which might need a new pdiff cwd = os.getcwd() for component in components: workpath = os.path.join(tree, component, "i18n") if os.path.isdir(workpath): os.chdir(workpath) for dirpath, dirnames, filenames in os.walk(".", followlinks=True, topdown=True): for entry in filenames: if not re_includeinpdiff.match(entry): continue (fname, fext) = os.path.splitext(entry) processfile = os.path.join(workpath, fname) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, fname) coroutine = genchanges(Options, processfile + ".diff", storename, processfile, maxdiffs, merged_pdiffs) pending_tasks.append(coroutine) os.chdir(cwd) for archobj in architectures: architecture = archobj.arch_string if architecture == "source": longarch = architecture packages = "Sources" maxsuite = maxsources else: longarch = "binary-%s" % architecture packages = "Packages" maxsuite = maxpackages for component in components: # Process Contents file = "%s/%s/Contents-%s" % (tree, component, architecture) storename = "%s/%s_%s_contents_%s" % (Options["TempDir"], suite, component, architecture) coroutine = genchanges(Options, file + ".diff", storename, file, maxcontents, merged_pdiffs) pending_tasks.append(coroutine) file = "%s/%s/%s/%s" % (tree, component, longarch, packages) storename = "%s/%s_%s_%s" % (Options["TempDir"], suite, component, architecture) coroutine = genchanges(Options, file + ".diff", storename, file, maxsuite, merged_pdiffs) pending_tasks.append(coroutine) asyncio.run(process_pdiff_tasks(pending_tasks, max_parallel))
def main(): global Cnf Cnf = utils.get_conf() Arguments = [('h', "help", "Queue-Report::Options::Help"), ('n', "new", "Queue-Report::Options::New"), ('8', '822', "Queue-Report::Options::822"), ('s', "sort", "Queue-Report::Options::Sort", "HasArg"), ('a', "age", "Queue-Report::Options::Age", "HasArg"), ('r', "rrd", "Queue-Report::Options::Rrd", "HasArg"), ('d', "directories", "Queue-Report::Options::Directories", "HasArg")] for i in ["help"]: key = "Queue-Report::Options::%s" % i if key not in Cnf: Cnf[key] = "" apt_pkg.parse_commandline(Cnf, Arguments, sys.argv) Options = Cnf.subtree("Queue-Report::Options") if Options["Help"]: usage() if "Queue-Report::Options::New" in Cnf: header() queue_names = [] if "Queue-Report::Options::Directories" in Cnf: for i in Cnf["Queue-Report::Options::Directories"].split(","): queue_names.append(i) elif "Queue-Report::Directories" in Cnf: queue_names = Cnf.value_list("Queue-Report::Directories") else: queue_names = ["byhand", "new"] if "Queue-Report::Options::Rrd" in Cnf: rrd_dir = Cnf["Queue-Report::Options::Rrd"] elif "Dir::Rrd" in Cnf: rrd_dir = Cnf["Dir::Rrd"] else: rrd_dir = None f = None if "Queue-Report::Options::822" in Cnf: # Open the report file f = sys.stdout filename822 = Cnf.get("Queue-Report::ReportLocations::822Location") if filename822: f = open(filename822, "w") session = DBConn().session() for queue_name in queue_names: queue = session.query(PolicyQueue).filter_by( queue_name=queue_name).first() if queue is not None: process_queue(queue, f, rrd_dir) else: utils.warn("Cannot find queue %s" % queue_name) if "Queue-Report::Options::822" in Cnf: f.close() if "Queue-Report::Options::New" in Cnf: footer()
def britney_changelog(packages, suite, session): old = {} current = {} Cnf = utils.get_conf() try: q = session.execute("SELECT changelog FROM suite WHERE id = :suiteid", {'suiteid': suite.suite_id}) brit_file = q.fetchone()[0] except: brit_file = None if brit_file: brit_file = os.path.join(Cnf['Dir::Root'], brit_file) else: return q = session.execute( """SELECT s.source, s.version, sa.id FROM source s, src_associations sa WHERE sa.suite = :suiteid AND sa.source = s.id""", {'suiteid': suite.suite_id}) for p in q.fetchall(): current[p[0]] = p[1] for p in packages.keys(): if p[2] == "source": old[p[0]] = p[1] new = {} for p in current.keys(): if p in old.keys(): if apt_pkg.version_compare(current[p], old[p]) > 0: new[p] = [current[p], old[p]] else: new[p] = [current[p], 0] query = "SELECT source, changelog FROM changelogs WHERE" for p in new.keys(): query += " source = '%s' AND version > '%s' AND version <= '%s'" \ % (p, new[p][1], new[p][0]) query += " AND architecture LIKE '%source%' AND distribution in \ ('unstable', 'experimental', 'testing-proposed-updates') OR" query += " False ORDER BY source, version DESC" q = session.execute(query) pu = None brit = utils.open_file(brit_file, 'w') for u in q: if pu and pu != u[0]: brit.write("\n") brit.write("%s\n" % u[1]) pu = u[0] if q.rowcount: brit.write("\n\n\n") for p in list(set(old.keys()).difference(current.keys())): brit.write("REMOVED: %s %s\n" % (p, old[p])) brit.flush() brit.close()