def end(): accept_count = SummaryStats().accept_count accept_bytes = SummaryStats().accept_bytes if accept_count: sets = "set" if accept_count > 1: sets = "sets" print("Accepted %d package %s, %s." % (accept_count, sets, utils.size_type(int(accept_bytes))), file=sys.stderr) Logger.log(["total", accept_count, accept_bytes]) if not Options["No-Action"] and not Options["Trainee"]: Logger.close()
def end(): accept_count = SummaryStats().accept_count accept_bytes = SummaryStats().accept_bytes if accept_count: sets = "set" if accept_count > 1: sets = "sets" sys.stderr.write("Accepted %d package %s, %s.\n" % (accept_count, sets, utils.size_type(int(accept_bytes)))) Logger.log(["total",accept_count,accept_bytes]) if not Options["No-Action"] and not Options["Trainee"]: Logger.close()
def check_files(): """ Prepare the dictionary of existing filenames, then walk through the archive pool/ directory to compare it. """ global db_files cnf = Config() print "Building list of database files..." q = DBConn().session().query(PoolFile).join(Location).order_by('path', 'location') print "Missing files:" db_files.clear() for f in q.all(): filename = os.path.abspath(os.path.join(f.location.path, f.filename)) db_files[filename] = "" if os.access(filename, os.R_OK) == 0: if f.last_used: print "(last used: %s) %s" % (f.last_used, filename) else: print "%s" % (filename) filename = os.path.join(cnf["Dir::Override"], 'override.unreferenced') if os.path.exists(filename): f = utils.open_file(filename) for filename in f.readlines(): filename = filename[:-1] excluded[filename] = "" print "Existent files not in db:" os.path.walk(os.path.join(cnf["Dir::Root"], 'pool/'), process_dir, None) print print "%s wasted..." % (utils.size_type(waste))
def clean(now_date, archives, max_delete, session): cnf = Config() count = 0 size = 0 Logger.log(["Cleaning out packages..."]) morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool') # Build directory as morguedir/morguesubdir/year/month/day dest = os.path.join(morguedir, morguesubdir, str(now_date.year), '%.2d' % now_date.month, '%.2d' % now_date.day) if not Options["No-Action"] and not os.path.exists(dest): os.makedirs(dest) # Delete from source Logger.log(["Deleting from source table..."]) q = session.execute(""" WITH deleted_sources AS ( DELETE FROM source USING files f WHERE source.file = f.id AND NOT EXISTS (SELECT 1 FROM files_archive_map af JOIN archive_delete_date ad ON af.archive_id = ad.archive_id WHERE af.file_id = source.file AND (af.last_used IS NULL OR af.last_used > ad.delete_date)) RETURNING source.id AS id, f.filename AS filename ), deleted_dsc_files AS ( DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources) RETURNING df.file AS file_id ), now_unused_source_files AS ( UPDATE files_archive_map af SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc. WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files) AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id) ) SELECT filename FROM deleted_sources""") for s in q: Logger.log(["delete source", s[0]]) if not Options["No-Action"]: session.commit() # Delete files from the pool old_files = session.query(ArchiveFile).filter( sql.text( 'files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)' )).join(Archive) if max_delete is not None: old_files = old_files.limit(max_delete) Logger.log(["Limiting removals to %d" % max_delete]) if archives is not None: archive_ids = [a.archive_id for a in archives] old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids)) for af in old_files: filename = af.path try: st = os.lstat(filename) except FileNotFoundError: Logger.log(["database referred to non-existing file", filename]) session.delete(af) continue Logger.log(["delete archive file", filename]) if stat.S_ISLNK(st.st_mode): count += 1 Logger.log(["delete symlink", filename]) if not Options["No-Action"]: os.unlink(filename) session.delete(af) elif stat.S_ISREG(st.st_mode): size += st.st_size count += 1 dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.lexists(dest_filename): dest_filename = utils.find_next_free(dest_filename) if not Options["No-Action"]: if af.archive.use_morgue: Logger.log(["move to morgue", filename, dest_filename]) utils.move(filename, dest_filename) else: Logger.log(["removed file", filename]) os.unlink(filename) session.delete(af) else: utils.fubar("%s is neither symlink nor file?!" % (filename)) if count > 0: Logger.log(["total", count, utils.size_type(size)]) # Delete entries in files no longer referenced by any archive query = """ DELETE FROM files f WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id) """ session.execute(query) if not Options["No-Action"]: session.commit()
def clean(now_date, delete_date, max_delete, session): cnf = Config() count = 0 size = 0 print "Cleaning out packages..." morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool') # Build directory as morguedir/morguesubdir/year/month/day dest = os.path.join(morguedir, morguesubdir, str(now_date.year), '%.2d' % now_date.month, '%.2d' % now_date.day) if not Options["No-Action"] and not os.path.exists(dest): os.makedirs(dest) # Delete from source print "Deleting from source table... " q = session.execute(""" SELECT s.id, f.filename FROM source s, files f WHERE f.last_used <= :deletedate AND s.file = f.id AND s.id NOT IN (SELECT src_id FROM extra_src_references)""", {'deletedate': delete_date}) for s in q.fetchall(): Logger.log(["delete source", s[1], s[0]]) if not Options["No-Action"]: session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]}) session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]}) if not Options["No-Action"]: session.commit() # Delete files from the pool old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date) if max_delete is not None: old_files = old_files.limit(max_delete) print "Limiting removals to %d" % max_delete for pf in old_files: filename = os.path.join(pf.location.path, pf.filename) if not os.path.exists(filename): utils.warn("can not find '%s'." % (filename)) continue Logger.log(["delete pool file", filename]) if os.path.isfile(filename): if os.path.islink(filename): count += 1 Logger.log(["delete symlink", filename]) if not Options["No-Action"]: os.unlink(filename) else: size += os.stat(filename)[stat.ST_SIZE] count += 1 dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.exists(dest_filename): dest_filename = utils.find_next_free(dest_filename) Logger.log(["move to morgue", filename, dest_filename]) if not Options["No-Action"]: utils.move(filename, dest_filename) if not Options["No-Action"]: session.delete(pf) session.commit() else: utils.fubar("%s is neither symlink nor file?!" % (filename)) if count > 0: Logger.log(["total", count, utils.size_type(size)]) print "Cleaned %d files, %s." % (count, utils.size_type(size))
def main(): global Options, Logger cnf = Config() summarystats = SummaryStats() Arguments = [('a', "automatic", "Dinstall::Options::Automatic"), ('h', "help", "Dinstall::Options::Help"), ('n', "no-action", "Dinstall::Options::No-Action"), ('p', "no-lock", "Dinstall::Options::No-Lock"), ('s', "no-mail", "Dinstall::Options::No-Mail"), ('d', "directory", "Dinstall::Options::Directory", "HasArg")] for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version", "directory"]: key = "Dinstall::Options::%s" % i if key not in cnf: cnf[key] = "" changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Dinstall::Options") if Options["Help"]: usage() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: Options["Automatic"] = "" # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'process-upload.lock'), os.O_RDWR | os.O_CREAT) try: fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if e.errno in (errno.EACCES, errno.EAGAIN): utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise # Initialise UrgencyLog() - it will deal with the case where we don't # want to log urgencies urgencylog = UrgencyLog() Logger = daklog.Logger("process-upload", Options["No-Action"]) # If we have a directory flag, use it to find our files if cnf["Dinstall::Options::Directory"] != "": # Note that we clobber the list of files we were given in this case # so warn if the user has done both if len(changes_files) > 0: utils.warn("Directory provided so ignoring files given on command line") changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)]) elif not len(changes_files) > 0: utils.fubar("No changes files given and no directory specified") else: Logger.log(["Using changes files from command-line", len(changes_files)]) process_changes(changes_files) if summarystats.accept_count: sets = "set" if summarystats.accept_count > 1: sets = "sets" print("Installed %d package %s, %s." % (summarystats.accept_count, sets, utils.size_type(int(summarystats.accept_bytes)))) Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) if summarystats.reject_count: sets = "set" if summarystats.reject_count > 1: sets = "sets" print("Rejected %d package %s." % (summarystats.reject_count, sets)) Logger.log(["rejected", summarystats.reject_count]) if not Options["No-Action"]: urgencylog.close() Logger.close()
def clean(now_date, archives, max_delete, session): cnf = Config() count = 0 size = 0 Logger.log(["Cleaning out packages..."]) morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue')) morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool') # Build directory as morguedir/morguesubdir/year/month/day dest = os.path.join(morguedir, morguesubdir, str(now_date.year), '%.2d' % now_date.month, '%.2d' % now_date.day) if not Options["No-Action"] and not os.path.exists(dest): os.makedirs(dest) # Delete from source Logger.log(["Deleting from source table..."]) q = session.execute(""" WITH deleted_sources AS ( DELETE FROM source USING files f WHERE source.file = f.id AND NOT EXISTS (SELECT 1 FROM files_archive_map af JOIN archive_delete_date ad ON af.archive_id = ad.archive_id WHERE af.file_id = source.file AND (af.last_used IS NULL OR af.last_used > ad.delete_date)) RETURNING source.id AS id, f.filename AS filename ), deleted_dsc_files AS ( DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources) RETURNING df.file AS file_id ), now_unused_source_files AS ( UPDATE files_archive_map af SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc. WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files) AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id) ) SELECT filename FROM deleted_sources""") for s in q: Logger.log(["delete source", s[0]]) if not Options["No-Action"]: session.commit() # Delete files from the pool old_files = session.query(ArchiveFile).filter('files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)').join(Archive) if max_delete is not None: old_files = old_files.limit(max_delete) Logger.log(["Limiting removals to %d" % max_delete]) if archives is not None: archive_ids = [ a.archive_id for a in archives ] old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids)) for af in old_files: filename = af.path if not os.path.exists(filename): Logger.log(["database referred to non-existing file", af.path]) session.delete(af) continue Logger.log(["delete archive file", filename]) if os.path.isfile(filename): if os.path.islink(filename): count += 1 Logger.log(["delete symlink", filename]) if not Options["No-Action"]: os.unlink(filename) else: size += os.stat(filename)[stat.ST_SIZE] count += 1 dest_filename = dest + '/' + os.path.basename(filename) # If the destination file exists; try to find another filename to use if os.path.lexists(dest_filename): dest_filename = utils.find_next_free(dest_filename) if not Options["No-Action"]: if af.archive.use_morgue: Logger.log(["move to morgue", filename, dest_filename]) utils.move(filename, dest_filename) else: Logger.log(["removed file", filename]) os.unlink(filename) if not Options["No-Action"]: session.delete(af) session.commit() else: utils.fubar("%s is neither symlink nor file?!" % (filename)) if count > 0: Logger.log(["total", count, utils.size_type(size)]) # Delete entries in files no longer referenced by any archive query = """ DELETE FROM files f WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id) """ session.execute(query) if not Options["No-Action"]: session.commit()
def main(): global Options, Logger cnf = Config() summarystats = SummaryStats() DBConn() Arguments = [('a',"automatic","Dinstall::Options::Automatic"), ('h',"help","Dinstall::Options::Help"), ('n',"no-action","Dinstall::Options::No-Action"), ('p',"no-lock", "Dinstall::Options::No-Lock"), ('s',"no-mail", "Dinstall::Options::No-Mail"), ('d',"directory", "Dinstall::Options::Directory", "HasArg")] for i in ["automatic", "help", "no-action", "no-lock", "no-mail", "version", "directory"]: if not cnf.has_key("Dinstall::Options::%s" % (i)): cnf["Dinstall::Options::%s" % (i)] = "" changes_files = apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) Options = cnf.subtree("Dinstall::Options") if Options["Help"]: usage() # -n/--dry-run invalidates some other options which would involve things happening if Options["No-Action"]: Options["Automatic"] = "" # Check that we aren't going to clash with the daily cron job if not Options["No-Action"] and os.path.exists("%s/daily.lock" % (cnf["Dir::Lock"])) and not Options["No-Lock"]: utils.fubar("Archive maintenance in progress. Try again later.") # Obtain lock if not in no-action mode and initialize the log if not Options["No-Action"]: lock_fd = os.open(os.path.join(cnf["Dir::Lock"], 'dinstall.lock'), os.O_RDWR | os.O_CREAT) try: fcntl.lockf(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError as e: if errno.errorcode[e.errno] == 'EACCES' or errno.errorcode[e.errno] == 'EAGAIN': utils.fubar("Couldn't obtain lock; assuming another 'dak process-upload' is already running.") else: raise # Initialise UrgencyLog() - it will deal with the case where we don't # want to log urgencies urgencylog = UrgencyLog() Logger = daklog.Logger("process-upload", Options["No-Action"]) # If we have a directory flag, use it to find our files if cnf["Dinstall::Options::Directory"] != "": # Note that we clobber the list of files we were given in this case # so warn if the user has done both if len(changes_files) > 0: utils.warn("Directory provided so ignoring files given on command line") changes_files = utils.get_changes_files(cnf["Dinstall::Options::Directory"]) Logger.log(["Using changes files from directory", cnf["Dinstall::Options::Directory"], len(changes_files)]) elif not len(changes_files) > 0: utils.fubar("No changes files given and no directory specified") else: Logger.log(["Using changes files from command-line", len(changes_files)]) # Sort the .changes files so that we process sourceful ones first changes_files.sort(utils.changes_compare) # Process the changes files for changes_file in changes_files: print "\n" + changes_file session = DBConn().session() process_it(changes_file, session) session.close() if summarystats.accept_count: sets = "set" if summarystats.accept_count > 1: sets = "sets" print "Installed %d package %s, %s." % (summarystats.accept_count, sets, utils.size_type(int(summarystats.accept_bytes))) Logger.log(["total", summarystats.accept_count, summarystats.accept_bytes]) if summarystats.reject_count: sets = "set" if summarystats.reject_count > 1: sets = "sets" print "Rejected %d package %s." % (summarystats.reject_count, sets) Logger.log(["rejected", summarystats.reject_count]) byebye() if not Options["No-Action"]: urgencylog.close() Logger.close()