def remove (from_dir, f):
fname = os.path.basename(f)
if os.access(f, os.R_OK):
Logger.log(["move file to morgue", from_dir, fname, del_dir])
if Options["Verbose"]:
print "Removing '%s' (to '%s')." % (fname, del_dir)
if Options["No-Action"]:
return
dest_filename = os.path.join(del_dir, fname)
# If the destination file exists; try to find another filename to use
if os.path.exists(dest_filename):
dest_filename = utils.find_next_free(dest_filename, 10)
Logger.log(["change destination file name", os.path.basename(dest_filename)])
utils.move(f, dest_filename, 0o660)
else:
Logger.log(["skipping file because of permission problem", fname])
utils.warn("skipping '%s', permission denied." % fname)
def remove(from_dir, f):
fname = os.path.basename(f)
if os.access(f, os.R_OK):
Logger.log(["move file to morgue", from_dir, fname, del_dir])
if Options["Verbose"]:
print("Removing '%s' (to '%s')." % (fname, del_dir))
if Options["No-Action"]:
return
dest_filename = os.path.join(del_dir, fname)
# If the destination file exists; try to find another filename to use
if os.path.exists(dest_filename):
dest_filename = utils.find_next_free(dest_filename, 10)
Logger.log([
"change destination file name",
os.path.basename(dest_filename)
])
utils.move(f, dest_filename, 0o660)
else:
Logger.log(["skipping file because of permission problem", fname])
utils.warn("skipping '%s', permission denied." % fname)
def changes_to_queue(upload, srcqueue, destqueue, session):
"""Move a changes file to a different queue and mark as approved for the
source queue"""
try:
chg = session.query(DBChange).filter_by(changesname=os.path.basename(upload.pkg.changes_file)).one()
except NoResultFound:
return False
chg.approved_for_id = srcqueue.policy_queue_id
for f in chg.files:
# update the changes_pending_files row
f.queue = destqueue
# Only worry about unprocessed files
if not f.processed:
utils.move(os.path.join(srcqueue.path, f.filename), destqueue.path, perms=int(destqueue.perms, 8))
utils.move(os.path.join(srcqueue.path, upload.pkg.changes_file), destqueue.path, perms=int(destqueue.perms, 8))
chg.in_queue = destqueue
session.commit()
return True
def clean(now_date, archives, max_delete, session):
cnf = Config()
count = 0
size = 0
Logger.log(["Cleaning out packages..."])
morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue'))
morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool')
# Build directory as morguedir/morguesubdir/year/month/day
dest = os.path.join(morguedir, morguesubdir, str(now_date.year),
'%.2d' % now_date.month, '%.2d' % now_date.day)
if not Options["No-Action"] and not os.path.exists(dest):
os.makedirs(dest)
# Delete from source
Logger.log(["Deleting from source table..."])
q = session.execute("""
WITH
deleted_sources AS (
DELETE FROM source
USING files f
WHERE source.file = f.id
AND NOT EXISTS (SELECT 1 FROM files_archive_map af
JOIN archive_delete_date ad ON af.archive_id = ad.archive_id
WHERE af.file_id = source.file
AND (af.last_used IS NULL OR af.last_used > ad.delete_date))
RETURNING source.id AS id, f.filename AS filename
),
deleted_dsc_files AS (
DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources)
RETURNING df.file AS file_id
),
now_unused_source_files AS (
UPDATE files_archive_map af
SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc.
WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files)
AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id)
)
SELECT filename FROM deleted_sources""")
for s in q:
Logger.log(["delete source", s[0]])
if not Options["No-Action"]:
session.commit()
# Delete files from the pool
old_files = session.query(ArchiveFile).filter(
sql.text(
'files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)'
)).join(Archive)
if max_delete is not None:
old_files = old_files.limit(max_delete)
Logger.log(["Limiting removals to %d" % max_delete])
if archives is not None:
archive_ids = [a.archive_id for a in archives]
old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids))
for af in old_files:
filename = af.path
try:
st = os.lstat(filename)
except FileNotFoundError:
Logger.log(["database referred to non-existing file", filename])
session.delete(af)
continue
Logger.log(["delete archive file", filename])
if stat.S_ISLNK(st.st_mode):
count += 1
Logger.log(["delete symlink", filename])
if not Options["No-Action"]:
os.unlink(filename)
session.delete(af)
elif stat.S_ISREG(st.st_mode):
size += st.st_size
count += 1
dest_filename = dest + '/' + os.path.basename(filename)
# If the destination file exists; try to find another filename to use
if os.path.lexists(dest_filename):
dest_filename = utils.find_next_free(dest_filename)
if not Options["No-Action"]:
if af.archive.use_morgue:
Logger.log(["move to morgue", filename, dest_filename])
utils.move(filename, dest_filename)
else:
Logger.log(["removed file", filename])
os.unlink(filename)
session.delete(af)
else:
utils.fubar("%s is neither symlink nor file?!" % (filename))
if count > 0:
Logger.log(["total", count, utils.size_type(size)])
# Delete entries in files no longer referenced by any archive
query = """
DELETE FROM files f
WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id)
"""
session.execute(query)
if not Options["No-Action"]:
session.commit()
def clean(now_date, delete_date, max_delete, session):
cnf = Config()
count = 0
size = 0
print "Cleaning out packages..."
morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue'))
morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool')
# Build directory as morguedir/morguesubdir/year/month/day
dest = os.path.join(morguedir,
morguesubdir,
str(now_date.year),
'%.2d' % now_date.month,
'%.2d' % now_date.day)
if not Options["No-Action"] and not os.path.exists(dest):
os.makedirs(dest)
# Delete from source
print "Deleting from source table... "
q = session.execute("""
SELECT s.id, f.filename FROM source s, files f
WHERE f.last_used <= :deletedate
AND s.file = f.id
AND s.id NOT IN (SELECT src_id FROM extra_src_references)""", {'deletedate': delete_date})
for s in q.fetchall():
Logger.log(["delete source", s[1], s[0]])
if not Options["No-Action"]:
session.execute("DELETE FROM dsc_files WHERE source = :s_id", {"s_id":s[0]})
session.execute("DELETE FROM source WHERE id = :s_id", {"s_id":s[0]})
if not Options["No-Action"]:
session.commit()
# Delete files from the pool
old_files = session.query(PoolFile).filter(PoolFile.last_used <= delete_date)
if max_delete is not None:
old_files = old_files.limit(max_delete)
print "Limiting removals to %d" % max_delete
for pf in old_files:
filename = os.path.join(pf.location.path, pf.filename)
if not os.path.exists(filename):
utils.warn("can not find '%s'." % (filename))
continue
Logger.log(["delete pool file", filename])
if os.path.isfile(filename):
if os.path.islink(filename):
count += 1
Logger.log(["delete symlink", filename])
if not Options["No-Action"]:
os.unlink(filename)
else:
size += os.stat(filename)[stat.ST_SIZE]
count += 1
dest_filename = dest + '/' + os.path.basename(filename)
# If the destination file exists; try to find another filename to use
if os.path.exists(dest_filename):
dest_filename = utils.find_next_free(dest_filename)
Logger.log(["move to morgue", filename, dest_filename])
if not Options["No-Action"]:
utils.move(filename, dest_filename)
if not Options["No-Action"]:
session.delete(pf)
session.commit()
else:
utils.fubar("%s is neither symlink nor file?!" % (filename))
if count > 0:
Logger.log(["total", count, utils.size_type(size)])
print "Cleaned %d files, %s." % (count, utils.size_type(size))
def clean(now_date, archives, max_delete, session):
cnf = Config()
count = 0
size = 0
Logger.log(["Cleaning out packages..."])
morguedir = cnf.get("Dir::Morgue", os.path.join("Dir::Pool", 'morgue'))
morguesubdir = cnf.get("Clean-Suites::MorgueSubDir", 'pool')
# Build directory as morguedir/morguesubdir/year/month/day
dest = os.path.join(morguedir,
morguesubdir,
str(now_date.year),
'%.2d' % now_date.month,
'%.2d' % now_date.day)
if not Options["No-Action"] and not os.path.exists(dest):
os.makedirs(dest)
# Delete from source
Logger.log(["Deleting from source table..."])
q = session.execute("""
WITH
deleted_sources AS (
DELETE FROM source
USING files f
WHERE source.file = f.id
AND NOT EXISTS (SELECT 1 FROM files_archive_map af
JOIN archive_delete_date ad ON af.archive_id = ad.archive_id
WHERE af.file_id = source.file
AND (af.last_used IS NULL OR af.last_used > ad.delete_date))
RETURNING source.id AS id, f.filename AS filename
),
deleted_dsc_files AS (
DELETE FROM dsc_files df WHERE df.source IN (SELECT id FROM deleted_sources)
RETURNING df.file AS file_id
),
now_unused_source_files AS (
UPDATE files_archive_map af
SET last_used = '1977-03-13 13:37:42' -- Kill it now. We waited long enough before removing the .dsc.
WHERE af.file_id IN (SELECT file_id FROM deleted_dsc_files)
AND NOT EXISTS (SELECT 1 FROM dsc_files df WHERE df.file = af.file_id)
)
SELECT filename FROM deleted_sources""")
for s in q:
Logger.log(["delete source", s[0]])
if not Options["No-Action"]:
session.commit()
# Delete files from the pool
old_files = session.query(ArchiveFile).filter('files_archive_map.last_used <= (SELECT delete_date FROM archive_delete_date ad WHERE ad.archive_id = files_archive_map.archive_id)').join(Archive)
if max_delete is not None:
old_files = old_files.limit(max_delete)
Logger.log(["Limiting removals to %d" % max_delete])
if archives is not None:
archive_ids = [ a.archive_id for a in archives ]
old_files = old_files.filter(ArchiveFile.archive_id.in_(archive_ids))
for af in old_files:
filename = af.path
if not os.path.exists(filename):
Logger.log(["database referred to non-existing file", af.path])
session.delete(af)
continue
Logger.log(["delete archive file", filename])
if os.path.isfile(filename):
if os.path.islink(filename):
count += 1
Logger.log(["delete symlink", filename])
if not Options["No-Action"]:
os.unlink(filename)
else:
size += os.stat(filename)[stat.ST_SIZE]
count += 1
dest_filename = dest + '/' + os.path.basename(filename)
# If the destination file exists; try to find another filename to use
if os.path.lexists(dest_filename):
dest_filename = utils.find_next_free(dest_filename)
if not Options["No-Action"]:
if af.archive.use_morgue:
Logger.log(["move to morgue", filename, dest_filename])
utils.move(filename, dest_filename)
else:
Logger.log(["removed file", filename])
os.unlink(filename)
if not Options["No-Action"]:
session.delete(af)
session.commit()
else:
utils.fubar("%s is neither symlink nor file?!" % (filename))
if count > 0:
Logger.log(["total", count, utils.size_type(size)])
# Delete entries in files no longer referenced by any archive
query = """
DELETE FROM files f
WHERE NOT EXISTS (SELECT 1 FROM files_archive_map af WHERE af.file_id = f.id)
"""
session.execute(query)
if not Options["No-Action"]:
session.commit()