def garbage_collect(status, conf, session, mirror):
"""update stage: list db and remove disappeared and expired packages"""
logging.info("garbage collection...")
for version in session.query(Package).filter(not_(Package.sticky)):
pkg = SourcePackage.from_db_model(version)
pkg_id = (pkg["package"], pkg["version"])
pkgdir = pkg.extraction_dir(conf["sources_dir"])
if pkg_id not in mirror.packages:
# package is in in Debsources db, but gone from mirror: we
# might have to garbage collect it (depending on expiry)
expire_days = conf["expire_days"]
age = None
if pkgdir.exists():
age = datetime.now() - datetime.fromtimestamp(
os.path.getmtime(pkgdir))
if not age or age.days >= expire_days:
_rm_package(pkg, conf, session, db_package=version)
else:
logging.debug("not removing %s as it is too young" % pkg)
if conf["force_triggers"]:
try:
notify_plugins(
conf["observers"],
"rm-package",
session,
pkg,
pkgdir,
triggers=conf["force_triggers"],
dry=conf["dry_run"],
)
except Exception:
logging.exception("trigger failure on %s" % pkg)
def garbage_collect(status, conf, session, mirror):
"""update stage: list db and remove disappeared and expired packages
"""
logging.info('garbage collection...')
for version in session.query(Package).filter(not_(Package.sticky)):
pkg = SourcePackage.from_db_model(version)
pkg_id = (pkg['package'], pkg['version'])
pkgdir = pkg.extraction_dir(conf['sources_dir'])
if pkg_id not in mirror.packages:
# package is in in Debsources db, but gone from mirror: we
# might have to garbage collect it (depending on expiry)
expire_days = conf['expire_days']
age = None
if os.path.exists(pkgdir):
age = datetime.now() - \
datetime.fromtimestamp(os.path.getmtime(pkgdir))
if not age or age.days >= expire_days:
_rm_package(pkg, conf, session, db_package=version)
else:
logging.debug('not removing %s as it is too young' % pkg)
if conf['force_triggers']:
try:
notify_plugins(conf['observers'], 'rm-package',
session, pkg, pkgdir,
triggers=conf['force_triggers'],
dry=conf['dry_run'])
except:
logging.exception('trigger failure on %s' % pkg)
def _get_debian_path(self, session, package, version, sources_dir):
"""
Returns the Debian path of a package version.
For example: main/h
contrib/libz
It's the path of a *version*, since a package can have multiple
versions in multiple areas (ie main/contrib/nonfree).
sources_dir: the sources directory, usually comes from the app config
"""
prefix = SourcePackage.pkg_prefix(package)
try:
p_id = session.query(PackageName) \
.filter(PackageName.name == package).first().id
varea = session.query(Package) \
.filter(and_(Package.name_id == p_id,
Package.version == version)) \
.first().area
except:
# the package or version doesn't exist in the database
# BUT: packages are stored for a longer time in the filesystem
# to allow codesearch.d.n and others less up-to-date platforms
# to point here.
# Problem: we don't know the area of such a package
# so we try in main, contrib and non-free.
for area in AREAS:
if os.path.exists(os.path.join(sources_dir, area,
prefix, package, version)):
return os.path.join(area, prefix)
raise InvalidPackageOrVersionError("%s %s" % (package, version))
return os.path.join(varea, prefix)
def _get_debian_path(self, session, package, version, sources_dir) -> Path:
"""
Returns the Debian path of a package version.
For example: main/h
contrib/libz
It's the path of a *version*, since a package can have multiple
versions in multiple areas (ie main/contrib/nonfree).
sources_dir: the sources directory, usually comes from the app config
"""
prefix = SourcePackage.pkg_prefix(package)
try:
p_id = (session.query(PackageName).filter(
PackageName.name == package).first().id)
varea = (session.query(Package).filter(
and_(Package.name_id == p_id,
Package.version == version)).first().area)
except Exception:
# the package or version doesn't exist in the database
# BUT: packages are stored for a longer time in the filesystem
# to allow codesearch.d.n and others less up-to-date platforms
# to point here.
# Problem: we don't know the area of such a package
# so we try in main, contrib and non-free.
for area in AREAS:
if Path.exists(
Path(sources_dir) / area / prefix / package / version):
return Path(area) / prefix
raise InvalidPackageOrVersionError("%s %s" % (package, version))
return Path(varea) / prefix
def remove_suite(conf, session, suite):
logging.info('remove sticky suite %s from the archive...' % suite)
db_suite = db_storage.lookup_db_suite(session, suite, sticky=True)
if not db_suite:
logging.error('sticky suite %s does not exist in DB, abort.' % suite)
return
sticky_suites = statistics.sticky_suites(session)
if updater.STAGE_GC in conf['stages']:
for package in session.query(Package) \
.join(Suite) \
.filter(Suite.suite == suite) \
.filter(Package.sticky):
pkg = SourcePackage.from_db_model(package)
other_suites = \
session.query(Suite.suite.distinct()) \
.filter(Suite.package_id == package.id) \
.filter(Suite.suite != suite)
other_suites = [row[0] for row in other_suites]
if not other_suites:
if not conf['single_transaction']:
with session.begin():
updater._rm_package(pkg,
conf,
session,
db_package=package)
else:
updater._rm_package(pkg, conf, session, db_package=package)
else:
other_sticky_suites = [
s for s in other_suites if s in sticky_suites
]
if not other_sticky_suites and not conf['dry_run']:
# package is only listed in "live" suites, drop sticky flag
logging.debug('clearing sticky bit on %s' % pkg)
package.sticky = False
suitemap = db_storage.lookup_suitemapping(session, package, suite)
if suitemap and not conf['dry_run']:
session.delete(suitemap)
if not conf['dry_run']:
session.delete(db_suite)
_remove_stats_for(conf, session, suite)
logging.info('sticky suite %s removed from the archive.' % suite)
def remove_suite(conf, session, suite):
logging.info('remove sticky suite %s from the archive...' % suite)
db_suite = db_storage.lookup_db_suite(session, suite, sticky=True)
if not db_suite:
logging.error('sticky suite %s does not exist in DB, abort.' % suite)
return
sticky_suites = statistics.sticky_suites(session)
if updater.STAGE_GC in conf['stages']:
for package in session.query(Package) \
.join(Suite) \
.filter(Suite.suite == suite) \
.filter(Package.sticky):
pkg = SourcePackage.from_db_model(package)
other_suites = \
session.query(Suite.suite.distinct()) \
.filter(Suite.package_id == package.id) \
.filter(Suite.suite != suite)
other_suites = [row[0] for row in other_suites]
if not other_suites:
if not conf['single_transaction']:
with session.begin():
updater._rm_package(pkg, conf, session,
db_package=package)
else:
updater._rm_package(pkg, conf, session, db_package=package)
else:
other_sticky_suites = [s for s in other_suites
if s in sticky_suites]
if not other_sticky_suites and not conf['dry_run']:
# package is only listed in "live" suites, drop sticky flag
logging.debug('clearing sticky bit on %s' % pkg)
package.sticky = False
suitemap = db_storage.lookup_suitemapping(session, package, suite)
if suitemap and not conf['dry_run']:
session.delete(suitemap)
if not conf['dry_run']:
session.delete(db_suite)
_remove_stats_for(conf, session, suite)
logging.info('sticky suite %s removed from the archive.' % suite)