def __target_suites(session, suites=None):
if not suites:
sticky_suites = statistics.sticky_suites(session)
suites = [suite
for suite in statistics.suites(session, suites='all')
if suite not in sticky_suites]
return suites
def remove_suite(conf, session, suite):
logging.info('remove sticky suite %s from the archive...' % suite)
db_suite = db_storage.lookup_db_suite(session, suite, sticky=True)
if not db_suite:
logging.error('sticky suite %s does not exist in DB, abort.' % suite)
return
sticky_suites = statistics.sticky_suites(session)
if updater.STAGE_GC in conf['stages']:
for package in session.query(Package) \
.join(Suite) \
.filter(Suite.suite == suite) \
.filter(Package.sticky):
pkg = SourcePackage.from_db_model(package)
other_suites = \
session.query(Suite.suite.distinct()) \
.filter(Suite.package_id == package.id) \
.filter(Suite.suite != suite)
other_suites = [row[0] for row in other_suites]
if not other_suites:
if not conf['single_transaction']:
with session.begin():
updater._rm_package(pkg,
conf,
session,
db_package=package)
else:
updater._rm_package(pkg, conf, session, db_package=package)
else:
other_sticky_suites = [
s for s in other_suites if s in sticky_suites
]
if not other_sticky_suites and not conf['dry_run']:
# package is only listed in "live" suites, drop sticky flag
logging.debug('clearing sticky bit on %s' % pkg)
package.sticky = False
suitemap = db_storage.lookup_suitemapping(session, package, suite)
if suitemap and not conf['dry_run']:
session.delete(suitemap)
if not conf['dry_run']:
session.delete(db_suite)
_remove_stats_for(conf, session, suite)
logging.info('sticky suite %s removed from the archive.' % suite)
def update_license_charts():
# License: historical histogramms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
mseries = getattr(statistics, 'history_copyright_' +
granularity)(session,
interval=period,
suite=suite)
chart_file = os.path.join(
conf['cache_dir'], 'stats', 'copyright_%s-license-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.multiseries_plot(mseries, chart_file, cols=3)
# License: overall pie chart
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, 'ALL'))
ratio = qry.get_ratio(session)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_overall-license_pie.png')
if not conf['dry_run']:
charts.pie_chart(overall_licenses, chart_file, ratio)
# License: bar chart and per suite pie chart.
all_suites = statistics.sticky_suites(session) \
+ __target_suites(session, None)
licenses_per_suite = []
results = statistics.get_licenses(session)
for suite in all_suites:
temp = dict(
(item[0], item[2]) for item in results if item[1] == suite)
licenses = statistics.licenses_summary(temp)
ratio = qry.get_ratio(session, suite=suite)
# draw license pie chart
if not conf['dry_run']:
chart_file = os.path.join(
conf['cache_dir'], 'stats', 'copyright_%s'
'-license_pie-current.png' % suite)
charts.pie_chart(licenses, chart_file, ratio)
licenses_per_suite.append(licenses)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_license_bar_plot.png')
if not conf['dry_run']:
charts.bar_chart(licenses_per_suite, all_suites, chart_file,
top_langs, 'Number of files')
def update_license_charts():
# License: historical histogramms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
mseries = getattr(statistics,
'history_copyright_' + granularity)(
session, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_%s-license-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.multiseries_plot(mseries, chart_file, cols=3)
# License: overall pie chart
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, 'ALL'))
ratio = qry.get_ratio(session)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_overall-license_pie.png')
if not conf['dry_run']:
charts.pie_chart(overall_licenses, chart_file, ratio)
# License: bar chart and per suite pie chart.
all_suites = statistics.sticky_suites(session) \
+ __target_suites(session, None)
licenses_per_suite = []
results = statistics.get_licenses(session)
for suite in all_suites:
temp = dict((item[0], item[2]) for item in results
if item[1] == suite)
licenses = statistics.licenses_summary(temp)
ratio = qry.get_ratio(session, suite=suite)
# draw license pie chart
if not conf['dry_run']:
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_%s'
'-license_pie-current.png'
% suite)
charts.pie_chart(licenses, chart_file, ratio)
licenses_per_suite.append(licenses)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_license_bar_plot.png')
if not conf['dry_run']:
charts.bar_chart(licenses_per_suite, all_suites, chart_file,
top_langs, 'Number of files')
def remove_suite(conf, session, suite):
logging.info('remove sticky suite %s from the archive...' % suite)
db_suite = db_storage.lookup_db_suite(session, suite, sticky=True)
if not db_suite:
logging.error('sticky suite %s does not exist in DB, abort.' % suite)
return
sticky_suites = statistics.sticky_suites(session)
if updater.STAGE_GC in conf['stages']:
for package in session.query(Package) \
.join(Suite) \
.filter(Suite.suite == suite) \
.filter(Package.sticky):
pkg = SourcePackage.from_db_model(package)
other_suites = \
session.query(Suite.suite.distinct()) \
.filter(Suite.package_id == package.id) \
.filter(Suite.suite != suite)
other_suites = [row[0] for row in other_suites]
if not other_suites:
if not conf['single_transaction']:
with session.begin():
updater._rm_package(pkg, conf, session,
db_package=package)
else:
updater._rm_package(pkg, conf, session, db_package=package)
else:
other_sticky_suites = [s for s in other_suites
if s in sticky_suites]
if not other_sticky_suites and not conf['dry_run']:
# package is only listed in "live" suites, drop sticky flag
logging.debug('clearing sticky bit on %s' % pkg)
package.sticky = False
suitemap = db_storage.lookup_suitemapping(session, package, suite)
if suitemap and not conf['dry_run']:
session.delete(suitemap)
if not conf['dry_run']:
session.delete(db_suite)
_remove_stats_for(conf, session, suite)
logging.info('sticky suite %s removed from the archive.' % suite)
def update_license_charts():
# License: historical histogramms
for (period, granularity) in CHARTS:
for suite in suites + ["ALL"]:
mseries = getattr(statistics, "history_copyright_" +
granularity)(session,
interval=period,
suite=suite)
filename = "copyright_%s-license-%s.png" % (
suite,
period.replace(" ", "-"),
)
chart_file = conf["cache_dir"] / "stats" / filename
if not conf["dry_run"]:
charts.multiseries_plot(mseries, chart_file, cols=3)
# License: overall pie chart
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, "ALL"))
ratio = qry.get_ratio(session)
chart_file = conf[
"cache_dir"] / "stats" / "copyright_overall-license_pie.png"
if not conf["dry_run"]:
charts.pie_chart(overall_licenses, chart_file, ratio)
# License: bar chart and per suite pie chart.
all_suites = statistics.sticky_suites(session) + __target_suites(
session, None)
licenses_per_suite = []
for suite in all_suites:
licenses = statistics.licenses_summary(
statistics.get_licenses(session, suite))
ratio = qry.get_ratio(session, suite=suite)
# draw license pie chart
if not conf["dry_run"]:
filename = "copyright_%s-license_pie-current.png" % suite
chart_file = conf["cache_dir"] / "stats" / filename
charts.pie_chart(licenses, chart_file, ratio)
licenses_per_suite.append(licenses)
chart_file = conf[
"cache_dir"] / "stats" / "copyright_license_bar_plot.png"
if not conf["dry_run"]:
charts.bar_chart(licenses_per_suite, all_suites, chart_file,
top_langs, "Number of files")
def list_suites(conf, session, archive):
"""return a mapping from suite names to a dict `{'archive': bool, 'db':
bool}`. The first field tells whether a suite is available via the local
mirror archive; the second whether it is available in Debsources DB.
"""
suites = {} # suite name -> {'archive': bool, 'db': bool}
def ensure_suite(suite):
if suite not in suites:
suites[suite] = {"archive": False, "db": False}
for suite in archive.ls_suites():
ensure_suite(suite)
suites[suite]["archive"] = True
for suite in statistics.sticky_suites(session):
ensure_suite(suite)
suites[suite]["db"] = True
return suites
def list_suites(conf, session, archive):
"""return a mapping from suite names to a dict `{'archive': bool, 'db':
bool}`. The first field tells whether a suite is available via the local
mirror archive; the second whether it is available in Debsources DB.
"""
suites = {} # suite name -> {'archive': bool, 'db': bool}
def ensure_suite(suite):
if suite not in suites:
suites[suite] = {'archive': False, 'db': False}
for suite in archive.ls_suites():
ensure_suite(suite)
suites[suite]['archive'] = True
for suite in statistics.sticky_suites(session):
ensure_suite(suite)
suites[suite]['db'] = True
return suites