def __target_suites(session, suites=None): if not suites: sticky_suites = statistics.sticky_suites(session) suites = [suite for suite in statistics.suites(session, suites='all') if suite not in sticky_suites] return suites
def remove_suite(conf, session, suite): logging.info('remove sticky suite %s from the archive...' % suite) db_suite = db_storage.lookup_db_suite(session, suite, sticky=True) if not db_suite: logging.error('sticky suite %s does not exist in DB, abort.' % suite) return sticky_suites = statistics.sticky_suites(session) if updater.STAGE_GC in conf['stages']: for package in session.query(Package) \ .join(Suite) \ .filter(Suite.suite == suite) \ .filter(Package.sticky): pkg = SourcePackage.from_db_model(package) other_suites = \ session.query(Suite.suite.distinct()) \ .filter(Suite.package_id == package.id) \ .filter(Suite.suite != suite) other_suites = [row[0] for row in other_suites] if not other_suites: if not conf['single_transaction']: with session.begin(): updater._rm_package(pkg, conf, session, db_package=package) else: updater._rm_package(pkg, conf, session, db_package=package) else: other_sticky_suites = [ s for s in other_suites if s in sticky_suites ] if not other_sticky_suites and not conf['dry_run']: # package is only listed in "live" suites, drop sticky flag logging.debug('clearing sticky bit on %s' % pkg) package.sticky = False suitemap = db_storage.lookup_suitemapping(session, package, suite) if suitemap and not conf['dry_run']: session.delete(suitemap) if not conf['dry_run']: session.delete(db_suite) _remove_stats_for(conf, session, suite) logging.info('sticky suite %s removed from the archive.' % suite)
def update_license_charts(): # License: historical histogramms for (period, granularity) in CHARTS: for suite in suites + ['ALL']: mseries = getattr(statistics, 'history_copyright_' + granularity)(session, interval=period, suite=suite) chart_file = os.path.join( conf['cache_dir'], 'stats', 'copyright_%s-license-%s.png' % (suite, period.replace(' ', '-'))) if not conf['dry_run']: charts.multiseries_plot(mseries, chart_file, cols=3) # License: overall pie chart overall_licenses = statistics.licenses_summary( statistics.get_licenses(session, 'ALL')) ratio = qry.get_ratio(session) chart_file = os.path.join(conf['cache_dir'], 'stats', 'copyright_overall-license_pie.png') if not conf['dry_run']: charts.pie_chart(overall_licenses, chart_file, ratio) # License: bar chart and per suite pie chart. all_suites = statistics.sticky_suites(session) \ + __target_suites(session, None) licenses_per_suite = [] results = statistics.get_licenses(session) for suite in all_suites: temp = dict( (item[0], item[2]) for item in results if item[1] == suite) licenses = statistics.licenses_summary(temp) ratio = qry.get_ratio(session, suite=suite) # draw license pie chart if not conf['dry_run']: chart_file = os.path.join( conf['cache_dir'], 'stats', 'copyright_%s' '-license_pie-current.png' % suite) charts.pie_chart(licenses, chart_file, ratio) licenses_per_suite.append(licenses) chart_file = os.path.join(conf['cache_dir'], 'stats', 'copyright_license_bar_plot.png') if not conf['dry_run']: charts.bar_chart(licenses_per_suite, all_suites, chart_file, top_langs, 'Number of files')
def update_license_charts(): # License: historical histogramms for (period, granularity) in CHARTS: for suite in suites + ['ALL']: mseries = getattr(statistics, 'history_copyright_' + granularity)( session, interval=period, suite=suite) chart_file = os.path.join(conf['cache_dir'], 'stats', 'copyright_%s-license-%s.png' % (suite, period.replace(' ', '-'))) if not conf['dry_run']: charts.multiseries_plot(mseries, chart_file, cols=3) # License: overall pie chart overall_licenses = statistics.licenses_summary( statistics.get_licenses(session, 'ALL')) ratio = qry.get_ratio(session) chart_file = os.path.join(conf['cache_dir'], 'stats', 'copyright_overall-license_pie.png') if not conf['dry_run']: charts.pie_chart(overall_licenses, chart_file, ratio) # License: bar chart and per suite pie chart. all_suites = statistics.sticky_suites(session) \ + __target_suites(session, None) licenses_per_suite = [] results = statistics.get_licenses(session) for suite in all_suites: temp = dict((item[0], item[2]) for item in results if item[1] == suite) licenses = statistics.licenses_summary(temp) ratio = qry.get_ratio(session, suite=suite) # draw license pie chart if not conf['dry_run']: chart_file = os.path.join(conf['cache_dir'], 'stats', 'copyright_%s' '-license_pie-current.png' % suite) charts.pie_chart(licenses, chart_file, ratio) licenses_per_suite.append(licenses) chart_file = os.path.join(conf['cache_dir'], 'stats', 'copyright_license_bar_plot.png') if not conf['dry_run']: charts.bar_chart(licenses_per_suite, all_suites, chart_file, top_langs, 'Number of files')
def remove_suite(conf, session, suite): logging.info('remove sticky suite %s from the archive...' % suite) db_suite = db_storage.lookup_db_suite(session, suite, sticky=True) if not db_suite: logging.error('sticky suite %s does not exist in DB, abort.' % suite) return sticky_suites = statistics.sticky_suites(session) if updater.STAGE_GC in conf['stages']: for package in session.query(Package) \ .join(Suite) \ .filter(Suite.suite == suite) \ .filter(Package.sticky): pkg = SourcePackage.from_db_model(package) other_suites = \ session.query(Suite.suite.distinct()) \ .filter(Suite.package_id == package.id) \ .filter(Suite.suite != suite) other_suites = [row[0] for row in other_suites] if not other_suites: if not conf['single_transaction']: with session.begin(): updater._rm_package(pkg, conf, session, db_package=package) else: updater._rm_package(pkg, conf, session, db_package=package) else: other_sticky_suites = [s for s in other_suites if s in sticky_suites] if not other_sticky_suites and not conf['dry_run']: # package is only listed in "live" suites, drop sticky flag logging.debug('clearing sticky bit on %s' % pkg) package.sticky = False suitemap = db_storage.lookup_suitemapping(session, package, suite) if suitemap and not conf['dry_run']: session.delete(suitemap) if not conf['dry_run']: session.delete(db_suite) _remove_stats_for(conf, session, suite) logging.info('sticky suite %s removed from the archive.' % suite)
def update_license_charts(): # License: historical histogramms for (period, granularity) in CHARTS: for suite in suites + ["ALL"]: mseries = getattr(statistics, "history_copyright_" + granularity)(session, interval=period, suite=suite) filename = "copyright_%s-license-%s.png" % ( suite, period.replace(" ", "-"), ) chart_file = conf["cache_dir"] / "stats" / filename if not conf["dry_run"]: charts.multiseries_plot(mseries, chart_file, cols=3) # License: overall pie chart overall_licenses = statistics.licenses_summary( statistics.get_licenses(session, "ALL")) ratio = qry.get_ratio(session) chart_file = conf[ "cache_dir"] / "stats" / "copyright_overall-license_pie.png" if not conf["dry_run"]: charts.pie_chart(overall_licenses, chart_file, ratio) # License: bar chart and per suite pie chart. all_suites = statistics.sticky_suites(session) + __target_suites( session, None) licenses_per_suite = [] for suite in all_suites: licenses = statistics.licenses_summary( statistics.get_licenses(session, suite)) ratio = qry.get_ratio(session, suite=suite) # draw license pie chart if not conf["dry_run"]: filename = "copyright_%s-license_pie-current.png" % suite chart_file = conf["cache_dir"] / "stats" / filename charts.pie_chart(licenses, chart_file, ratio) licenses_per_suite.append(licenses) chart_file = conf[ "cache_dir"] / "stats" / "copyright_license_bar_plot.png" if not conf["dry_run"]: charts.bar_chart(licenses_per_suite, all_suites, chart_file, top_langs, "Number of files")
def list_suites(conf, session, archive): """return a mapping from suite names to a dict `{'archive': bool, 'db': bool}`. The first field tells whether a suite is available via the local mirror archive; the second whether it is available in Debsources DB. """ suites = {} # suite name -> {'archive': bool, 'db': bool} def ensure_suite(suite): if suite not in suites: suites[suite] = {"archive": False, "db": False} for suite in archive.ls_suites(): ensure_suite(suite) suites[suite]["archive"] = True for suite in statistics.sticky_suites(session): ensure_suite(suite) suites[suite]["db"] = True return suites
def list_suites(conf, session, archive): """return a mapping from suite names to a dict `{'archive': bool, 'db': bool}`. The first field tells whether a suite is available via the local mirror archive; the second whether it is available in Debsources DB. """ suites = {} # suite name -> {'archive': bool, 'db': bool} def ensure_suite(suite): if suite not in suites: suites[suite] = {'archive': False, 'db': False} for suite in archive.ls_suites(): ensure_suite(suite) suites[suite]['archive'] = True for suite in statistics.sticky_suites(session): ensure_suite(suite) suites[suite]['db'] = True return suites