def slocCountsMatchReferenceDb(self):
slocs_jessie = {
'ansic': 166724,
'asm': 65,
'awk': 25,
'cpp': 41458,
'cs': 1213,
'java': 916,
'lex': 223,
'lisp': 2193,
'makefile': 2092,
'ml': 5044,
'objc': 836,
'perl': 1199,
'python': 2916,
'ruby': 193,
'sed': 16,
'sh': 29984,
'sql': 237,
'xml': 14932,
'yacc': 312,
}
slocs_python = 8740
slocs_cpp_exp = 87521
self.assertEqual(slocs_jessie,
statistics.sloccount_summary(self.session,
suite='jessie'))
self.assertEqual(slocs_python,
statistics.sloccount_lang(self.session, 'python'))
self.assertEqual(slocs_cpp_exp,
statistics.sloccount_lang(self.session, 'cpp',
suite='experimental'))
def slocCountsMatchReferenceDb(self):
slocs_jessie = {
'ansic': 166724,
'asm': 65,
'awk': 25,
'cpp': 41458,
'cs': 1213,
'java': 916,
'lex': 223,
'lisp': 2193,
'makefile': 2092,
'ml': 5044,
'objc': 836,
'perl': 1199,
'python': 2916,
'ruby': 193,
'sed': 16,
'sh': 29984,
'sql': 237,
'xml': 14932,
'yacc': 312,
}
slocs_python = 9193
slocs_cpp_exp = 87521
self.assertEqual(
slocs_jessie,
statistics.sloccount_summary(self.session, suite='jessie'))
self.assertEqual(slocs_python,
statistics.sloccount_lang(self.session, 'python'))
self.assertEqual(
slocs_cpp_exp,
statistics.sloccount_lang(self.session,
'cpp',
suite='experimental'))
def slocCountsMatchReferenceDb(self):
slocs_jessie = {
"ansic": 166724,
"asm": 65,
"awk": 25,
"cpp": 41458,
"cs": 1213,
"java": 916,
"lex": 223,
"lisp": 2193,
"makefile": 2104,
"ml": 5044,
"objc": 836,
"perl": 1199,
"python": 2916,
"ruby": 193,
"sed": 16,
"sh": 30045,
"sql": 237,
"xml": 14932,
"yacc": 312,
}
slocs_python = 9193
slocs_cpp_exp = 87521
self.assertEqual(
slocs_jessie,
statistics.sloccount_summary(self.session, suite="jessie"))
self.assertEqual(slocs_python,
statistics.sloccount_lang(self.session, "python"))
self.assertEqual(
slocs_cpp_exp,
statistics.sloccount_lang(self.session,
"cpp",
suite="experimental"),
)
def update_charts(status, conf, session, suites=None):
"""update stage: rebuild charts"""
logging.info('update charts...')
ensure_stats_dir(conf)
suites = __target_suites(session, suites)
CHARTS = [ # <period, granularity> paris
('1 month', 'hourly'),
('1 year', 'daily'),
('5 years', 'weekly'),
('20 years', 'monthly'),
]
# size charts, various metrics
for metric in ['source_packages', 'disk_usage', 'source_files', 'ctags']:
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
series = getattr(statistics, 'history_size_' + granularity)(
session, metric, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-%s-%s.png' %
(suite, metric,
period.replace(' ', '-')))
if not conf['dry_run']:
charts.size_plot(series, chart_file)
# sloccount: historical histograms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
# historical histogram
mseries = getattr(statistics, 'history_sloc_' + granularity)(
session, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.sloc_plot(mseries, chart_file)
# sloccount: current pie charts
for suite in suites + ['ALL']:
sloc_suite = suite
if sloc_suite == 'ALL':
sloc_suite = None
slocs = statistics.sloccount_summary(session, suite=sloc_suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc_pie-current.png' % suite)
if not conf['dry_run']:
charts.sloc_pie(slocs, chart_file)
def update_charts(status, conf, session, suites=None):
"""update stage: rebuild charts"""
from debsources import charts
logging.info("update charts...")
ensure_stats_dir(conf)
suites = __target_suites(session, suites)
CHARTS = [ # <period, granularity> paris
("1 month", "hourly"),
("1 year", "daily"),
("5 years", "weekly"),
("20 years", "monthly"),
]
# size charts, various metrics
for metric in ["source_packages", "disk_usage", "source_files", "ctags"]:
for (period, granularity) in CHARTS:
for suite in suites + ["ALL"]:
series = getattr(statistics, "history_size_" + granularity)(
session, metric, interval=period, suite=suite)
filename = "%s-%s-%s.png" % (suite, metric,
period.replace(" ", "-"))
chart_file = conf["cache_dir"] / "stats" / filename
if not conf["dry_run"]:
charts.size_plot(series, chart_file)
# sloccount: historical histograms
for (period, granularity) in CHARTS:
for suite in suites + ["ALL"]:
# historical histogram
mseries = getattr(statistics,
"history_sloc_" + granularity)(session,
interval=period,
suite=suite)
filename = "%s-sloc-%s.png" % (suite, period.replace(" ", "-"))
chart_file = conf["cache_dir"] / "stats" / filename
if not conf["dry_run"]:
charts.multiseries_plot(mseries, chart_file)
# sloccount: current pie charts
sloc_per_suite = []
for suite in suites + ["ALL"]:
sloc_suite = suite
if sloc_suite == "ALL":
sloc_suite = None
slocs = statistics.sloccount_summary(session, suite=sloc_suite)
if suite not in ["ALL"]:
sloc_per_suite.append(slocs)
filename = "%s-sloc_pie-current.png" % suite
chart_file = conf["cache_dir"] / "stats" / filename
if not conf["dry_run"]:
charts.pie_chart(slocs, chart_file)
# sloccount: bar chart plot
if "charts_top_langs" in conf.keys():
top_langs = int(conf["charts_top_langs"])
else:
top_langs = 6
chart_file = conf["cache_dir"] / "stats" / "sloc_bar_plot.png"
charts.bar_chart(sloc_per_suite, suites, chart_file, top_langs, "SLOC")
def update_license_charts():
# License: historical histogramms
for (period, granularity) in CHARTS:
for suite in suites + ["ALL"]:
mseries = getattr(statistics, "history_copyright_" +
granularity)(session,
interval=period,
suite=suite)
filename = "copyright_%s-license-%s.png" % (
suite,
period.replace(" ", "-"),
)
chart_file = conf["cache_dir"] / "stats" / filename
if not conf["dry_run"]:
charts.multiseries_plot(mseries, chart_file, cols=3)
# License: overall pie chart
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, "ALL"))
ratio = qry.get_ratio(session)
chart_file = conf[
"cache_dir"] / "stats" / "copyright_overall-license_pie.png"
if not conf["dry_run"]:
charts.pie_chart(overall_licenses, chart_file, ratio)
# License: bar chart and per suite pie chart.
all_suites = statistics.sticky_suites(session) + __target_suites(
session, None)
licenses_per_suite = []
for suite in all_suites:
licenses = statistics.licenses_summary(
statistics.get_licenses(session, suite))
ratio = qry.get_ratio(session, suite=suite)
# draw license pie chart
if not conf["dry_run"]:
filename = "copyright_%s-license_pie-current.png" % suite
chart_file = conf["cache_dir"] / "stats" / filename
charts.pie_chart(licenses, chart_file, ratio)
licenses_per_suite.append(licenses)
chart_file = conf[
"cache_dir"] / "stats" / "copyright_license_bar_plot.png"
if not conf["dry_run"]:
charts.bar_chart(licenses_per_suite, all_suites, chart_file,
top_langs, "Number of files")
# LICENSE CHARTS
if "copyright" in conf["hooks"]:
update_license_charts()
def update_statistics(status, conf, session, suites=None):
"""update stage: update statistics
by default act on all non-sticky, major suites present in the DB. Pass
`suites` to override
"""
logging.info("update statistics...")
ensure_cache_dir(conf)
suites = __target_suites(session, suites)
now = datetime.utcnow()
stats_file = conf["cache_dir"] / "stats.data"
if stats_file.exists():
# If stats.data exists, load and update it, otherwise start from
# scratch. Note: this means that we need to be careful about changing
# stats keys, to avoid orphans.
# TODO: add check about orphan stats.data entries to debsources-fsck
stats = statistics.load_metadata_cache(stats_file)
else:
stats = {}
def store_sloccount_stats(summary, d, prefix, db_obj):
"""Update stats dictionary `d`, and DB object `db_obj`, with per
language sloccount statistics available in `summary`, generating
dictionary keys that start with `prefix`. Missing languages in summary
will be stored as 0-value entries.
"""
total_slocs = 0
for lang in SLOCCOUNT_LANGUAGES:
k = prefix + "." + lang
v = 0
if lang in summary:
v = summary[lang]
d[k] = v
setattr(db_obj, "lang_" + lang, v)
total_slocs += v
d[prefix] = total_slocs
# compute overall stats
suite = "ALL"
siz = HistorySize(suite, timestamp=now)
loc = HistorySlocCount(suite, timestamp=now)
for stat in ["disk_usage", "source_packages", "source_files", "ctags"]:
v = getattr(statistics, stat)(session)
stats["total." + stat] = v
setattr(siz, stat, v)
store_sloccount_stats(statistics.sloccount_summary(session), stats,
"total.sloccount", loc)
if not conf["dry_run"] and "db" in conf["backends"]:
session.add(siz)
session.add(loc)
# Update HistorySize
suite_key = "debian_"
hist_siz = {suite: HistorySize(suite, timestamp=now) for suite in suites}
for stat in ["disk_usage", "source_packages", "source_files", "ctags"]:
stats_result = statistics.stats_grouped_by(session, stat)
for res in stats_result:
if res[0] in suites:
stats[suite_key + res[0] + "." + stat] = res[1]
setattr(hist_siz[res[0]], stat, res[1])
if not conf["dry_run"] and "db" in conf["backends"]:
for siz in hist_siz.values():
session.add(siz)
# update historySlocCount
sloccount_res = statistics.stats_grouped_by(session, "sloccount")
hist_loc = {
suite: HistorySlocCount(suite, timestamp=now)
for suite in suites
}
for suite in suites:
temp = {item[1]: item[2] for item in sloccount_res if item[0] == suite}
store_sloccount_stats(dict(temp), stats,
suite_key + suite + ".sloccount",
hist_loc[suite])
if not conf["dry_run"] and "db" in conf["backends"]:
for loc in hist_loc.values():
session.add(loc)
session.flush()
# cache computed stats to on-disk stats file
if not conf["dry_run"] and "fs" in conf["backends"]:
statistics.save_metadata_cache(stats, stats_file)
def update_license_statistics(suites):
# compute License stats
license_stats_file = conf["cache_dir"] / "license_stats.data"
dual_license_file = conf["cache_dir"] / "dual_license.data"
license_stats = dict()
license_d_stats = dict()
hist_lic = {
suite: HistoryCopyright(suite, timestamp=now)
for suite in suites
}
results = statistics.get_licenses(session)
for suite in suites:
temp = {item[0]: item[2] for item in results if item[1] == suite}
summary = statistics.licenses_summary(temp)
for res in summary:
license_stats[suite + "." + res.rstrip()] = summary[res]
setattr(hist_lic[suite], "license", res.replace("_", " "))
setattr(hist_lic[suite], "files", summary[res])
if not conf["dry_run"] and "db" in conf["backends"]:
session.add(hist_lic[suite])
# no historical here, only save to file
dual_query = statistics.licenses_summary_w_dual(temp)
for res in dual_query:
license_d_stats[suite + "." + res.rstrip()] = dual_query[res]
# overall dual licenses
overall_d_licenses = statistics.licenses_summary_w_dual(
statistics.get_licenses(session, "ALL"))
for stat in overall_d_licenses:
license_d_stats["overall." + stat] = overall_d_licenses[stat]
# save dual licenses to file
if not conf["dry_run"] and "fs" in conf["backends"]:
statistics.save_metadata_cache(license_d_stats, dual_license_file)
session.flush()
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, "ALL"))
for stat in overall_licenses:
lic = HistoryCopyright("ALL", timestamp=now)
setattr(lic, "license", stat.replace("_", " "))
setattr(lic, "files", overall_licenses[stat])
license_stats["overall." + stat] = overall_licenses[stat]
if not conf["dry_run"] and "db" in conf["backends"]:
session.add(lic)
session.flush()
if not conf["dry_run"] and "fs" in conf["backends"]:
statistics.save_metadata_cache(license_stats, license_stats_file)
if "copyright" in conf["hooks"]:
update_license_statistics(suites)
def update_charts(status, conf, session, suites=None):
"""update stage: rebuild charts"""
from debsources import charts
logging.info('update charts...')
ensure_stats_dir(conf)
suites = __target_suites(session, suites)
CHARTS = [ # <period, granularity> paris
('1 month', 'hourly'),
('1 year', 'daily'),
('5 years', 'weekly'),
('20 years', 'monthly'),
]
# size charts, various metrics
for metric in ['source_packages', 'disk_usage', 'source_files', 'ctags']:
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
series = getattr(statistics, 'history_size_' + granularity)(
session, metric, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-%s-%s.png' %
(suite, metric,
period.replace(' ', '-')))
if not conf['dry_run']:
charts.size_plot(series, chart_file)
# sloccount: historical histograms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
# historical histogram
mseries = getattr(statistics, 'history_sloc_' + granularity)(
session, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.sloc_plot(mseries, chart_file)
# sloccount: current pie charts
sloc_per_suite = []
for suite in suites + ['ALL']:
sloc_suite = suite
if sloc_suite == 'ALL':
sloc_suite = None
slocs = statistics.sloccount_summary(session, suite=sloc_suite)
if suite not in ['ALL']:
sloc_per_suite.append(slocs)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc_pie-current.png' % suite)
if not conf['dry_run']:
charts.sloc_pie(slocs, chart_file)
# sloccount: bar chart plot
if 'charts_top_langs' in conf.keys():
top_langs = int(conf['charts_top_langs'])
else:
top_langs = 6
chart_file = os.path.join(conf['cache_dir'], 'stats', 'sloc_bar_plot.png')
charts.bar_chart(sloc_per_suite, suites, chart_file, top_langs)
def update_statistics(status, conf, session, suites=None):
"""update stage: update statistics
by default act on all non-sticky, major suites present in the DB. Pass
`suites` to override
"""
logging.info('update statistics...')
ensure_cache_dir(conf)
suites = __target_suites(session, suites)
now = datetime.utcnow()
stats_file = os.path.join(conf['cache_dir'], 'stats.data')
if os.path.exists(stats_file):
# If stats.data exists, load and update it, otherwise start from
# scratch. Note: this means that we need to be careful about changing
# stats keys, to avoid orphans.
# TODO: add check about orphan stats.data entries to debsources-fsck
stats = statistics.load_metadata_cache(stats_file)
else:
stats = {}
def store_sloccount_stats(summary, d, prefix, db_obj):
"""Update stats dictionary `d`, and DB object `db_obj`, with per
language sloccount statistics available in `summary`, generating
dictionary keys that start with `prefix`. Missing languages in summary
will be stored as 0-value entries.
"""
total_slocs = 0
for lang in SLOCCOUNT_LANGUAGES:
k = prefix + '.' + lang
v = 0
if lang in summary:
v = summary[lang]
d[k] = v
setattr(db_obj, 'lang_' + lang, v)
total_slocs += v
d[prefix] = total_slocs
# compute overall stats
suite = 'ALL'
siz = HistorySize(suite, timestamp=now)
loc = HistorySlocCount(suite, timestamp=now)
for stat in ['disk_usage', 'source_packages', 'source_files', 'ctags']:
v = getattr(statistics, stat)(session)
stats['total.' + stat] = v
setattr(siz, stat, v)
store_sloccount_stats(statistics.sloccount_summary(session),
stats, 'total.sloccount', loc)
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(siz)
session.add(loc)
# compute per-suite stats
for suite in suites:
siz = HistorySize(suite, timestamp=now)
loc = HistorySlocCount(suite, timestamp=now)
suite_key = 'debian_' + suite + '.'
for stat in ['disk_usage', 'source_packages', 'source_files', 'ctags']:
v = getattr(statistics, stat)(session, suite)
stats[suite_key + stat] = v
setattr(siz, stat, v)
store_sloccount_stats(statistics.sloccount_summary(session, suite),
stats, suite_key + 'sloccount', loc)
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(siz)
session.add(loc)
session.flush()
# cache computed stats to on-disk stats file
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(stats, stats_file)
def update_charts(status, conf, session, suites=None):
"""update stage: rebuild charts"""
from debsources import charts
logging.info('update charts...')
ensure_stats_dir(conf)
suites = __target_suites(session, suites)
CHARTS = [ # <period, granularity> paris
('1 month', 'hourly'),
('1 year', 'daily'),
('5 years', 'weekly'),
('20 years', 'monthly'),
]
# size charts, various metrics
for metric in ['source_packages', 'disk_usage', 'source_files', 'ctags']:
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
series = getattr(statistics, 'history_size_' + granularity)(
session, metric, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-%s-%s.png' %
(suite, metric,
period.replace(' ', '-')))
if not conf['dry_run']:
charts.size_plot(series, chart_file)
# sloccount: historical histograms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
# historical histogram
mseries = getattr(statistics, 'history_sloc_' + granularity)(
session, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.multiseries_plot(mseries, chart_file)
# sloccount: current pie charts
sloc_per_suite = []
for suite in suites + ['ALL']:
sloc_suite = suite
if sloc_suite == 'ALL':
sloc_suite = None
slocs = statistics.sloccount_summary(session, suite=sloc_suite)
if suite not in ['ALL']:
sloc_per_suite.append(slocs)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc_pie-current.png' % suite)
if not conf['dry_run']:
charts.pie_chart(slocs, chart_file)
# sloccount: bar chart plot
if 'charts_top_langs' in conf.keys():
top_langs = int(conf['charts_top_langs'])
else:
top_langs = 6
chart_file = os.path.join(conf['cache_dir'], 'stats', 'sloc_bar_plot.png')
charts.bar_chart(sloc_per_suite, suites, chart_file, top_langs, 'SLOC')
def update_license_charts():
# License: historical histogramms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
mseries = getattr(statistics,
'history_copyright_' + granularity)(
session, interval=period, suite=suite)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_%s-license-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.multiseries_plot(mseries, chart_file, cols=3)
# License: overall pie chart
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, 'ALL'))
ratio = qry.get_ratio(session)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_overall-license_pie.png')
if not conf['dry_run']:
charts.pie_chart(overall_licenses, chart_file, ratio)
# License: bar chart and per suite pie chart.
all_suites = statistics.sticky_suites(session) \
+ __target_suites(session, None)
licenses_per_suite = []
for suite in all_suites:
licenses = statistics.licenses_summary(
statistics.get_licenses(session, suite))
ratio = qry.get_ratio(session, suite=suite)
# draw license pie chart
if not conf['dry_run']:
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_%s'
'-license_pie-current.png'
% suite)
charts.pie_chart(licenses, chart_file, ratio)
licenses_per_suite.append(licenses)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_license_bar_plot.png')
if not conf['dry_run']:
charts.bar_chart(licenses_per_suite, all_suites, chart_file,
top_langs, 'Number of files')
# LICENSE CHARTS
if 'copyright' in conf['hooks']:
update_license_charts()
def update_statistics(status, conf, session, suites=None):
"""update stage: update statistics
by default act on all non-sticky, major suites present in the DB. Pass
`suites` to override
"""
logging.info('update statistics...')
ensure_cache_dir(conf)
suites = __target_suites(session, suites)
now = datetime.utcnow()
stats_file = os.path.join(conf['cache_dir'], 'stats.data')
if os.path.exists(stats_file):
# If stats.data exists, load and update it, otherwise start from
# scratch. Note: this means that we need to be careful about changing
# stats keys, to avoid orphans.
# TODO: add check about orphan stats.data entries to debsources-fsck
stats = statistics.load_metadata_cache(stats_file)
else:
stats = {}
def store_sloccount_stats(summary, d, prefix, db_obj):
"""Update stats dictionary `d`, and DB object `db_obj`, with per
language sloccount statistics available in `summary`, generating
dictionary keys that start with `prefix`. Missing languages in summary
will be stored as 0-value entries.
"""
total_slocs = 0
for lang in SLOCCOUNT_LANGUAGES:
k = prefix + '.' + lang
v = 0
if lang in summary:
v = summary[lang]
d[k] = v
setattr(db_obj, 'lang_' + lang, v)
total_slocs += v
d[prefix] = total_slocs
# compute overall stats
suite = 'ALL'
siz = HistorySize(suite, timestamp=now)
loc = HistorySlocCount(suite, timestamp=now)
for stat in ['disk_usage', 'source_packages', 'source_files', 'ctags']:
v = getattr(statistics, stat)(session)
stats['total.' + stat] = v
setattr(siz, stat, v)
store_sloccount_stats(statistics.sloccount_summary(session),
stats, 'total.sloccount', loc)
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(siz)
session.add(loc)
# Update HistorySize
suite_key = 'debian_'
hist_siz = dict((suite, HistorySize(suite, timestamp=now))
for suite in suites)
for stat in ['disk_usage', 'source_packages', 'source_files', 'ctags']:
stats_result = statistics.stats_grouped_by(session, stat)
for res in stats_result:
if res[0] in suites:
stats[suite_key + res[0] + '.' + stat] = res[1]
setattr(hist_siz[res[0]], stat, res[1])
if not conf['dry_run'] and 'db' in conf['backends']:
for siz in hist_siz.values():
session.add(siz)
# update historySlocCount
sloccount_res = statistics.stats_grouped_by(session, 'sloccount')
hist_loc = dict((suite, HistorySlocCount(suite, timestamp=now))
for suite in suites)
for suite in suites:
temp = dict((item[1], item[2]) for item in sloccount_res
if item[0] == suite)
store_sloccount_stats(dict(temp), stats,
suite_key + suite + ".sloccount",
hist_loc[suite])
if not conf['dry_run'] and 'db' in conf['backends']:
for loc in hist_loc.values():
session.add(loc)
session.flush()
# cache computed stats to on-disk stats file
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(stats, stats_file)
def update_license_statistics(suites):
# compute License stats
license_stats_file = os.path.join(conf['cache_dir'],
'license_stats.data')
dual_license_file = os.path.join(conf['cache_dir'],
'dual_license.data')
license_stats = dict()
license_d_stats = dict()
hist_lic = dict((suite, HistoryCopyright(suite, timestamp=now))
for suite in suites)
results = statistics.get_licenses(session)
for suite in suites:
temp = dict((item[0], item[2]) for item in results
if item[1] == suite)
summary = statistics.licenses_summary(temp)
for res in summary:
license_stats[suite + "." + res.rstrip()] = summary[res]
setattr(hist_lic[suite], 'license', res.replace('_', ' '))
setattr(hist_lic[suite], 'files', summary[res])
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(hist_lic[suite])
# no historical here, only save to file
dual_query = statistics.licenses_summary_w_dual(temp)
for res in dual_query:
license_d_stats[suite + "." + res.rstrip()] = dual_query[res]
# overall dual licenses
overall_d_licenses = statistics.licenses_summary_w_dual(
statistics.get_licenses(session, 'ALL'))
for stat in overall_d_licenses:
license_d_stats['overall.' + stat] = overall_d_licenses[stat]
# save dual licenses to file
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(license_d_stats, dual_license_file)
session.flush()
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, 'ALL'))
for stat in overall_licenses:
lic = HistoryCopyright('ALL', timestamp=now)
setattr(lic, 'license', stat.replace('_', ' '))
setattr(lic, 'files', overall_licenses[stat])
license_stats['overall.' + stat] = overall_licenses[stat]
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(lic)
session.flush()
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(license_stats, license_stats_file)
if 'copyright' in conf['hooks']:
update_license_statistics(suites)
def update_charts(status, conf, session, suites=None):
"""update stage: rebuild charts"""
from debsources import charts
logging.info('update charts...')
ensure_stats_dir(conf)
suites = __target_suites(session, suites)
CHARTS = [ # <period, granularity> paris
('1 month', 'hourly'),
('1 year', 'daily'),
('5 years', 'weekly'),
('20 years', 'monthly'),
]
# size charts, various metrics
for metric in ['source_packages', 'disk_usage', 'source_files', 'ctags']:
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
series = getattr(statistics, 'history_size_' + granularity)(
session, metric, interval=period, suite=suite)
chart_file = os.path.join(
conf['cache_dir'], 'stats',
'%s-%s-%s.png' % (suite, metric, period.replace(' ', '-')))
if not conf['dry_run']:
charts.size_plot(series, chart_file)
# sloccount: historical histograms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
# historical histogram
mseries = getattr(statistics,
'history_sloc_' + granularity)(session,
interval=period,
suite=suite)
chart_file = os.path.join(
conf['cache_dir'], 'stats',
'%s-sloc-%s.png' % (suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.multiseries_plot(mseries, chart_file)
# sloccount: current pie charts
sloc_per_suite = []
for suite in suites + ['ALL']:
sloc_suite = suite
if sloc_suite == 'ALL':
sloc_suite = None
slocs = statistics.sloccount_summary(session, suite=sloc_suite)
if suite not in ['ALL']:
sloc_per_suite.append(slocs)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'%s-sloc_pie-current.png' % suite)
if not conf['dry_run']:
charts.pie_chart(slocs, chart_file)
# sloccount: bar chart plot
if 'charts_top_langs' in conf.keys():
top_langs = int(conf['charts_top_langs'])
else:
top_langs = 6
chart_file = os.path.join(conf['cache_dir'], 'stats', 'sloc_bar_plot.png')
charts.bar_chart(sloc_per_suite, suites, chart_file, top_langs, 'SLOC')
def update_license_charts():
# License: historical histogramms
for (period, granularity) in CHARTS:
for suite in suites + ['ALL']:
mseries = getattr(statistics, 'history_copyright_' +
granularity)(session,
interval=period,
suite=suite)
chart_file = os.path.join(
conf['cache_dir'], 'stats', 'copyright_%s-license-%s.png' %
(suite, period.replace(' ', '-')))
if not conf['dry_run']:
charts.multiseries_plot(mseries, chart_file, cols=3)
# License: overall pie chart
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, 'ALL'))
ratio = qry.get_ratio(session)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_overall-license_pie.png')
if not conf['dry_run']:
charts.pie_chart(overall_licenses, chart_file, ratio)
# License: bar chart and per suite pie chart.
all_suites = statistics.sticky_suites(session) \
+ __target_suites(session, None)
licenses_per_suite = []
results = statistics.get_licenses(session)
for suite in all_suites:
temp = dict(
(item[0], item[2]) for item in results if item[1] == suite)
licenses = statistics.licenses_summary(temp)
ratio = qry.get_ratio(session, suite=suite)
# draw license pie chart
if not conf['dry_run']:
chart_file = os.path.join(
conf['cache_dir'], 'stats', 'copyright_%s'
'-license_pie-current.png' % suite)
charts.pie_chart(licenses, chart_file, ratio)
licenses_per_suite.append(licenses)
chart_file = os.path.join(conf['cache_dir'], 'stats',
'copyright_license_bar_plot.png')
if not conf['dry_run']:
charts.bar_chart(licenses_per_suite, all_suites, chart_file,
top_langs, 'Number of files')
# LICENSE CHARTS
if 'copyright' in conf['hooks']:
update_license_charts()
def update_statistics(status, conf, session, suites=None):
"""update stage: update statistics
by default act on all non-sticky, major suites present in the DB. Pass
`suites` to override
"""
logging.info('update statistics...')
ensure_cache_dir(conf)
suites = __target_suites(session, suites)
now = datetime.utcnow()
stats_file = os.path.join(conf['cache_dir'], 'stats.data')
if os.path.exists(stats_file):
# If stats.data exists, load and update it, otherwise start from
# scratch. Note: this means that we need to be careful about changing
# stats keys, to avoid orphans.
# TODO: add check about orphan stats.data entries to debsources-fsck
stats = statistics.load_metadata_cache(stats_file)
else:
stats = {}
def store_sloccount_stats(summary, d, prefix, db_obj):
"""Update stats dictionary `d`, and DB object `db_obj`, with per
language sloccount statistics available in `summary`, generating
dictionary keys that start with `prefix`. Missing languages in summary
will be stored as 0-value entries.
"""
total_slocs = 0
for lang in SLOCCOUNT_LANGUAGES:
k = prefix + '.' + lang
v = 0
if lang in summary:
v = summary[lang]
d[k] = v
setattr(db_obj, 'lang_' + lang, v)
total_slocs += v
d[prefix] = total_slocs
# compute overall stats
suite = 'ALL'
siz = HistorySize(suite, timestamp=now)
loc = HistorySlocCount(suite, timestamp=now)
for stat in ['disk_usage', 'source_packages', 'source_files', 'ctags']:
v = getattr(statistics, stat)(session)
stats['total.' + stat] = v
setattr(siz, stat, v)
store_sloccount_stats(statistics.sloccount_summary(session), stats,
'total.sloccount', loc)
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(siz)
session.add(loc)
# Update HistorySize
suite_key = 'debian_'
hist_siz = dict(
(suite, HistorySize(suite, timestamp=now)) for suite in suites)
for stat in ['disk_usage', 'source_packages', 'source_files', 'ctags']:
stats_result = statistics.stats_grouped_by(session, stat)
for res in stats_result:
if res[0] in suites:
stats[suite_key + res[0] + '.' + stat] = res[1]
setattr(hist_siz[res[0]], stat, res[1])
if not conf['dry_run'] and 'db' in conf['backends']:
for siz in hist_siz.values():
session.add(siz)
# update historySlocCount
sloccount_res = statistics.stats_grouped_by(session, 'sloccount')
hist_loc = dict(
(suite, HistorySlocCount(suite, timestamp=now)) for suite in suites)
for suite in suites:
temp = dict(
(item[1], item[2]) for item in sloccount_res if item[0] == suite)
store_sloccount_stats(dict(temp), stats,
suite_key + suite + ".sloccount",
hist_loc[suite])
if not conf['dry_run'] and 'db' in conf['backends']:
for loc in hist_loc.values():
session.add(loc)
session.flush()
# cache computed stats to on-disk stats file
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(stats, stats_file)
def update_license_statistics(suites):
# compute License stats
license_stats_file = os.path.join(conf['cache_dir'],
'license_stats.data')
dual_license_file = os.path.join(conf['cache_dir'],
'dual_license.data')
license_stats = dict()
license_d_stats = dict()
hist_lic = dict((suite, HistoryCopyright(suite, timestamp=now))
for suite in suites)
results = statistics.get_licenses(session)
for suite in suites:
temp = dict(
(item[0], item[2]) for item in results if item[1] == suite)
summary = statistics.licenses_summary(temp)
for res in summary:
license_stats[suite + "." + res.rstrip()] = summary[res]
setattr(hist_lic[suite], 'license', res.replace('_', ' '))
setattr(hist_lic[suite], 'files', summary[res])
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(hist_lic[suite])
# no historical here, only save to file
dual_query = statistics.licenses_summary_w_dual(temp)
for res in dual_query:
license_d_stats[suite + "." + res.rstrip()] = dual_query[res]
# overall dual licenses
overall_d_licenses = statistics.licenses_summary_w_dual(
statistics.get_licenses(session, 'ALL'))
for stat in overall_d_licenses:
license_d_stats['overall.' + stat] = overall_d_licenses[stat]
# save dual licenses to file
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(license_d_stats, dual_license_file)
session.flush()
overall_licenses = statistics.licenses_summary(
statistics.get_licenses(session, 'ALL'))
for stat in overall_licenses:
lic = HistoryCopyright('ALL', timestamp=now)
setattr(lic, 'license', stat.replace('_', ' '))
setattr(lic, 'files', overall_licenses[stat])
license_stats['overall.' + stat] = overall_licenses[stat]
if not conf['dry_run'] and 'db' in conf['backends']:
session.add(lic)
session.flush()
if not conf['dry_run'] and 'fs' in conf['backends']:
statistics.save_metadata_cache(license_stats, license_stats_file)
if 'copyright' in conf['hooks']:
update_license_statistics(suites)
