def collect(dryrun=False):
"""Collect and push uploader-related metrics."""
canonical, noncanonical = per_affiliation_uploader_count()
uploaders = main_universe_uploader_count()
print('Active Canonical Uploaders: %s' % canonical)
print('Active Non-Canonical Uploaders: %s' % noncanonical)
print('Current Users with Main/Universe Upload Rights: %s' % uploaders)
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
gauge = Gauge('foundations_recent_uploaders',
'Active Recent Ubuntu Uploaders',
['affiliation'],
registry=registry)
gauge.labels('Canonical Uploaders').set(canonical)
gauge.labels('Non-Canonical Uploaders').set(noncanonical)
Gauge('foundations_main_universe_uploaders',
'Current Users with Main/Universe Upload Rights',
None,
registry=registry).set(uploaders)
util.push2gateway('foundations-active-contributors', registry)
def collect(dryrun=False):
"""Submit data to Push Gateway."""
triage = lp.get_team_daily_triage_count(team='ubuntu-server',
distro='Ubuntu',
blacklist=BLACKLIST)
backlog = lp.get_team_backlog_count(team='ubuntu-server', distro='Ubuntu')
print('Backlog Total: %s' % backlog)
print('Triage Total: %s' % triage)
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
Gauge('server_triage_backlog_total',
'Bugs in team backlog',
None,
registry=registry).set(backlog)
Gauge('server_triage_daily_triage_total',
'Bugs to review daily',
None,
registry=registry).set(triage)
util.push2gateway('triage', registry)
def collect(team_name, dryrun=False):
"""Submit data to Push Gateway."""
lp_team_name = util.get_launchpad_team_name(team_name)
triage = lp.get_team_daily_triage_count(team=lp_team_name,
distro='Ubuntu',
blacklist=BLACKLIST)
backlog = lp.get_team_backlog_count(team=lp_team_name,
distro='Ubuntu')
print('Backlog Total: %s' % backlog)
print('Triage Total: %s' % triage)
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
Gauge('{}_triage_backlog_total'.format(team_name),
'Bugs in team backlog',
None,
registry=registry).set(backlog)
Gauge('{}_triage_daily_triage_total'.format(team_name),
'Bugs to review daily',
None,
registry=registry).set(triage)
util.push2gateway('%s-triage' % team_name, registry)
def collect(team_name, dryrun=False):
"""Collect and push errors.u.c related metrics."""
# check to see if its a vaild team LP team
try:
lp.LP.people[team_name]
except KeyError:
print('Team %s does not exist in LP.' % team_name)
return
mcp_data = team_subscribed_mcp_count(team_name)
for series in mcp_data:
print("%s: %s" % (series, mcp_data[series]['sum_top_ten_counts']))
if not dryrun:
# metric names can not have a hyphen in them
team_name = team_name.replace('-', '_')
print('Pushing data...')
registry = CollectorRegistry()
gauge = Gauge('%s_errors_mcp_sum_top_ten' % team_name,
"Sum of yesterday's top ten crashes in errors",
['series'],
registry=registry)
for series in mcp_data:
gauge.labels(series).set(
mcp_data[series]['sum_top_ten_counts'])
util.push2gateway('%s_mcp_errors' % team_name, registry)
def collect(dryrun=False):
"""Push published cloud image counts."""
registry = CollectorRegistry()
count_gauge = Gauge('foundations_cloud_images_published',
'The number of cloud images published',
['image_type', 'cloud', 'release', 'arch'],
registry=registry)
latest_serial_gauge = Gauge('foundations_cloud_images_current_serial',
'The date portion of the latest serial',
['image_type', 'cloud', 'release'],
registry=registry)
latest_serial_age_gauge = Gauge(
'foundations_cloud_images_current_serial_age',
'The time in days between the last serial and today',
['image_type', 'cloud', 'release'],
registry=registry)
for image_type in ['daily', 'release']:
for cloud_name in CLOUD_NAMES[image_type]:
print('Counting {} images for {}...'.format(
image_type, cloud_name))
if 'aws' in cloud_name:
image_counts, latest_serials = do_aws_specific_collection(
cloud_name, image_type, latest_serial_gauge,
latest_serial_age_gauge)
else:
image_counts, latest_serials = parse_simplestreams_for_images(
cloud_name, image_type)
for release in image_counts:
for arch in image_counts[release]:
count = image_counts[release][arch]
print('Found {} {} images for {} {} {}'.format(
count, image_type, cloud_name, release, arch))
count_gauge.labels(image_type, cloud_name, release,
arch).set(count)
for release in latest_serials:
serial = latest_serials[release]
latest_serial_gauge.labels(image_type, cloud_name,
release).set(serial)
latest_serial_age_gauge.labels(
image_type, cloud_name,
release).set(_determine_serial_age(serial))
print('Finding serials for docker-core...')
docker_core_serials = get_current_download_serials(DOCKER_CORE_ROOT)
for release, serial in docker_core_serials.items():
age = _determine_serial_age(serial)
print('Found {} latest serial: {} ({} days old)'.format(
release, serial, age))
latest_serial_gauge.labels('daily', 'docker-core', release).set(serial)
latest_serial_age_gauge.labels('daily', 'docker-core',
release).set(age)
if not dryrun:
print('Pushing data...')
util.push2gateway('cloud-image-count-foundations', registry)
def collect(dryrun=False):
"""Submit data to Push Gateway."""
latest_release_prefix = _get_latest_release_prefix()
counts = {}
for tag in TAGS:
counts[tag] = _get_tag_counts(latest_release_prefix, tag)
print(counts)
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
gauge = Gauge('distro_rls_bug_tasks',
'', ['tag', 'team_name'],
registry=registry)
for tag in TAGS:
for team_name in counts[tag]:
gauge.labels(tag, team_name).set(counts[tag][team_name])
util.push2gateway('distro_rls_bug_tasks', registry)
def collect(team_name, dryrun=False):
"""Submit data to Push Gateway."""
results_by_component = get_merge_data(team_name)
print('%s' % (results_by_component, ))
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
gauge = Gauge('{}_mom'.format(team_name),
'', ['component', 'status'],
registry=registry)
for component in results_by_component:
for status in results_by_component[component]:
labels = gauge.labels(
component, # pylint: disable=no-member
status)
labels.set(results_by_component[component][status])
util.push2gateway('%s-merge' % team_name, registry)
def collect(dryrun=False):
"""Push published cloud image counts."""
registry = CollectorRegistry()
count_gauge = Gauge('foundations_cloud_images_published',
'The number of cloud images published',
['image_type', 'cloud', 'release', 'arch'],
registry=registry)
latest_serial_gauge = Gauge('foundations_cloud_images_current_serial',
'The date portion of the latest serial',
['image_type', 'cloud', 'release'],
registry=registry)
latest_serial_age_gauge = Gauge(
'foundations_cloud_images_current_serial_age',
'The time in days between the last serial and today',
['image_type', 'cloud', 'release'],
registry=registry)
for image_type in ['daily', 'release']:
for cloud_name in CLOUD_NAMES[image_type]:
print('Counting {} images for {}...'.format(
image_type, cloud_name))
image_counts, latest_serials = parse_simplestreams_for_images(
cloud_name, image_type)
for release in image_counts:
for arch in image_counts[release]:
count = image_counts[release][arch]
print('Found {} {} images for {} {} {}'.format(
count, image_type, cloud_name, release, arch))
count_gauge.labels(image_type, cloud_name, release,
arch).set(count)
for release in latest_serials:
serial = latest_serials[release]
latest_serial_gauge.labels(image_type, cloud_name,
release).set(serial)
serial_datetime = datetime.datetime.strptime(
str(serial), '%Y%m%d')
serial_age = (datetime.date.today() -
serial_datetime.date()).days
latest_serial_age_gauge.labels(image_type, cloud_name,
release).set(serial_age)
if not dryrun:
print('Pushing data...')
util.push2gateway('cloud-image-count-foundations', registry)
def collect(team_name, dryrun=False):
"""Submit data to Push Gateway."""
results = get_merge_data(team_name)
print('%s' % (results))
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
Gauge('{}_mom_local_total'.format(team_name),
'',
None,
registry=registry).set(results['local'])
Gauge('{}_mom_modified_total'.format(team_name),
'',
None,
registry=registry).set(results['modified'])
Gauge('{}_mom_needs_merge_total'.format(team_name),
'',
None,
registry=registry).set(results['needs-merge'])
Gauge('{}_mom_needs_sync_total'.format(team_name),
'',
None,
registry=registry).set(results['needs-sync'])
Gauge('{}_mom_repackaged_total'.format(team_name),
'',
None,
registry=registry).set(results['repackaged'])
Gauge('{}_mom_unmodified_total'.format(team_name),
'',
None,
registry=registry).set(results['unmodified'])
if team_name == 'server':
util.push2gateway('merge', registry)
else:
util.push2gateway('merge-%s' % team_name, registry)
def collect(view_id, creds_path, metric_prefix, dry_run=False):
"""Submit data to Push Gateway."""
registry = CollectorRegistry()
try:
analytics = initialize_analyticsreporting(creds_path, SCOPES)
# Get historical, all-time total counters.
response = get_report(analytics, view_id, '2010-01-01')
set_gauges(registry, response, metric_prefix)
except Exception: # pylint: disable=broad-except
logging.exception('Error collecting metrics')
finally:
if not dry_run:
# If set_gauges bombed out for any reason, just upload blank data.
# Make sure no proxy is used.
for var in ('http_proxy', 'http_proxy'):
for envvar in (var.lower(), var.upper()):
os.environ.pop(envvar, None)
util.push2gateway(metric_prefix, registry)
else: # Debugging enabled.
pprint.pprint([(x.name, x.samples) for x in registry.collect()])
def collect(team_name, dryrun=False):
"""Push upload data."""
date = datetime.now().date().strftime('%Y-%m-%d')
results = generate_upload_report(date, team_name)
print('%s: %s' % (date, results))
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
Gauge('{}_uploads_daily_dev_total'.format(team_name),
'Uploads to dev release',
None,
registry=registry).set(results['dev'])
Gauge('{}_uploads_daily_sru_total'.format(team_name),
'Uploads to supported release (SRU)',
None,
registry=registry).set(results['sru'])
util.push2gateway('%s-upload' % team_name, registry)
def collect(queue_name, dryrun=False):
"""Collect and push autopkgtest queue depth metrics."""
queue_details = QUEUES_JSON[queue_name]
for release in queue_details:
for arch in queue_details[release]:
count = len(queue_details[release][arch])
print('%s %s: %i' % (release.title(), arch, count))
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
for release in queue_details:
for arch in queue_details[release]:
count = len(queue_details[release][arch])
Gauge('autopkgtest_queue_size_%s_%s_%s' %
(queue_name, release, arch),
"Autopkgtest queue size",
None,
registry=registry).set(count)
util.push2gateway('foundations-autopkgtest-%s' %
queue_name, registry)
def collect(dryrun=False):
"""Submit data to Push Gateway."""
try:
devel = distro_info.UbuntuDistroInfo().devel()
except distro_info.DistroDataOutdated:
devel = distro_info.UbuntuDistroInfo().stable()
devel_results = get_iso_size_data(devel)
print('%s: %s' % (devel, devel_results))
lts = distro_info.UbuntuDistroInfo().lts()
lts_results = get_iso_size_data(lts, True)
print('%s: %s' % (lts, lts_results))
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
Gauge('server_iso_devel_amd64_size_total',
'dev amd64 size',
None,
registry=registry).set(devel_results['amd64'])
Gauge('server_iso_devel_arm64_size_total',
'dev arm64 size',
None,
registry=registry).set(devel_results['arm64'])
Gauge('server_iso_devel_i386_size_total',
'dev i386 size',
None,
registry=registry).set(devel_results['i386'])
Gauge('server_iso_devel_ppc64el_size_total',
'dev ppc64el size',
None,
registry=registry).set(devel_results['ppc64el'])
Gauge('server_iso_devel_s390x_size_total',
'dev s390x size',
None,
registry=registry).set(devel_results['s390x'])
Gauge('server_iso_lts_amd64_size_total',
'lts amd64 size',
None,
registry=registry).set(lts_results['amd64'])
Gauge('server_iso_lts_arm64_size_total',
'lts arm64 size',
None,
registry=registry).set(lts_results['arm64'])
Gauge('server_iso_lts_i386_size_total',
'lts i386 size',
None,
registry=registry).set(lts_results['i386'])
Gauge('server_iso_lts_ppc64el_size_total',
'lts ppc64el size',
None,
registry=registry).set(lts_results['ppc64el'])
Gauge('server_iso_lts_s390x_size_total',
'lts s390x size',
None,
registry=registry).set(lts_results['s390x'])
util.push2gateway('server-iso', registry)
latest = list(csv_handle)[-1]
valid, not_considered = [int(x) for x in latest[1:3]]
median_age, backlog = [int(x) for x in latest[4:6]]
gauge = Gauge('foundations_devel_proposed_migration_size',
'Number of packages waiting in devel-proposed',
['candidate'],
registry=registry)
gauge.labels('Valid Candidates').set(valid)
gauge.labels('Not Considered').set(not_considered)
Gauge('foundations_devel_proposed_migration_age',
'Median age of packages waiting in devel-proposed',
None,
registry=registry).set(median_age)
Gauge('foundations_devel_proposed_migration_backlog',
'Size of devel-proposed backlog (packages x days)',
None,
registry=registry).set(backlog)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
REGISTRY = CollectorRegistry()
try:
get_proposed_migration_queue(REGISTRY)
finally:
util.push2gateway('proposed_migration', REGISTRY)
def collect(pkg, repo='', dryrun=False):
"""Submit data to Push Gateway."""
print(pkg)
project_new = lp.get_bug_count(pkg, status='New')
project_total = lp.get_bug_count(pkg)
ubuntu_new = lp.get_ubuntu_bug_count(pkg, status='New')
ubuntu_total = lp.get_ubuntu_bug_count(pkg)
reviews = lp.get_active_review_count(pkg)
print('%s total bugs (%s new)' % (project_total, project_new))
print('%s pkg bugs (%s new)' % (ubuntu_total, ubuntu_new))
contrib = util.get_contributors(repo)
contrib_internal = [x for x in contrib if x.endswith('@canonical.com')]
contrib_external = [x for x in contrib if not x.endswith('@canonical.com')]
print('Total Contributors: %s' % len(contrib))
print('Total Internal Contributors: %s' % len(contrib_internal))
print('Total External Contributors: %s' % len(contrib_external))
if not dryrun:
print('Pushing data...')
# Prometheus does not like '-' in names (e.g. cloud-init)
pkg_str = pkg.replace('-', '') if '-' in pkg else pkg
registry = CollectorRegistry()
Gauge('server_%s_bug_total' % pkg_str,
'Bugs in project',
None,
registry=registry).set(project_total - project_new)
Gauge('server_%s_bug_new_total' % pkg_str,
'Bugs in project, marked new',
None,
registry=registry).set(project_new)
Gauge('server_%s_bug_ubuntu_total' % pkg_str,
'Bugs in Ubuntu pkg',
None,
registry=registry).set(ubuntu_total - ubuntu_new)
Gauge('server_%s_bug_ubuntu_new_total' % pkg_str,
'Bugs in Ubuntu pkg, marked new',
None,
registry=registry).set(ubuntu_new)
Gauge('server_%s_review_total' % pkg_str,
'Active reviews',
None,
registry=registry).set(reviews)
Gauge('server_%s_contrib_total' % pkg_str,
'Contributors',
None,
registry=registry).set(len(contrib))
Gauge('server_%s_contrib_external_total' % pkg_str,
'External Contributors',
None,
registry=registry).set(len(contrib_external))
Gauge('server_%s_contrib_internal_total' % pkg_str,
'Internal Contributors',
None,
registry=registry).set(len(contrib_internal))
util.push2gateway(pkg_str, registry)
def collect(dryrun=False): # pylint: disable=too-many-branches
"""Collect and push SRU-related metrics."""
sru_queues = sru_queue_count()
ready_srus = sru_verified_and_ready_count()
proposed_sru_age_data = proposed_package_ages()
unapproved_sru_age_data = unapproved_sru_ages()
q_name = 'Proposed Uploads in the Unapproved Queue per Series'
print('Number of %s:' % q_name)
for series, count in sru_queues.items():
print('%s: %s' % (series, count))
print('Age in days of oldest %s:' % q_name.replace('Uploads', 'Upload'))
for series in unapproved_sru_age_data:
print('%s: %s' %
(series, unapproved_sru_age_data[series]['oldest_age_in_days']))
print('Backlog age in days of %s:' % q_name)
for series in unapproved_sru_age_data:
print('%s: %s' %
(series, unapproved_sru_age_data[series]['ten_day_backlog_age']))
print('Number of backlogged %s:' % q_name)
for series in unapproved_sru_age_data:
print(
'%s: %s' %
(series, unapproved_sru_age_data[series]['ten_day_backlog_count']))
topic = 'Updates in Proposed per Series'
print('Number of Publishable %s:' % topic)
for series, count in ready_srus.items():
print('%s: %s' % (series, count))
for category in ('unverified', 'verified', 'vfailed'):
print('Number of backlogged %s %s:' % (category, topic))
for series in proposed_sru_age_data:
print('%s: %s' % (series, proposed_sru_age_data[series][
'fourteen_day_%s_backlog_count' % category]))
print('Backlog age in days of %s %s:' % (category, topic))
for series in proposed_sru_age_data:
print('%s: %s' % (series, proposed_sru_age_data[series][
'fourteen_day_%s_backlog_age' % category]))
if not dryrun:
print('Pushing data...')
registry = CollectorRegistry()
gauge = Gauge('distro_sru_unapproved_proposed_count',
'Number of %s' % q_name, ['series'],
registry=registry)
for series, count in sru_queues.items():
gauge.labels(series).set(count)
gauge = Gauge('distro_sru_unapproved_proposed_oldest_age',
'Age in days of oldest %s' %
q_name.replace('Uploads', 'Upload'), ['series'],
registry=registry)
for series in unapproved_sru_age_data:
gauge.labels(series).set(
unapproved_sru_age_data[series]['oldest_age_in_days'])
gauge = Gauge('distro_sru_unapproved_proposed_ten_day_backlog_age',
'Backlog age in days of %s' % q_name, ['series'],
registry=registry)
for series in unapproved_sru_age_data:
gauge.labels(series).set(
unapproved_sru_age_data[series]['ten_day_backlog_age'])
gauge = Gauge('distro_sru_unapproved_proposed_ten_day_backlog_count',
'Number of backlogged %s' % q_name, ['series'],
registry=registry)
for series in unapproved_sru_age_data:
gauge.labels(series).set(
unapproved_sru_age_data[series]['ten_day_backlog_count'])
gauge = Gauge('distro_sru_verified_and_ready_count',
'Number of Publishable Updates in Proposed per Series',
['series'],
registry=registry)
for series, count in ready_srus.items():
gauge.labels(series).set(count)
for cat in ('unverified', 'verified', 'vfailed'):
gauge = Gauge(
'distro_sru_%s_proposed_fourteen_day_backlog_age' % cat,
'Backlog age in days of %s %s' % (cat, topic), ['series'],
registry=registry)
for series in proposed_sru_age_data:
gauge.labels(series).set(
proposed_sru_age_data[series]['fourteen_day_%s_backlog_age'
% cat])
gauge = Gauge(
'distro_sru_%s_proposed_fourteen_day_backlog_count' % cat,
'Number of backlogged %s %s' % (cat, topic), ['series'],
registry=registry)
for series in proposed_sru_age_data:
gauge.labels(series).set(proposed_sru_age_data[series][
'fourteen_day_%s_backlog_count' % cat])
util.push2gateway('foundations-sru', registry)
