def collect(dryrun=False): """Collect and push uploader-related metrics.""" canonical, noncanonical = per_affiliation_uploader_count() uploaders = main_universe_uploader_count() print('Active Canonical Uploaders: %s' % canonical) print('Active Non-Canonical Uploaders: %s' % noncanonical) print('Current Users with Main/Universe Upload Rights: %s' % uploaders) if not dryrun: print('Pushing data...') registry = CollectorRegistry() gauge = Gauge('foundations_recent_uploaders', 'Active Recent Ubuntu Uploaders', ['affiliation'], registry=registry) gauge.labels('Canonical Uploaders').set(canonical) gauge.labels('Non-Canonical Uploaders').set(noncanonical) Gauge('foundations_main_universe_uploaders', 'Current Users with Main/Universe Upload Rights', None, registry=registry).set(uploaders) util.push2gateway('foundations-active-contributors', registry)
def collect(dryrun=False): """Submit data to Push Gateway.""" triage = lp.get_team_daily_triage_count(team='ubuntu-server', distro='Ubuntu', blacklist=BLACKLIST) backlog = lp.get_team_backlog_count(team='ubuntu-server', distro='Ubuntu') print('Backlog Total: %s' % backlog) print('Triage Total: %s' % triage) if not dryrun: print('Pushing data...') registry = CollectorRegistry() Gauge('server_triage_backlog_total', 'Bugs in team backlog', None, registry=registry).set(backlog) Gauge('server_triage_daily_triage_total', 'Bugs to review daily', None, registry=registry).set(triage) util.push2gateway('triage', registry)
def collect(team_name, dryrun=False): """Submit data to Push Gateway.""" lp_team_name = util.get_launchpad_team_name(team_name) triage = lp.get_team_daily_triage_count(team=lp_team_name, distro='Ubuntu', blacklist=BLACKLIST) backlog = lp.get_team_backlog_count(team=lp_team_name, distro='Ubuntu') print('Backlog Total: %s' % backlog) print('Triage Total: %s' % triage) if not dryrun: print('Pushing data...') registry = CollectorRegistry() Gauge('{}_triage_backlog_total'.format(team_name), 'Bugs in team backlog', None, registry=registry).set(backlog) Gauge('{}_triage_daily_triage_total'.format(team_name), 'Bugs to review daily', None, registry=registry).set(triage) util.push2gateway('%s-triage' % team_name, registry)
def collect(team_name, dryrun=False): """Collect and push errors.u.c related metrics.""" # check to see if its a vaild team LP team try: lp.LP.people[team_name] except KeyError: print('Team %s does not exist in LP.' % team_name) return mcp_data = team_subscribed_mcp_count(team_name) for series in mcp_data: print("%s: %s" % (series, mcp_data[series]['sum_top_ten_counts'])) if not dryrun: # metric names can not have a hyphen in them team_name = team_name.replace('-', '_') print('Pushing data...') registry = CollectorRegistry() gauge = Gauge('%s_errors_mcp_sum_top_ten' % team_name, "Sum of yesterday's top ten crashes in errors", ['series'], registry=registry) for series in mcp_data: gauge.labels(series).set( mcp_data[series]['sum_top_ten_counts']) util.push2gateway('%s_mcp_errors' % team_name, registry)
def collect(dryrun=False): """Push published cloud image counts.""" registry = CollectorRegistry() count_gauge = Gauge('foundations_cloud_images_published', 'The number of cloud images published', ['image_type', 'cloud', 'release', 'arch'], registry=registry) latest_serial_gauge = Gauge('foundations_cloud_images_current_serial', 'The date portion of the latest serial', ['image_type', 'cloud', 'release'], registry=registry) latest_serial_age_gauge = Gauge( 'foundations_cloud_images_current_serial_age', 'The time in days between the last serial and today', ['image_type', 'cloud', 'release'], registry=registry) for image_type in ['daily', 'release']: for cloud_name in CLOUD_NAMES[image_type]: print('Counting {} images for {}...'.format( image_type, cloud_name)) if 'aws' in cloud_name: image_counts, latest_serials = do_aws_specific_collection( cloud_name, image_type, latest_serial_gauge, latest_serial_age_gauge) else: image_counts, latest_serials = parse_simplestreams_for_images( cloud_name, image_type) for release in image_counts: for arch in image_counts[release]: count = image_counts[release][arch] print('Found {} {} images for {} {} {}'.format( count, image_type, cloud_name, release, arch)) count_gauge.labels(image_type, cloud_name, release, arch).set(count) for release in latest_serials: serial = latest_serials[release] latest_serial_gauge.labels(image_type, cloud_name, release).set(serial) latest_serial_age_gauge.labels( image_type, cloud_name, release).set(_determine_serial_age(serial)) print('Finding serials for docker-core...') docker_core_serials = get_current_download_serials(DOCKER_CORE_ROOT) for release, serial in docker_core_serials.items(): age = _determine_serial_age(serial) print('Found {} latest serial: {} ({} days old)'.format( release, serial, age)) latest_serial_gauge.labels('daily', 'docker-core', release).set(serial) latest_serial_age_gauge.labels('daily', 'docker-core', release).set(age) if not dryrun: print('Pushing data...') util.push2gateway('cloud-image-count-foundations', registry)
def collect(dryrun=False): """Submit data to Push Gateway.""" latest_release_prefix = _get_latest_release_prefix() counts = {} for tag in TAGS: counts[tag] = _get_tag_counts(latest_release_prefix, tag) print(counts) if not dryrun: print('Pushing data...') registry = CollectorRegistry() gauge = Gauge('distro_rls_bug_tasks', '', ['tag', 'team_name'], registry=registry) for tag in TAGS: for team_name in counts[tag]: gauge.labels(tag, team_name).set(counts[tag][team_name]) util.push2gateway('distro_rls_bug_tasks', registry)
def collect(team_name, dryrun=False): """Submit data to Push Gateway.""" results_by_component = get_merge_data(team_name) print('%s' % (results_by_component, )) if not dryrun: print('Pushing data...') registry = CollectorRegistry() gauge = Gauge('{}_mom'.format(team_name), '', ['component', 'status'], registry=registry) for component in results_by_component: for status in results_by_component[component]: labels = gauge.labels( component, # pylint: disable=no-member status) labels.set(results_by_component[component][status]) util.push2gateway('%s-merge' % team_name, registry)
def collect(dryrun=False): """Push published cloud image counts.""" registry = CollectorRegistry() count_gauge = Gauge('foundations_cloud_images_published', 'The number of cloud images published', ['image_type', 'cloud', 'release', 'arch'], registry=registry) latest_serial_gauge = Gauge('foundations_cloud_images_current_serial', 'The date portion of the latest serial', ['image_type', 'cloud', 'release'], registry=registry) latest_serial_age_gauge = Gauge( 'foundations_cloud_images_current_serial_age', 'The time in days between the last serial and today', ['image_type', 'cloud', 'release'], registry=registry) for image_type in ['daily', 'release']: for cloud_name in CLOUD_NAMES[image_type]: print('Counting {} images for {}...'.format( image_type, cloud_name)) image_counts, latest_serials = parse_simplestreams_for_images( cloud_name, image_type) for release in image_counts: for arch in image_counts[release]: count = image_counts[release][arch] print('Found {} {} images for {} {} {}'.format( count, image_type, cloud_name, release, arch)) count_gauge.labels(image_type, cloud_name, release, arch).set(count) for release in latest_serials: serial = latest_serials[release] latest_serial_gauge.labels(image_type, cloud_name, release).set(serial) serial_datetime = datetime.datetime.strptime( str(serial), '%Y%m%d') serial_age = (datetime.date.today() - serial_datetime.date()).days latest_serial_age_gauge.labels(image_type, cloud_name, release).set(serial_age) if not dryrun: print('Pushing data...') util.push2gateway('cloud-image-count-foundations', registry)
def collect(team_name, dryrun=False): """Submit data to Push Gateway.""" results = get_merge_data(team_name) print('%s' % (results)) if not dryrun: print('Pushing data...') registry = CollectorRegistry() Gauge('{}_mom_local_total'.format(team_name), '', None, registry=registry).set(results['local']) Gauge('{}_mom_modified_total'.format(team_name), '', None, registry=registry).set(results['modified']) Gauge('{}_mom_needs_merge_total'.format(team_name), '', None, registry=registry).set(results['needs-merge']) Gauge('{}_mom_needs_sync_total'.format(team_name), '', None, registry=registry).set(results['needs-sync']) Gauge('{}_mom_repackaged_total'.format(team_name), '', None, registry=registry).set(results['repackaged']) Gauge('{}_mom_unmodified_total'.format(team_name), '', None, registry=registry).set(results['unmodified']) if team_name == 'server': util.push2gateway('merge', registry) else: util.push2gateway('merge-%s' % team_name, registry)
def collect(view_id, creds_path, metric_prefix, dry_run=False): """Submit data to Push Gateway.""" registry = CollectorRegistry() try: analytics = initialize_analyticsreporting(creds_path, SCOPES) # Get historical, all-time total counters. response = get_report(analytics, view_id, '2010-01-01') set_gauges(registry, response, metric_prefix) except Exception: # pylint: disable=broad-except logging.exception('Error collecting metrics') finally: if not dry_run: # If set_gauges bombed out for any reason, just upload blank data. # Make sure no proxy is used. for var in ('http_proxy', 'http_proxy'): for envvar in (var.lower(), var.upper()): os.environ.pop(envvar, None) util.push2gateway(metric_prefix, registry) else: # Debugging enabled. pprint.pprint([(x.name, x.samples) for x in registry.collect()])
def collect(team_name, dryrun=False): """Push upload data.""" date = datetime.now().date().strftime('%Y-%m-%d') results = generate_upload_report(date, team_name) print('%s: %s' % (date, results)) if not dryrun: print('Pushing data...') registry = CollectorRegistry() Gauge('{}_uploads_daily_dev_total'.format(team_name), 'Uploads to dev release', None, registry=registry).set(results['dev']) Gauge('{}_uploads_daily_sru_total'.format(team_name), 'Uploads to supported release (SRU)', None, registry=registry).set(results['sru']) util.push2gateway('%s-upload' % team_name, registry)
def collect(queue_name, dryrun=False): """Collect and push autopkgtest queue depth metrics.""" queue_details = QUEUES_JSON[queue_name] for release in queue_details: for arch in queue_details[release]: count = len(queue_details[release][arch]) print('%s %s: %i' % (release.title(), arch, count)) if not dryrun: print('Pushing data...') registry = CollectorRegistry() for release in queue_details: for arch in queue_details[release]: count = len(queue_details[release][arch]) Gauge('autopkgtest_queue_size_%s_%s_%s' % (queue_name, release, arch), "Autopkgtest queue size", None, registry=registry).set(count) util.push2gateway('foundations-autopkgtest-%s' % queue_name, registry)
def collect(dryrun=False): """Submit data to Push Gateway.""" try: devel = distro_info.UbuntuDistroInfo().devel() except distro_info.DistroDataOutdated: devel = distro_info.UbuntuDistroInfo().stable() devel_results = get_iso_size_data(devel) print('%s: %s' % (devel, devel_results)) lts = distro_info.UbuntuDistroInfo().lts() lts_results = get_iso_size_data(lts, True) print('%s: %s' % (lts, lts_results)) if not dryrun: print('Pushing data...') registry = CollectorRegistry() Gauge('server_iso_devel_amd64_size_total', 'dev amd64 size', None, registry=registry).set(devel_results['amd64']) Gauge('server_iso_devel_arm64_size_total', 'dev arm64 size', None, registry=registry).set(devel_results['arm64']) Gauge('server_iso_devel_i386_size_total', 'dev i386 size', None, registry=registry).set(devel_results['i386']) Gauge('server_iso_devel_ppc64el_size_total', 'dev ppc64el size', None, registry=registry).set(devel_results['ppc64el']) Gauge('server_iso_devel_s390x_size_total', 'dev s390x size', None, registry=registry).set(devel_results['s390x']) Gauge('server_iso_lts_amd64_size_total', 'lts amd64 size', None, registry=registry).set(lts_results['amd64']) Gauge('server_iso_lts_arm64_size_total', 'lts arm64 size', None, registry=registry).set(lts_results['arm64']) Gauge('server_iso_lts_i386_size_total', 'lts i386 size', None, registry=registry).set(lts_results['i386']) Gauge('server_iso_lts_ppc64el_size_total', 'lts ppc64el size', None, registry=registry).set(lts_results['ppc64el']) Gauge('server_iso_lts_s390x_size_total', 'lts s390x size', None, registry=registry).set(lts_results['s390x']) util.push2gateway('server-iso', registry)
latest = list(csv_handle)[-1] valid, not_considered = [int(x) for x in latest[1:3]] median_age, backlog = [int(x) for x in latest[4:6]] gauge = Gauge('foundations_devel_proposed_migration_size', 'Number of packages waiting in devel-proposed', ['candidate'], registry=registry) gauge.labels('Valid Candidates').set(valid) gauge.labels('Not Considered').set(not_considered) Gauge('foundations_devel_proposed_migration_age', 'Median age of packages waiting in devel-proposed', None, registry=registry).set(median_age) Gauge('foundations_devel_proposed_migration_backlog', 'Size of devel-proposed backlog (packages x days)', None, registry=registry).set(backlog) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) REGISTRY = CollectorRegistry() try: get_proposed_migration_queue(REGISTRY) finally: util.push2gateway('proposed_migration', REGISTRY)
def collect(pkg, repo='', dryrun=False): """Submit data to Push Gateway.""" print(pkg) project_new = lp.get_bug_count(pkg, status='New') project_total = lp.get_bug_count(pkg) ubuntu_new = lp.get_ubuntu_bug_count(pkg, status='New') ubuntu_total = lp.get_ubuntu_bug_count(pkg) reviews = lp.get_active_review_count(pkg) print('%s total bugs (%s new)' % (project_total, project_new)) print('%s pkg bugs (%s new)' % (ubuntu_total, ubuntu_new)) contrib = util.get_contributors(repo) contrib_internal = [x for x in contrib if x.endswith('@canonical.com')] contrib_external = [x for x in contrib if not x.endswith('@canonical.com')] print('Total Contributors: %s' % len(contrib)) print('Total Internal Contributors: %s' % len(contrib_internal)) print('Total External Contributors: %s' % len(contrib_external)) if not dryrun: print('Pushing data...') # Prometheus does not like '-' in names (e.g. cloud-init) pkg_str = pkg.replace('-', '') if '-' in pkg else pkg registry = CollectorRegistry() Gauge('server_%s_bug_total' % pkg_str, 'Bugs in project', None, registry=registry).set(project_total - project_new) Gauge('server_%s_bug_new_total' % pkg_str, 'Bugs in project, marked new', None, registry=registry).set(project_new) Gauge('server_%s_bug_ubuntu_total' % pkg_str, 'Bugs in Ubuntu pkg', None, registry=registry).set(ubuntu_total - ubuntu_new) Gauge('server_%s_bug_ubuntu_new_total' % pkg_str, 'Bugs in Ubuntu pkg, marked new', None, registry=registry).set(ubuntu_new) Gauge('server_%s_review_total' % pkg_str, 'Active reviews', None, registry=registry).set(reviews) Gauge('server_%s_contrib_total' % pkg_str, 'Contributors', None, registry=registry).set(len(contrib)) Gauge('server_%s_contrib_external_total' % pkg_str, 'External Contributors', None, registry=registry).set(len(contrib_external)) Gauge('server_%s_contrib_internal_total' % pkg_str, 'Internal Contributors', None, registry=registry).set(len(contrib_internal)) util.push2gateway(pkg_str, registry)
def collect(dryrun=False): # pylint: disable=too-many-branches """Collect and push SRU-related metrics.""" sru_queues = sru_queue_count() ready_srus = sru_verified_and_ready_count() proposed_sru_age_data = proposed_package_ages() unapproved_sru_age_data = unapproved_sru_ages() q_name = 'Proposed Uploads in the Unapproved Queue per Series' print('Number of %s:' % q_name) for series, count in sru_queues.items(): print('%s: %s' % (series, count)) print('Age in days of oldest %s:' % q_name.replace('Uploads', 'Upload')) for series in unapproved_sru_age_data: print('%s: %s' % (series, unapproved_sru_age_data[series]['oldest_age_in_days'])) print('Backlog age in days of %s:' % q_name) for series in unapproved_sru_age_data: print('%s: %s' % (series, unapproved_sru_age_data[series]['ten_day_backlog_age'])) print('Number of backlogged %s:' % q_name) for series in unapproved_sru_age_data: print( '%s: %s' % (series, unapproved_sru_age_data[series]['ten_day_backlog_count'])) topic = 'Updates in Proposed per Series' print('Number of Publishable %s:' % topic) for series, count in ready_srus.items(): print('%s: %s' % (series, count)) for category in ('unverified', 'verified', 'vfailed'): print('Number of backlogged %s %s:' % (category, topic)) for series in proposed_sru_age_data: print('%s: %s' % (series, proposed_sru_age_data[series][ 'fourteen_day_%s_backlog_count' % category])) print('Backlog age in days of %s %s:' % (category, topic)) for series in proposed_sru_age_data: print('%s: %s' % (series, proposed_sru_age_data[series][ 'fourteen_day_%s_backlog_age' % category])) if not dryrun: print('Pushing data...') registry = CollectorRegistry() gauge = Gauge('distro_sru_unapproved_proposed_count', 'Number of %s' % q_name, ['series'], registry=registry) for series, count in sru_queues.items(): gauge.labels(series).set(count) gauge = Gauge('distro_sru_unapproved_proposed_oldest_age', 'Age in days of oldest %s' % q_name.replace('Uploads', 'Upload'), ['series'], registry=registry) for series in unapproved_sru_age_data: gauge.labels(series).set( unapproved_sru_age_data[series]['oldest_age_in_days']) gauge = Gauge('distro_sru_unapproved_proposed_ten_day_backlog_age', 'Backlog age in days of %s' % q_name, ['series'], registry=registry) for series in unapproved_sru_age_data: gauge.labels(series).set( unapproved_sru_age_data[series]['ten_day_backlog_age']) gauge = Gauge('distro_sru_unapproved_proposed_ten_day_backlog_count', 'Number of backlogged %s' % q_name, ['series'], registry=registry) for series in unapproved_sru_age_data: gauge.labels(series).set( unapproved_sru_age_data[series]['ten_day_backlog_count']) gauge = Gauge('distro_sru_verified_and_ready_count', 'Number of Publishable Updates in Proposed per Series', ['series'], registry=registry) for series, count in ready_srus.items(): gauge.labels(series).set(count) for cat in ('unverified', 'verified', 'vfailed'): gauge = Gauge( 'distro_sru_%s_proposed_fourteen_day_backlog_age' % cat, 'Backlog age in days of %s %s' % (cat, topic), ['series'], registry=registry) for series in proposed_sru_age_data: gauge.labels(series).set( proposed_sru_age_data[series]['fourteen_day_%s_backlog_age' % cat]) gauge = Gauge( 'distro_sru_%s_proposed_fourteen_day_backlog_count' % cat, 'Number of backlogged %s %s' % (cat, topic), ['series'], registry=registry) for series in proposed_sru_age_data: gauge.labels(series).set(proposed_sru_age_data[series][ 'fourteen_day_%s_backlog_count' % cat]) util.push2gateway('foundations-sru', registry)