def set_build_latency(metric_sec, bucket, must_be_never_leased):
q = model.Build.query(
model.Build.bucket == bucket,
model.Build.status == model.BuildStatus.SCHEDULED,
)
if must_be_never_leased:
q = q.filter(model.Build.never_leased == True)
else:
# Reuse the index that has never_leased
q = q.filter(model.Build.never_leased.IN((True, False)))
now = utils.utcnow()
dist = gae_ts_mon.Distribution(gae_ts_mon.GeometricBucketer())
for e in q.iter(projection=[model.Build.create_time]):
latency = (now - e.create_time).total_seconds()
dist.add(latency)
if dist.count == 0:
dist.add(0)
metric_sec.set(dist, {'bucket': bucket},
target_fields=GLOBAL_TARGET_FIELDS)
str_attr_field('status_legacy'),
'user_agent':
BuildMetricField(
value_type=str,
field_type=gae_ts_mon.StringField,
getter=lambda b:
(dict(buildtags.parse(t) for t in b.tags).get('user_agent')),
),
}
_METRIC_PREFIX_PROD = 'buildbucket/builds/'
_METRIC_PREFIX_EXPERIMENTAL = 'buildbucket/builds-experimental/'
# Maximum number of concurrent counting/latency queries.
_CONCURRENT_QUERY_LIMIT = 100
BUCKETER_24_HR = gae_ts_mon.GeometricBucketer(growth_factor=10**0.05)
BUCKETER_48_HR = gae_ts_mon.GeometricBucketer(growth_factor=10**0.053)
BUCKETER_5_SEC = gae_ts_mon.GeometricBucketer(growth_factor=10**0.0374)
BUCKETER_1K = gae_ts_mon.GeometricBucketer(growth_factor=10**0.031)
def _fields_for(build, field_names):
"""Returns field values for a build"""
result = {}
for field_name in field_names:
field = _BUILD_FIELDS.get(field_name)
if not field:
raise ValueError('invalid field %r' % field_name)
val = field.getter(build) or field.value_type()
result[field_name] = field.value_type(val)
return result
_JOBS_PER_SHARD = 500
# Override default target fields for app-global metrics.
_TARGET_FIELDS = {
'job_name': '', # module name
'hostname': '', # version
'task_num': 0, # instance ID
}
### All the metrics.
# Custom bucketer with 12% resolution in the range of 1..10**5. Used for job
# cycle times.
_bucketer = gae_ts_mon.GeometricBucketer(growth_factor=10**0.05,
num_finite_buckets=100)
# Regular (instance-local) metrics: jobs/completed and jobs/durations.
# Both have the following metric fields:
# - project_id: e.g. 'chromium'
# - subproject_id: e.g. 'blink'. Set to empty string if not used.
# - pool: e.g. 'Chrome'
# - spec_name: name of a job specification, e.g. '<master>:<builder>'
# for buildbot jobs.
# - result: one of 'success', 'failure', or 'infra-failure'.
_jobs_completed = gae_ts_mon.CounterMetric(
'jobs/completed',
'Number of completed jobs.', [
gae_ts_mon.StringField('spec_name'),
gae_ts_mon.StringField('project_id'),
gae_ts_mon.StringField('subproject_id'),
lease_requests_expired = gae_ts_mon.CounterMetric(
'machine_provider/lease_requests/expired',
'Number of lease requests expired.',
None,
)
lease_requests_fulfilled = gae_ts_mon.CounterMetric(
'machine_provider/lease_requests/fulfilled',
'Number of lease requests fulfilled.',
None,
)
lease_requests_fulfilled_time = gae_ts_mon.CumulativeDistributionMetric(
'machine_provider/lease_requests/fulfilled/time',
'Time taken to fulfill a lease request.',
None,
bucketer=gae_ts_mon.GeometricBucketer(growth_factor=10**0.04),
)
lease_requests_received = gae_ts_mon.CounterMetric(
'machine_provider/lease_requests/received',
'Number of lease requests received.',
None,
)
pubsub_messages_sent = gae_ts_mon.CounterMetric(
'machine_provider/pubsub_messages/sent',
'Number of Pub/Sub messages sent.',
[gae_ts_mon.StringField('target')],
)