def _ignored_metric(self):
return metrics.Boolean(
self._metric_root_path + 'ignored',
description=(
'A boolean, for servers ignored for test infra prod alerts.'),
field_spec=[ts_mon.StringField('target_data_center'),
ts_mon.StringField('target_hostname'),])
def _presence_metric(self):
return metrics.Boolean(
self._metric_root_path + 'presence',
description=(
'A boolean indicating whether a server is in the machines db.'),
field_spec=[ts_mon.StringField('target_data_center'),
ts_mon.StringField('target_hostname'),])
def _roles_metric(self):
return metrics.String(
self._metric_root_path + 'roles',
description=(
'A string indicating the role of a server in the machines db.'),
field_spec=[ts_mon.StringField('target_data_center'),
ts_mon.StringField('target_hostname'),])
def calculate_spares_needed(self, target_total):
"""Calculate and log the spares needed to achieve a target.
Return how many working spares are needed to achieve the
given `target_total` with all DUTs working.
The spares count may be positive or negative. Positive
values indicate spares are needed to replace broken DUTs in
order to reach the target; negative numbers indicate that
no spares are needed, and that a corresponding number of
working devices can be returned.
If the new target total would require returning ineligible
DUTs, an error is logged, and the target total is adjusted
so that those DUTs are not exchanged.
@param target_total The new target pool size.
@return The number of spares needed.
"""
num_ineligible = len(self.ineligible_hosts)
spares_needed = target_total >= num_ineligible
metrics.Boolean(
'chromeos/autotest/balance_pools/exhausted_pools',
'True for each pool/model which requests more DUTs than supplied',
# TODO(jrbarnette) The 'board' field is a legacy. We need
# to leave it here until we do the extra work Monarch
# requires to delete a field.
field_spec=[
ts_mon.StringField('pool'),
ts_mon.StringField('board'),
ts_mon.StringField('model'),
]).set(
not spares_needed,
fields={
'pool': self.pool,
'board': self.labels.get('model', ''),
'model': self.labels.get('model', ''),
},
)
if not spares_needed:
_log_error(
'%s pool (%s): Target of %d is below minimum of %d DUTs.',
self.pool,
self.labels,
target_total,
num_ineligible,
)
_log_error('Adjusting target to %d DUTs.', num_ineligible)
target_total = num_ineligible
else:
_log_message('%s %s pool: Target of %d is above minimum.',
self.labels.get('model', ''), self.pool, target_total)
adjustment = target_total - self.total_hosts
return len(self.broken_hosts) + adjustment
def _exchange_labels(dry_run, hosts, target_pool, spare_pool):
"""Reassign a list of DUTs from one pool to another.
For all the given hosts, remove all labels associated with
`spare_pool`, and add the labels for `target_pool`.
If `dry_run` is true, perform no changes, but log the `atest`
commands needed to accomplish the necessary label changes.
@param dry_run Whether the logging is for a dry run or
for actual execution.
@param hosts List of DUTs (AFE hosts) to be reassigned.
@param target_pool The `_DUTPool` object from which the hosts
are drawn.
@param spare_pool The `_DUTPool` object to which the hosts
will be added.
"""
_log_info(dry_run, 'Transferring %d DUTs from %s to %s.',
len(hosts), spare_pool.pool, target_pool.pool)
metrics.Counter(
'chromeos/autotest/balance_pools/duts_moved',
'DUTs transferred between pools',
# TODO(jrbarnette) The 'board' field is a legacy. We need to
# leave it here until we do the extra work Monarch requires to
# delete a field.
field_spec=[
ts_mon.StringField('board'),
ts_mon.StringField('model'),
ts_mon.StringField('source_pool'),
ts_mon.StringField('target_pool'),
]
).increment_by(
len(hosts),
fields={
'board': target_pool.labels.get('model', ''),
'model': target_pool.labels.get('model', ''),
'source_pool': spare_pool.pool,
'target_pool': target_pool.pool,
},
)
if not hosts:
return
additions = target_pool.pool_labels
removals = spare_pool.pool_labels
for host in hosts:
if not dry_run:
_log_message('Updating host: %s.', host.hostname)
host.remove_labels(removals)
host.add_labels(additions)
else:
_log_message('atest label remove -m %s %s',
host.hostname, ' '.join(removals))
_log_message('atest label add -m %s %s',
host.hostname, ' '.join(additions))
class Poller(threading.Thread):
commits_metric = ts_mon.CounterMetric(
'bugdroid/commits', 'Counter of commits processed by bugdroid', [
ts_mon.StringField('poller'),
ts_mon.StringField('project'),
ts_mon.StringField('status')
])
def __init__(self,
interval_in_minutes=15,
setup_refresh_interval_minutes=0,
run_once=False):
threading.Thread.__init__(self, name=str(hash(self)))
self.interval = interval_in_minutes * 60
self.refresh_interval = setup_refresh_interval_minutes
self.run_once = run_once
if setup_refresh_interval_minutes:
self.setup_refresh = (
datetime.datetime.now() +
datetime.timedelta(minutes=setup_refresh_interval_minutes))
else:
self.setup_refresh = None
def execute(self):
raise NotImplementedError()
def setup(self): # pylint: disable=R0201
return True
def run(self):
try:
while True:
if self.setup_refresh and self.setup_refresh < datetime.datetime.now(
):
LOGGER.info('Re-running Poller setup')
self.setup()
self.setup_refresh = (
datetime.datetime.now() +
datetime.timedelta(minutes=self.refresh_interval))
self.execute()
if self.run_once:
return
time.sleep(self.interval)
except Exception:
LOGGER.exception('Unhandled Poller exception.')
def start(self):
if self.setup():
super(Poller, self).start()
def _RecordDurationMetric(batches):
"""Records a span duration metric for each span.
Args:
batches: A sequence of span batches (lists)
Yields:
Re-yields the same batches
"""
m = metrics.CumulativeSecondsDistribution(
_SPAN_DURATION_METRIC,
description="The durations of Spans consumed by export_to_cloud_trace",
field_spec=[ts_mon.StringField('name')])
for batch in batches:
batch = tuple(batch) # Needed because we will consume the iterator.
for span in batch:
try:
time_delta = (_ParseDatetime(span['endTime']) -
_ParseDatetime(span['startTime']))
m.add(time_delta.total_seconds(),
fields={'name': span['name']})
except KeyError:
log.error(
"Span %s did not have required fields 'endTime', "
"'startTime', and 'name'.", json.dumps(span))
yield batch
def main(argv):
"""Entry point for dut_mon."""
logging.getLogger().setLevel(logging.INFO)
with ts_mon_config.SetupTsMonGlobalState('dut_mon', indirect=True):
afe = frontend.AFE()
counters = collections.defaultdict(lambda: 0)
field_spec = [ts_mon.StringField('board'),
ts_mon.StringField('model'),
ts_mon.StringField('pool'),
ts_mon.BooleanField('is_locked'),
ts_mon.StringField('status'),
]
dut_count = metrics.Gauge('chromeos/autotest/dut_mon/dut_count',
description='The number of duts in a given '
'state and bucket.',
field_spec=field_spec)
tick_count = metrics.Counter('chromeos/autotest/dut_mon/tick',
description='Tick counter of dut_mon.')
while True:
# Note: We reset all counters to zero in each loop rather than
# creating a new defaultdict, because we want to ensure that any
# gauges that were previously set to a nonzero value by this process
# get set back to zero if necessary.
for k in counters:
counters[k] = 0
logging.info('Fetching all hosts.')
hosts = afe.get_hosts()
logging.info('Fetched %s hosts.', len(hosts))
for host in hosts:
fields = _get_bucket_for_host(host)
counters[fields] += 1
for field, value in counters.iteritems():
logging.info('%s %s', field, value)
dut_count.set(value, fields=field.__dict__)
tick_count.increment()
logging.info('Sleeping for 2 minutes.')
time.sleep(120)
def _RecordSpanMetrics(span):
"""Increments the count of spans logged.
Args:
span: The span to record.
"""
m = metrics.Counter(
_SPAN_COUNT_METRIC,
description="A count of spans logged by a client.",
field_spec=[ts_mon.StringField('name')])
m.increment(fields={'name': span.name})
def _EmitImportanceMetric(master_config, important_configs,
experimental_configs):
"""Emit monarch metrics about which slave configs were important."""
importance = {build_config: True for build_config in important_configs}
for build_config in experimental_configs:
importance[build_config] = False
m = metrics.BooleanMetric(
'chromeos/cbuildbot/master/has_important_slave',
description='Slaves that were considered '
'important by master.',
field_spec=[
ts_mon.StringField('master_config'),
ts_mon.StringField('slave_config')
])
for slave_config, is_important in importance.items():
m.set(is_important,
fields={
'master_config': master_config,
'slave_config': slave_config
})
def testGetMetricFieldSpec(self):
"""Test each field type gets its FieldSpec."""
fields = {
'int': 12,
'bool': True,
'str': 'string',
}
expected_fieldspec = [ts_mon.IntegerField('int'),
ts_mon.BooleanField('bool'),
ts_mon.StringField('str')]
self.assertEqual(ts_mon_config.GetMetricFieldSpec(fields),
expected_fieldspec)
class ApiRateLimiter(RateLimiter):
blocked_requests = ts_mon.CounterMetric(
'monorail/apiratelimiter/blocked_request',
'Count of requests that exceeded the rate limit and were blocked.',
None)
limit_exceeded = ts_mon.CounterMetric(
'monorail/apiratelimiter/rate_exceeded',
'Count of requests that exceeded the rate limit.', None)
cost_thresh_exceeded = ts_mon.CounterMetric(
'monorail/apiratelimiter/cost_thresh_exceeded',
'Count of requests that were expensive to process', None)
checks = ts_mon.CounterMetric(
'monorail/apiratelimiter/check',
'Count of checks done, by fail/success type.',
[ts_mon.StringField('type')])
#pylint: disable=arguments-differ
def CheckStart(self, client_id, client_email, now=None):
if now is None:
now = time.time()
keysets = _CreateApiCacheKeys(client_id, client_email, now)
qpm_limit = client_config_svc.GetQPMDict().get(client_email,
DEFAULT_API_QPM)
window_limit = qpm_limit * N_MINUTES
self._AuxCheckStart(keysets, window_limit,
settings.api_ratelimiting_enabled,
ApiRateLimitExceeded(client_id, client_email))
#pylint: disable=arguments-differ
def CheckEnd(self, client_id, client_email, now, start_time):
if not settings.ratelimiting_cost_enabled:
return
elapsed_ms = (now - start_time) * 1000
if elapsed_ms < settings.api_ratelimiting_cost_thresh_ms:
return
keysets = _CreateApiCacheKeys(client_id, client_email, start_time)
self._AuxCheckEnd(
keysets, 'API Rate Limit Cost Threshold Exceeded: %s, %s' %
(client_id, client_email), settings.api_ratelimiting_cost_penalty)
def testEnqueue(self):
"""Test that _Indirect enqueues messages correctly."""
metric = metrics.Boolean
with parallel.Manager() as manager:
q = manager.Queue()
self.PatchObject(metrics, 'MESSAGE_QUEUE', q)
proxy_metric = metric('foo')
proxy_metric.example('arg1', {'field_name': 'value'})
message = q.get(timeout=10)
expected_metric_kwargs = {
'field_spec': [ts_mon.StringField('field_name')],
'description': 'No description.',
}
self.assertEqual(
message,
metrics.MetricCall(metric.__name__, ('foo',), expected_metric_kwargs,
'example', ('arg1', {'field_name': 'value'}), {},
False))
class SpamService(object):
"""The persistence layer for spam reports."""
issue_actions = ts_mon.CounterMetric(
'monorail/spam_svc/issue',
'Count of things that happen to issues.',
[ts_mon.StringField('type')])
comment_actions = ts_mon.CounterMetric(
'monorail/spam_svc/comment',
'Count of things that happen to comments.',
[ts_mon.StringField('type')])
ml_engine_failures = ts_mon.CounterMetric(
'monorail/spam_svc/ml_engine_failure',
'Failures calling the ML Engine API',
None)
def __init__(self):
self.report_tbl = sql.SQLTableManager(SPAMREPORT_TABLE_NAME)
self.verdict_tbl = sql.SQLTableManager(SPAMVERDICT_TABLE_NAME)
self.issue_tbl = sql.SQLTableManager(ISSUE_TABLE)
# ML Engine library is lazy loaded below.
self.ml_engine = None
def LookupIssuesFlaggers(self, cnxn, issue_ids):
"""Returns users who've reported the issues or their comments as spam.
Returns a dictionary {issue_id: (issue_reporters, comment_reporters)}
issue_reportes is a list of users who flagged the issue;
comment_reporters element is a dictionary {comment_id: [user_ids]} where
user_ids are the users who flagged that comment.
"""
rows = self.report_tbl.Select(
cnxn, cols=['issue_id', 'user_id', 'comment_id'],
issue_id=issue_ids)
reporters = collections.defaultdict(
# Return a tuple of (issue_reporters, comment_reporters) as described
# above.
lambda: ([], collections.defaultdict(list)))
for row in rows:
issue_id = int(row[0])
user_id = row[1]
if row[2]:
comment_id = row[2]
reporters[issue_id][1][comment_id].append(user_id)
else:
reporters[issue_id][0].append(user_id)
return reporters
def LookupIssueFlaggers(self, cnxn, issue_id):
"""Returns users who've reported the issue or its comments as spam.
Returns a tuple. First element is a list of users who flagged the issue;
second element is a dictionary of comment id to a list of users who flagged
that comment.
"""
return self.LookupIssuesFlaggers(cnxn, [issue_id])[issue_id]
def LookupIssueFlagCounts(self, cnxn, issue_ids):
"""Returns a map of issue_id to flag counts"""
rows = self.report_tbl.Select(cnxn, cols=['issue_id', 'COUNT(*)'],
issue_id=issue_ids, group_by=['issue_id'])
counts = {}
for row in rows:
counts[int(row[0])] = row[1]
return counts
def LookupIssueVerdicts(self, cnxn, issue_ids):
"""Returns a map of issue_id to most recent spam verdicts"""
rows = self.verdict_tbl.Select(cnxn,
cols=['issue_id', 'reason', 'MAX(created)'],
issue_id=issue_ids, comment_id=None,
group_by=['issue_id'])
counts = {}
for row in rows:
counts[int(row[0])] = row[1]
return counts
def LookupIssueVerdictHistory(self, cnxn, issue_ids):
"""Returns a map of issue_id to most recent spam verdicts"""
rows = self.verdict_tbl.Select(cnxn, cols=[
'issue_id', 'reason', 'created', 'is_spam', 'classifier_confidence',
'user_id', 'overruled'],
issue_id=issue_ids, order_by=[('issue_id', []), ('created', [])])
# TODO: group by issue_id, make class instead of dict for verdict.
verdicts = []
for row in rows:
verdicts.append({
'issue_id': row[0],
'reason': row[1],
'created': row[2],
'is_spam': row[3],
'classifier_confidence': row[4],
'user_id': row[5],
'overruled': row[6],
})
return verdicts
def LookupCommentVerdictHistory(self, cnxn, comment_ids):
"""Returns a map of issue_id to most recent spam verdicts"""
rows = self.verdict_tbl.Select(cnxn, cols=[
'comment_id', 'reason', 'created', 'is_spam', 'classifier_confidence',
'user_id', 'overruled'],
comment_id=comment_ids, order_by=[('comment_id', []), ('created', [])])
# TODO: group by comment_id, make class instead of dict for verdict.
verdicts = []
for row in rows:
verdicts.append({
'comment_id': row[0],
'reason': row[1],
'created': row[2],
'is_spam': row[3],
'classifier_confidence': row[4],
'user_id': row[5],
'overruled': row[6],
})
return verdicts
def FlagIssues(self, cnxn, issue_service, issues, reporting_user_id,
flagged_spam):
"""Creates or deletes a spam report on an issue."""
verdict_updates = []
if flagged_spam:
rows = [(issue.issue_id, issue.reporter_id, reporting_user_id)
for issue in issues]
self.report_tbl.InsertRows(cnxn, SPAMREPORT_ISSUE_COLS, rows,
ignore=True)
else:
issue_ids = [issue.issue_id for issue in issues]
self.report_tbl.Delete(
cnxn, issue_id=issue_ids, user_id=reporting_user_id,
comment_id=None)
project_id = issues[0].project_id
# Now record new verdicts and update issue.is_spam, if they've changed.
ids = [issue.issue_id for issue in issues]
counts = self.LookupIssueFlagCounts(cnxn, ids)
previous_verdicts = self.LookupIssueVerdicts(cnxn, ids)
for issue_id in counts:
# If the flag counts changed enough to toggle the is_spam bit, need to
# record a new verdict and update the Issue.
# No number of user spam flags can overturn an admin's verdict.
if previous_verdicts.get(issue_id) == REASON_MANUAL:
continue
# If enough spam flags come in, mark the issue as spam.
if (flagged_spam and counts[issue_id] >= settings.spam_flag_thresh):
verdict_updates.append(issue_id)
if len(verdict_updates) == 0:
return
# Some of the issues may have exceed the flag threshold, so issue verdicts
# and mark as spam in those cases.
rows = [(issue_id, flagged_spam, REASON_THRESHOLD, project_id)
for issue_id in verdict_updates]
self.verdict_tbl.InsertRows(cnxn, THRESHVERDICT_ISSUE_COLS, rows,
ignore=True)
update_issues = []
for issue in issues:
if issue.issue_id in verdict_updates:
issue.is_spam = flagged_spam
update_issues.append(issue)
if flagged_spam:
self.issue_actions.increment_by(len(update_issues), {'type': 'flag'})
issue_service.UpdateIssues(cnxn, update_issues, update_cols=['is_spam'])
def FlagComment(self, cnxn, issue_id, comment_id, reported_user_id,
reporting_user_id, flagged_spam):
"""Creates or deletes a spam report on a comment."""
# TODO(seanmccullough): Bulk comment flagging? There's no UI for that.
if flagged_spam:
self.report_tbl.InsertRow(
cnxn, ignore=True, issue_id=issue_id,
comment_id=comment_id, reported_user_id=reported_user_id,
user_id=reporting_user_id)
self.comment_actions.increment({'type': 'flag'})
else:
self.report_tbl.Delete(
cnxn, issue_id=issue_id, comment_id=comment_id,
user_id=reporting_user_id)
def RecordClassifierIssueVerdict(self, cnxn, issue, is_spam, confidence,
fail_open):
reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER
self.verdict_tbl.InsertRow(cnxn, issue_id=issue.issue_id, is_spam=is_spam,
reason=reason, classifier_confidence=confidence,
project_id=issue.project_id)
if is_spam:
self.issue_actions.increment({'type': 'classifier'})
# This is called at issue creation time, so there's nothing else to do here.
def RecordManualIssueVerdicts(self, cnxn, issue_service, issues, user_id,
is_spam):
rows = [(user_id, issue.issue_id, is_spam, REASON_MANUAL, issue.project_id)
for issue in issues]
issue_ids = [issue.issue_id for issue in issues]
# Overrule all previous verdicts.
self.verdict_tbl.Update(cnxn, {'overruled': True}, [
('issue_id IN (%s)' % sql.PlaceHolders(issue_ids), issue_ids)
], commit=False)
self.verdict_tbl.InsertRows(cnxn, MANUALVERDICT_ISSUE_COLS, rows,
ignore=True)
for issue in issues:
issue.is_spam = is_spam
if is_spam:
self.issue_actions.increment_by(len(issues), {'type': 'manual'})
else:
issue_service.AllocateNewLocalIDs(cnxn, issues)
# This will commit the transaction.
issue_service.UpdateIssues(cnxn, issues, update_cols=['is_spam'])
def RecordManualCommentVerdict(self, cnxn, issue_service, user_service,
comment_id, user_id, is_spam):
# TODO(seanmccullough): Bulk comment verdicts? There's no UI for that.
self.verdict_tbl.InsertRow(cnxn, ignore=True,
user_id=user_id, comment_id=comment_id, is_spam=is_spam,
reason=REASON_MANUAL)
comment = issue_service.GetComment(cnxn, comment_id)
comment.is_spam = is_spam
issue = issue_service.GetIssue(cnxn, comment.issue_id, use_cache=False)
issue_service.SoftDeleteComment(
cnxn, issue, comment, user_id, user_service, is_spam, True, is_spam)
if is_spam:
self.comment_actions.increment({'type': 'manual'})
def RecordClassifierCommentVerdict(self, cnxn, comment, is_spam, confidence,
fail_open):
reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER
self.verdict_tbl.InsertRow(cnxn, comment_id=comment.id, is_spam=is_spam,
reason=reason, classifier_confidence=confidence,
project_id=comment.project_id)
if is_spam:
self.comment_actions.increment({'type': 'classifier'})
def _predict(self, instance):
"""Requests a prediction from the ML Engine API.
Sample API response:
{'predictions': [{
'classes': ['0', '1'],
'scores': [0.4986788034439087, 0.5013211965560913]
}]}
This hits the default model.
Returns:
A floating point number representing the confidence
the instance is spam.
"""
model_name = 'projects/%s/models/%s' % (
settings.classifier_project_id, settings.spam_model_name)
body = {'instances': [{"inputs": instance["word_hashes"]}]}
if not self.ml_engine:
self.ml_engine = ml_helpers.setup_ml_engine()
request = self.ml_engine.projects().predict(name=model_name, body=body)
response = request.execute()
logging.info('ML Engine API response: %r' % response)
prediction = response['predictions'][0]
# Ensure the class confidence we return is for the spam, not the ham label.
# The spam label, '1', is usually at index 1 but I'm not sure of any
# guarantees around label order.
if prediction['classes'][1] == SPAM_CLASS_LABEL:
return prediction['scores'][1]
elif prediction['classes'][0] == SPAM_CLASS_LABEL:
return prediction['scores'][0]
else:
raise Exception('No predicted classes found.')
def _IsExempt(self, author, is_project_member):
"""Return True if the user is exempt from spam checking."""
if author.email is not None and author.email.endswith(
settings.spam_whitelisted_suffixes):
logging.info('%s whitelisted from spam filtering', author.email)
return True
if is_project_member:
logging.info('%s is a project member, assuming ham', author.email)
return True
return False
def ClassifyIssue(self, issue, firstComment, reporter, is_project_member):
"""Classify an issue as either spam or ham.
Args:
issue: the Issue.
firstComment: the first Comment on issue.
reporter: User PB for the Issue reporter.
is_project_member: True if reporter is a member of issue's project.
Returns a JSON dict of classifier prediction results from
the ML Engine API.
"""
instance = ml_helpers.GenerateFeaturesRaw(
[issue.summary, firstComment.content],
settings.spam_feature_hashes)
return self._classify(instance, reporter, is_project_member)
def ClassifyComment(self, comment_content, commenter, is_project_member=True):
"""Classify a comment as either spam or ham.
Args:
comment: the comment text.
commenter: User PB for the user who authored the comment.
Returns a JSON dict of classifier prediction results from
the ML Engine API.
"""
instance = ml_helpers.GenerateFeaturesRaw(
['', comment_content],
settings.spam_feature_hashes)
return self._classify(instance, commenter, is_project_member)
def _classify(self, instance, author, is_project_member):
# Fail-safe: not spam.
result = self.ham_classification()
if self._IsExempt(author, is_project_member):
return result
if not self.ml_engine:
self.ml_engine = ml_helpers.setup_ml_engine()
# If setup_ml_engine returns None, it failed to init.
if not self.ml_engine:
logging.error("ML Engine not initialized.")
self.ml_engine_failures.increment()
result['failed_open'] = True
return result
remaining_retries = 3
while remaining_retries > 0:
try:
result['confidence_is_spam'] = self._predict(instance)
result['failed_open'] = False
return result
except Exception as ex:
remaining_retries = remaining_retries - 1
self.ml_engine_failures.increment()
logging.error('Error calling ML Engine API: %s' % ex)
result['failed_open'] = True
return result
def ham_classification(self):
return {'confidence_is_spam': 0.0,
'failed_open': False}
def GetIssueClassifierQueue(
self, cnxn, _issue_service, project_id, offset=0, limit=10):
"""Returns list of recent issues with spam verdicts,
ranked in ascending order of confidence (so uncertain items are first).
"""
# TODO(seanmccullough): Optimize pagination. This query probably gets
# slower as the number of SpamVerdicts grows, regardless of offset
# and limit values used here. Using offset,limit in general may not
# be the best way to do this.
issue_results = self.verdict_tbl.Select(cnxn,
cols=['issue_id', 'is_spam', 'reason', 'classifier_confidence',
'created'],
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('issue_id IS NOT NULL', []),
],
order_by=[
('classifier_confidence ASC', []),
('created ASC', []),
],
group_by=['issue_id'],
offset=offset,
limit=limit,
)
ret = []
for row in issue_results:
ret.append(ModerationItem(
issue_id=int(row[0]),
is_spam=row[1] == 1,
reason=row[2],
classifier_confidence=row[3],
verdict_time='%s' % row[4],
))
count = self.verdict_tbl.SelectValue(cnxn,
col='COUNT(*)',
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('issue_id IS NOT NULL', []),
])
return ret, count
def GetIssueFlagQueue(
self, cnxn, _issue_service, project_id, offset=0, limit=10):
"""Returns list of recent issues that have been flagged by users"""
issue_flags = self.report_tbl.Select(cnxn,
cols = ["Issue.project_id", "Report.issue_id", "count(*) as count",
"max(Report.created) as latest",
"count(distinct Report.user_id) as users"],
left_joins=["Issue ON Issue.id = Report.issue_id"],
where=[('Report.issue_id IS NOT NULL', []),
("Issue.project_id == %v", [project_id])],
order_by=[('count DESC', [])],
group_by=['Report.issue_id'],
offset=offset, limit=limit)
ret = []
for row in issue_flags:
ret.append(ModerationItem(
project_id=row[0],
issue_id=row[1],
count=row[2],
latest_report=row[3],
num_users=row[4],
))
count = self.verdict_tbl.SelectValue(cnxn,
col='COUNT(DISTINCT Report.issue_id)',
where=[('Issue.project_id = %s', [project_id])],
left_joins=["Issue ON Issue.id = SpamReport.issue_id"])
return ret, count
def GetCommentClassifierQueue(
self, cnxn, _issue_service, project_id, offset=0, limit=10):
"""Returns list of recent comments with spam verdicts,
ranked in ascending order of confidence (so uncertain items are first).
"""
# TODO(seanmccullough): Optimize pagination. This query probably gets
# slower as the number of SpamVerdicts grows, regardless of offset
# and limit values used here. Using offset,limit in general may not
# be the best way to do this.
comment_results = self.verdict_tbl.Select(cnxn,
cols=['issue_id', 'is_spam', 'reason', 'classifier_confidence',
'created'],
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('comment_id IS NOT NULL', []),
],
order_by=[
('classifier_confidence ASC', []),
('created ASC', []),
],
group_by=['comment_id'],
offset=offset,
limit=limit,
)
ret = []
for row in comment_results:
ret.append(ModerationItem(
comment_id=int(row[0]),
is_spam=row[1] == 1,
reason=row[2],
classifier_confidence=row[3],
verdict_time='%s' % row[4],
))
count = self.verdict_tbl.SelectValue(cnxn,
col='COUNT(*)',
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('comment_id IS NOT NULL', []),
])
return ret, count
def GetTrainingIssues(self, cnxn, issue_service, since, offset=0, limit=100):
"""Returns list of recent issues with human-labeled spam/ham verdicts.
"""
# get all of the manual verdicts in the past day.
results = self.verdict_tbl.Select(cnxn,
cols=['issue_id'],
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('issue_id IS NOT NULL', []),
('created > %s', [since.isoformat()]),
],
offset=offset,
limit=limit,
)
issue_ids = [int(row[0]) for row in results if row[0]]
issues = issue_service.GetIssues(cnxn, issue_ids)
comments = issue_service.GetCommentsForIssues(cnxn, issue_ids)
first_comments = {}
for issue in issues:
first_comments[issue.issue_id] = (comments[issue.issue_id][0].content
if issue.issue_id in comments else "[Empty]")
count = self.verdict_tbl.SelectValue(cnxn,
col='COUNT(*)',
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('issue_id IS NOT NULL', []),
('created > %s', [since.isoformat()]),
])
return issues, first_comments, count
def GetTrainingComments(self, cnxn, issue_service, since, offset=0,
limit=100):
"""Returns list of recent comments with human-labeled spam/ham verdicts.
"""
# get all of the manual verdicts in the past day.
results = self.verdict_tbl.Select(
cnxn,
distinct=True,
cols=['comment_id'],
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('comment_id IS NOT NULL', []),
('created > %s', [since.isoformat()]),
],
offset=offset,
limit=limit,
)
comment_ids = [int(row[0]) for row in results if row[0]]
# Don't care about sequence numbers in this context yet.
comments = issue_service.GetCommentsByID(cnxn, comment_ids,
defaultdict(int))
return comments
def ExpungeUsersInSpam(self, cnxn, user_ids):
"""Removes all references to given users from Spam DB tables.
This method will not commit the operations. This method will
not make changes to in-memory data.
"""
commit = False
self.report_tbl.Delete(cnxn, reported_user_id=user_ids, commit=commit)
delta = {'user_id': framework_constants.DELETED_USER_ID}
self.report_tbl.Update(cnxn, delta, user_id=user_ids, commit=commit)
self.verdict_tbl.Update(cnxn, delta, user_id=user_ids, commit=commit)
import datetime
import logging
import random
import re
import threading
import time
from infra_libs import ts_mon
from contextlib import contextmanager
from google.appengine.api import app_identity
PHASE_TIME = ts_mon.CumulativeDistributionMetric(
'monorail/servlet/phase_time', 'Time spent in profiler phases, in ms',
[ts_mon.StringField('phase')])
# trace_service does not like very long names.
MAX_PHASE_NAME_LENGTH = 200
class Profiler(object):
"""Object to record and help display request processing profiling info.
The Profiler class holds a list of phase objects, which can hold additional
phase objects (which are subphases). Each phase or subphase represents some
meaningful part of this application's HTTP request processing.
"""
_COLORS = [
'900', '090', '009', '360', '306', '036', '630', '630', '063', '333'
class BugdroidGitPollerHandler(poller_handlers.BasePollerHandler):
"""Handler for updating bugs with information from commits."""
bug_comments_metric = ts_mon.CounterMetric(
'bugdroid/bug_comments', 'Counter of comments added to bugs',
[ts_mon.StringField('project'),
ts_mon.StringField('status')])
def __init__(self,
monorail,
logger,
default_project,
no_merge=None,
public_bugs=True,
test_mode=False,
issues_labels=None,
*args,
**kwargs):
self.monorail_client = monorail
self.logger = logger
self.default_project = default_project
self.no_merge = no_merge or []
self.public_bugs = public_bugs
self.test_mode = test_mode
if issues_labels:
self.issues_labels = dict((p.key, p.value) for p in issues_labels)
else:
self.issues_labels = {}
super(BugdroidGitPollerHandler, self).__init__(*args, **kwargs)
def _ApplyMergeMergedLabel(self, issue, branch):
if not branch or not issue:
return
label = '%s-%s' % (self.issues_labels.get('merge',
'merge-merged'), branch)
issue.add_label(label)
self.logger.debug('Adding %s', label)
label = self.issues_labels.get('approved', 'merge-approved')
if issue.has_label(label):
issue.remove_label(label)
self.logger.debug('Removing %s', label)
mstone = branch_utils.get_mstone(branch, False)
if mstone:
label = 'merge-approved-%s' % mstone
if issue.has_label(label):
issue.remove_label(label)
self.logger.debug('Removing %s', label)
def ProcessLogEntry(self, log_entry):
project_bugs = log_parser.get_issues(
log_entry, default_project=self.default_project)
self.logger.info('Processing commit %s : bugs %s', log_entry.revision,
str(project_bugs))
if project_bugs:
comment = self._CreateMessage(log_entry)
self.logger.debug(comment)
for project, bugs in project_bugs.iteritems():
for bug in bugs:
try:
issue = self.monorail_client.get_issue(project, bug)
issue.set_comment(comment[:24 * 1024])
branch = scm_helper.GetBranch(log_entry)
# Apply merge labels if this commit landed on a branch.
if branch and not (log_entry.scm in ['git', 'gerrit']
and scm_helper.GetBranch(log_entry,
full=True)
in self.no_merge):
self._ApplyMergeMergedLabel(issue, branch)
self.logger.debug('Attempting to save issue: %d',
issue.id)
if not self.test_mode:
self.monorail_client.update_issue(
project, issue,
log_parser.should_send_email(log_entry.msg))
else:
self.logger.debug('Test mode, skipping')
except Exception:
self.bug_comments_metric.increment({
'project': project,
'status': 'failure'
})
raise
else:
self.bug_comments_metric.increment({
'project': project,
'status': 'success'
})
def _CreateMessage(self, log_entry):
msg = ''
msg += 'The following revision refers to this bug:\n'
msg += ' %s\n\n' % log_entry.GetCommitUrl()
msg += self._BuildLogSpecial(log_entry)
return msg
def _BuildLogSpecial(self, log_entry):
"""Generate git-log style message, with links to files in the Web UI."""
rtn = 'commit %s\n' % log_entry.commit
rtn += 'Author: %s <%s>\n' % (log_entry.author_name,
log_entry.author_email)
rtn += 'Date: %s\n' % log_entry.committer_date
if self.public_bugs:
rtn += '\n%s\n' % log_entry.msg
for path in log_entry.paths:
if path.action == 'delete':
# Use parent and copy_from_path for deletions, otherwise we get links
# to https://.../<commit>//dev/null
rtn += '[%s] %s\n' % (
path.action,
log_entry.GetPathUrl(
path.copy_from_path, parent=True, universal=True))
else:
rtn += '[%s] %s\n' % (path.action,
log_entry.GetPathUrl(path.filename,
universal=True))
return rtn
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from framework import authdata
from framework import sql
from framework import xsrf
from gae_ts_mon.handlers import TSMonJSHandler
from google.appengine.api import users
from infra_libs import ts_mon
STANDARD_FIELDS = [
ts_mon.StringField('client_id'),
ts_mon.StringField('host_name'),
ts_mon.BooleanField('document_visible'),
]
# User action metrics.
ISSUE_CREATE_LATENCY_METRIC = ts_mon.CumulativeDistributionMetric(
'monorail/frontend/issue_create_latency',
('Latency between Issue Entry form submission and page load of '
'the subsequent issue page.'),
field_spec=STANDARD_FIELDS,
units=ts_mon.MetricsDataUnits.MILLISECONDS)
ISSUE_UPDATE_LATENCY_METRIC = ts_mon.CumulativeDistributionMetric(
'monorail/frontend/issue_update_latency',
('Latency between Issue Update form submission and page load of '
'the subsequent issue page.'),
class RateLimiter(object):
blocked_requests = ts_mon.CounterMetric(
'monorail/ratelimiter/blocked_request',
'Count of requests that exceeded the rate limit and were blocked.',
None)
limit_exceeded = ts_mon.CounterMetric(
'monorail/ratelimiter/rate_exceeded',
'Count of requests that exceeded the rate limit.', None)
cost_thresh_exceeded = ts_mon.CounterMetric(
'monorail/ratelimiter/cost_thresh_exceeded',
'Count of requests that were expensive to process', None)
checks = ts_mon.CounterMetric(
'monorail/ratelimiter/check',
'Count of checks done, by fail/success type.',
[ts_mon.StringField('type')])
def __init__(self, _cache=memcache, fail_open=True, **_kwargs):
self.fail_open = fail_open
def CheckStart(self, request, now=None):
if (modules.get_current_module_name() not in MODULE_WHITELIST
or users.is_current_user_admin()):
return
logging.info('X-AppEngine-Country: %s' %
request.headers.get(COUNTRY_HEADER, 'ZZ'))
if now is None:
now = time.time()
keysets, country, ip, user_email = _CacheKeys(request, now)
# There are either two or three sets of keys in keysets.
# Three if the user's country is in COUNTRY_LIMITS, otherwise two.
self._AuxCheckStart(
keysets, COUNTRY_LIMITS.get(country, DEFAULT_LIMIT),
settings.ratelimiting_enabled,
RateLimitExceeded(country=country, ip=ip, user_email=user_email))
def _AuxCheckStart(self, keysets, limit, ratelimiting_enabled,
exception_obj):
for keys in keysets:
count = 0
try:
counters = memcache.get_multi(keys)
count = sum(counters.values())
self.checks.increment({'type': 'success'})
except Exception as e:
logging.error(e)
if not self.fail_open:
self.checks.increment({'type': 'fail_closed'})
raise exception_obj
self.checks.increment({'type': 'fail_open'})
if count > limit:
# Since webapp2 won't let us return a 429 error code
# <http://tools.ietf.org/html/rfc6585#section-4>, we can't
# monitor rate limit exceeded events with our standard tools.
# We return a 400 with a custom error message to the client,
# and this logging is so we can monitor it internally.
logging.info('%s, %d' % (exception_obj.message, count))
self.limit_exceeded.increment()
if ratelimiting_enabled:
self.blocked_requests.increment()
raise exception_obj
k = keys[0]
# Only update the latest *time* bucket for each prefix (reverse chron).
memcache.add(k, 0, time=EXPIRE_AFTER_SECS)
memcache.incr(k, initial_value=0)
def CheckEnd(self, request, now, start_time):
"""If a request was expensive to process, charge some extra points
against this set of buckets.
We pass in both now and start_time so we can update the buckets
based on keys created from start_time instead of now.
now and start_time are float seconds.
"""
if (modules.get_current_module_name() not in MODULE_WHITELIST
or not settings.ratelimiting_cost_enabled):
return
elapsed_ms = (now - start_time) * 1000
# Would it kill the python lib maintainers to have timedelta.total_ms()?
if elapsed_ms < settings.ratelimiting_cost_thresh_ms:
return
# TODO: Look into caching the keys instead of generating them twice
# for every request. Say, return them from CheckStart so they can
# be bassed back in here later.
keysets, country, ip, user_email = _CacheKeys(request, start_time)
self._AuxCheckEnd(
keysets, 'Rate Limit Cost Threshold Exceeded: %s, %s, %s' %
(country, ip, user_email), settings.ratelimiting_cost_penalty)
def _AuxCheckEnd(self, keysets, log_str, ratelimiting_cost_penalty):
self.cost_thresh_exceeded.increment()
for keys in keysets:
logging.info(log_str)
# Only update the latest *time* bucket for each prefix (reverse chron).
k = keys[0]
memcache.add(k, 0, time=EXPIRE_AFTER_SECS)
memcache.incr(k, delta=ratelimiting_cost_penalty, initial_value=0)
def RecordSubmissionMetrics(action_history, submitted_change_strategies):
"""Record submission metrics in monarch.
Args:
action_history: A CLActionHistory instance for all cl actions for all
changes in submitted_change_strategies.
submitted_change_strategies: A dictionary from GerritPatchTuples to
submission strategy strings. These changes will have their handling
times recorded in monarch.
"""
# TODO(phobbs) move to top level after crbug.com/755415
handling_time_metric = metrics.CumulativeSecondsDistribution(
constants.MON_CL_HANDLE_TIME)
wall_clock_time_metric = metrics.CumulativeSecondsDistribution(
constants.MON_CL_WALL_CLOCK_TIME)
precq_time_metric = metrics.CumulativeSecondsDistribution(
constants.MON_CL_PRECQ_TIME)
wait_time_metric = metrics.CumulativeSecondsDistribution(
constants.MON_CL_WAIT_TIME)
cq_run_time_metric = metrics.CumulativeSecondsDistribution(
constants.MON_CL_CQRUN_TIME)
cq_tries_metric = metrics.CumulativeSmallIntegerDistribution(
constants.MON_CL_CQ_TRIES)
# These 3 false rejection metrics are different in subtle but important ways.
# false_rejections: distribution of the number of times a CL was rejected,
# broken down by what it was rejected by (cq vs. pre-cq). Every CL will emit
# two data points to this distribution.
false_rejection_metric = metrics.CumulativeSmallIntegerDistribution(
constants.MON_CL_FALSE_REJ)
# false_rejections_total: distribution of the total number of times a CL
# was rejected (not broken down by phase). Note that there is no way to
# independently calculate this from |false_rejections| distribution above,
# since one cannot reconstruct after the fact which pre-cq and cq data points
# (for the same underlying CL) belong together.
false_rejection_total_metric = metrics.CumulativeSmallIntegerDistribution(
constants.MON_CL_FALSE_REJ_TOTAL)
# false_rejection_count: counter of the total number of false rejections that
# have occurred (broken down by phase)
false_rejection_count_metric = metrics.Counter(
constants.MON_CL_FALSE_REJ_COUNT)
# This metric excludes rejections which were forgiven by the CL-exonerator
# service.
false_rejections_minus_exonerations_metric = \
metrics.CumulativeSmallIntegerDistribution(
constants.MON_CQ_FALSE_REJ_MINUS_EXONERATIONS,
description='The number of rejections - exonerations per CL.',
field_spec=[ts_mon.StringField('submission_strategy')],
)
precq_false_rejections = action_history.GetFalseRejections(
bot_type=constants.PRE_CQ)
cq_false_rejections = action_history.GetFalseRejections(
bot_type=constants.CQ)
exonerations = action_history.GetExonerations()
for change, strategy in submitted_change_strategies.iteritems():
strategy = strategy or ''
handling_time = clactions.GetCLHandlingTime(change, action_history)
wall_clock_time = clactions.GetCLWallClockTime(change, action_history)
precq_time = clactions.GetPreCQTime(change, action_history)
wait_time = clactions.GetCQWaitTime(change, action_history)
run_time = clactions.GetCQRunTime(change, action_history)
pickups = clactions.GetCQAttemptsCount(change, action_history)
fields = {'submission_strategy': strategy}
handling_time_metric.add(handling_time, fields=fields)
wall_clock_time_metric.add(wall_clock_time, fields=fields)
precq_time_metric.add(precq_time, fields=fields)
wait_time_metric.add(wait_time, fields=fields)
cq_run_time_metric.add(run_time, fields=fields)
cq_tries_metric.add(pickups, fields=fields)
rejection_types = (
(constants.PRE_CQ, precq_false_rejections),
(constants.CQ, cq_false_rejections),
)
total_rejections = 0
for by, rej in rejection_types:
c = len(rej.get(change, []))
f = dict(fields, rejected_by=by)
false_rejection_metric.add(c, fields=f)
false_rejection_count_metric.increment_by(c, fields=f)
total_rejections += c
false_rejection_total_metric.add(total_rejections, fields=fields)
n_exonerations = len(exonerations.get(change, []))
# TODO(crbug.com/804900) max(0, ...) is required because of an accounting
# bug where sometimes this quantity is negative.
net_rejections = total_rejections - n_exonerations
if net_rejections < 0:
logging.error(
'Exonerations is larger than total rejections for CL %s.'
' See crbug.com/804900', change)
false_rejections_minus_exonerations_metric.add(max(0, net_rejections),
fields=fields)
LOGGER = logging.getLogger(__name__)
LoopResults = collections.namedtuple(
'LoopResults',
[
# True on no errors or if all failed attempts were successfully retried.
'success',
# Total number of errors seen (some may have been fixed with retries).
'error_count',
],
)
count_metric = ts_mon.CounterMetric(
'proc/outer_loop/count',
'Counter of loop iterations for this process, by success or failure',
[ts_mon.StringField('status')])
success_metric = ts_mon.BooleanMetric('proc/outer_loop/success',
'Set immediately before the loop exits',
None)
durations_metric = ts_mon.CumulativeDistributionMetric(
'proc/outer_loop/durations',
'Times (in seconds) taken to execute the task', None)
def loop(task, sleep_timeout, duration=None, max_errors=None, time_mod=time):
"""Runs the task in a loop for a given duration.
Handles and logs all uncaught exceptions. ``task`` callback should return True
on success, and False (or raise an exception) in error.
Doesn't leak any exceptions (including KeyboardInterrupt).
import os
import sys
import urlparse
from infra.libs import git2
from infra.libs.service_utils import outer_loop
from infra.services.gsubtreed import gsubtreed
from infra_libs import logs
from infra_libs import ts_mon
# Return value of parse_args.
Options = collections.namedtuple('Options', 'repo loop_opts json_output')
commits_counter = ts_mon.CounterMetric(
'gsubtreed/commit_count', 'Number of commits processed by gsubtreed',
[ts_mon.StringField('path')])
def parse_args(args): # pragma: no cover
def check_url(s):
parsed = urlparse.urlparse(s)
if parsed.scheme not in ('https', 'git', 'file'):
raise argparse.ArgumentTypeError(
'Repo URL must use https, git or file protocol.')
if not parsed.path.strip('/'):
raise argparse.ArgumentTypeError('URL is missing a path?')
return git2.Repo(s)
parser = argparse.ArgumentParser('./run.py %s' % __package__)
parser.add_argument('--dry_run',
action='store_true',
class MonorailApi(remote.Service):
# Class variables. Handy to mock.
_services = None
_mar = None
api_requests = ts_mon.CounterMetric(
'monorail/api_requests',
'Number of requests to Monorail api',
[ts_mon.StringField('client_id'), ts_mon.StringField('client_email')])
ratelimiter = ratelimiter.ApiRateLimiter()
@classmethod
def _set_services(cls, services):
cls._services = services
def mar_factory(self, request):
if not self._mar:
self._mar = monorailrequest.MonorailApiRequest(request, self._services)
return self._mar
def aux_delete_comment(self, request, delete=True):
mar = self.mar_factory(request)
action_name = 'delete' if delete else 'undelete'
issue = self._services.issue.GetIssueByLocalID(
mar.cnxn, mar.project_id, request.issueId)
all_comments = self._services.issue.GetCommentsForIssue(
mar.cnxn, issue.issue_id)
try:
issue_comment = all_comments[request.commentId]
except IndexError:
raise issue_svc.NoSuchIssueException(
'The issue %s:%d does not have comment %d.' %
(mar.project_name, request.issueId, request.commentId))
if not permissions.CanDelete(
mar.auth.user_id, mar.auth.effective_ids, mar.perms,
issue_comment.deleted_by, issue_comment.user_id, mar.project,
permissions.GetRestrictions(issue), mar.granted_perms):
raise permissions.PermissionException(
'User is not allowed to %s the comment %d of issue %s:%d' %
(action_name, request.commentId, mar.project_name,
request.issueId))
self._services.issue.SoftDeleteComment(
mar.cnxn, mar.project_id, request.issueId, request.commentId,
mar.auth.user_id, self._services.user, delete=delete)
return api_pb2_v1.IssuesCommentsDeleteResponse()
def increment_request_limit(self, request, client_id, client_email):
"""Check whether the requester has exceeded API quotas limit,
and increment request count in DB and ts_mon.
"""
mar = self.mar_factory(request)
# soft_limit == hard_limit for api_request, so this function either
# returns False if under limit, or raise ExcessiveActivityException
if not actionlimit.NeedCaptcha(
mar.auth.user_pb, actionlimit.API_REQUEST, skip_lifetime_check=True):
actionlimit.CountAction(
mar.auth.user_pb, actionlimit.API_REQUEST, delta=1)
self._services.user.UpdateUser(
mar.cnxn, mar.auth.user_id, mar.auth.user_pb)
# Avoid value explosision and protect PII info
if not framework_helpers.IsServiceAccount(client_email):
client_email = '*****@*****.**'
self.api_requests.increment_by(
1, {'client_id': client_id, 'client_email': client_email})
@monorail_api_method(
api_pb2_v1.ISSUES_COMMENTS_DELETE_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesCommentsDeleteResponse,
path='projects/{projectId}/issues/{issueId}/comments/{commentId}',
http_method='DELETE',
name='issues.comments.delete')
def issues_comments_delete(self, request):
"""Delete a comment."""
return self.aux_delete_comment(request, True)
@monorail_api_method(
api_pb2_v1.ISSUES_COMMENTS_INSERT_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesCommentsInsertResponse,
path='projects/{projectId}/issues/{issueId}/comments',
http_method='POST',
name='issues.comments.insert')
def issues_comments_insert(self, request):
"""Add a comment."""
mar = self.mar_factory(request)
issue = self._services.issue.GetIssueByLocalID(
mar.cnxn, mar.project_id, request.issueId)
old_owner_id = tracker_bizobj.GetOwnerId(issue)
if not permissions.CanCommentIssue(
mar.auth.effective_ids, mar.perms, mar.project, issue,
mar.granted_perms):
raise permissions.PermissionException(
'User is not allowed to comment this issue (%s, %d)' %
(request.projectId, request.issueId))
updates_dict = {}
if request.updates:
if request.updates.moveToProject:
move_to = request.updates.moveToProject.lower()
move_to_project = issuedetail.CheckMoveIssueRequest(
self._services, mar, issue, True, move_to, mar.errors)
if mar.errors.AnyErrors():
raise endpoints.BadRequestException(mar.errors.move_to)
updates_dict['move_to_project'] = move_to_project
updates_dict['summary'] = request.updates.summary
updates_dict['status'] = request.updates.status
if request.updates.owner:
if request.updates.owner == framework_constants.NO_USER_NAME:
updates_dict['owner'] = framework_constants.NO_USER_SPECIFIED
else:
updates_dict['owner'] = self._services.user.LookupUserID(
mar.cnxn, request.updates.owner)
updates_dict['cc_add'], updates_dict['cc_remove'] = (
api_pb2_v1_helpers.split_remove_add(request.updates.cc))
updates_dict['cc_add'] = self._services.user.LookupUserIDs(
mar.cnxn, updates_dict['cc_add'], autocreate=True).values()
updates_dict['cc_remove'] = self._services.user.LookupUserIDs(
mar.cnxn, updates_dict['cc_remove']).values()
updates_dict['labels_add'], updates_dict['labels_remove'] = (
api_pb2_v1_helpers.split_remove_add(request.updates.labels))
blocked_on_add_strs, blocked_on_remove_strs = (
api_pb2_v1_helpers.split_remove_add(request.updates.blockedOn))
updates_dict['blocked_on_add'] = api_pb2_v1_helpers.issue_global_ids(
blocked_on_add_strs, issue.project_id, mar,
self._services)
updates_dict['blocked_on_remove'] = api_pb2_v1_helpers.issue_global_ids(
blocked_on_remove_strs, issue.project_id, mar,
self._services)
blocking_add_strs, blocking_remove_strs = (
api_pb2_v1_helpers.split_remove_add(request.updates.blocking))
updates_dict['blocking_add'] = api_pb2_v1_helpers.issue_global_ids(
blocking_add_strs, issue.project_id, mar,
self._services)
updates_dict['blocking_remove'] = api_pb2_v1_helpers.issue_global_ids(
blocking_remove_strs, issue.project_id, mar,
self._services)
components_add_strs, components_remove_strs = (
api_pb2_v1_helpers.split_remove_add(request.updates.components))
updates_dict['components_add'] = (
api_pb2_v1_helpers.convert_component_ids(
mar.config, components_add_strs))
updates_dict['components_remove'] = (
api_pb2_v1_helpers.convert_component_ids(
mar.config, components_remove_strs))
if request.updates.mergedInto:
merge_project_name, merge_local_id = tracker_bizobj.ParseIssueRef(
request.updates.mergedInto)
merge_into_project = self._services.project.GetProjectByName(
mar.cnxn, merge_project_name or issue.project_name)
merge_into_issue = self._services.issue.GetIssueByLocalID(
mar.cnxn, merge_into_project.project_id, merge_local_id)
merge_allowed = tracker_helpers.IsMergeAllowed(
merge_into_issue, mar, self._services)
if not merge_allowed:
raise permissions.PermissionException(
'User is not allowed to merge into issue %s:%s' %
(merge_into_issue.project_name, merge_into_issue.local_id))
updates_dict['merged_into'] = merge_into_issue.issue_id
(updates_dict['field_vals_add'], updates_dict['field_vals_remove'],
updates_dict['fields_clear'], updates_dict['fields_labels_add'],
updates_dict['fields_labels_remove']) = (
api_pb2_v1_helpers.convert_field_values(
request.updates.fieldValues, mar, self._services))
field_helpers.ValidateCustomFields(
mar, self._services,
(updates_dict.get('field_vals_add', []) +
updates_dict.get('field_vals_remove', [])),
mar.config, mar.errors)
if mar.errors.AnyErrors():
raise endpoints.BadRequestException(
'Invalid field values: %s' % mar.errors.custom_fields)
_, comment = self._services.issue.DeltaUpdateIssue(
cnxn=mar.cnxn, services=self._services,
reporter_id=mar.auth.user_id,
project_id=mar.project_id, config=mar.config, issue=issue,
status=updates_dict.get('status'), owner_id=updates_dict.get('owner'),
cc_add=updates_dict.get('cc_add', []),
cc_remove=updates_dict.get('cc_remove', []),
comp_ids_add=updates_dict.get('components_add', []),
comp_ids_remove=updates_dict.get('components_remove', []),
labels_add=(updates_dict.get('labels_add', []) +
updates_dict.get('fields_labels_add', [])),
labels_remove=(updates_dict.get('labels_remove', []) +
updates_dict.get('fields_labels_remove', [])),
field_vals_add=updates_dict.get('field_vals_add', []),
field_vals_remove=updates_dict.get('field_vals_remove', []),
fields_clear=updates_dict.get('fields_clear', []),
blocked_on_add=updates_dict.get('blocked_on_add', []),
blocked_on_remove=updates_dict.get('blocked_on_remove', []),
blocking_add=updates_dict.get('blocking_add', []),
blocking_remove=updates_dict.get('blocking_remove', []),
merged_into=updates_dict.get('merged_into'),
index_now=False,
comment=request.content,
summary=updates_dict.get('summary'),
)
move_comment = None
if 'move_to_project' in updates_dict:
move_to_project = updates_dict['move_to_project']
old_text_ref = 'issue %s:%s' % (issue.project_name, issue.local_id)
tracker_fulltext.UnindexIssues([issue.issue_id])
moved_back_iids = self._services.issue.MoveIssues(
mar.cnxn, move_to_project, [issue], self._services.user)
new_text_ref = 'issue %s:%s' % (issue.project_name, issue.local_id)
if issue.issue_id in moved_back_iids:
content = 'Moved %s back to %s again.' % (old_text_ref, new_text_ref)
else:
content = 'Moved %s to now be %s.' % (old_text_ref, new_text_ref)
move_comment = self._services.issue.CreateIssueComment(
mar.cnxn, move_to_project.project_id, issue.local_id, mar.auth.user_id,
content, amendments=[
tracker_bizobj.MakeProjectAmendment(move_to_project.project_name)])
if 'merged_into' in updates_dict:
new_starrers = tracker_helpers.GetNewIssueStarrers(
mar.cnxn, self._services, issue.issue_id, merge_into_issue.issue_id)
tracker_helpers.AddIssueStarrers(
mar.cnxn, self._services, mar,
merge_into_issue.issue_id, merge_into_project, new_starrers)
_merge_comment = tracker_helpers.MergeCCsAndAddComment(
self._services, mar, issue, merge_into_project, merge_into_issue)
merge_into_issue_cmnts = self._services.issue.GetCommentsForIssue(
mar.cnxn, merge_into_issue.issue_id)
notify.PrepareAndSendIssueChangeNotification(
merge_into_issue.issue_id, framework_helpers.GetHostPort(),
mar.auth.user_id, len(merge_into_issue_cmnts) - 1, send_email=True)
tracker_fulltext.IndexIssues(
mar.cnxn, [issue], self._services.user, self._services.issue,
self._services.config)
comment = comment or move_comment
if comment is None:
return api_pb2_v1.IssuesCommentsInsertResponse()
cmnts = self._services.issue.GetCommentsForIssue(mar.cnxn, issue.issue_id)
seq = len(cmnts) - 1
if request.sendEmail:
notify.PrepareAndSendIssueChangeNotification(
issue.issue_id, framework_helpers.GetHostPort(),
comment.user_id, seq, send_email=True, old_owner_id=old_owner_id)
can_delete = permissions.CanDelete(
mar.auth.user_id, mar.auth.effective_ids, mar.perms,
comment.deleted_by, comment.user_id, mar.project,
permissions.GetRestrictions(issue), granted_perms=mar.granted_perms)
return api_pb2_v1.IssuesCommentsInsertResponse(
id=seq,
kind='monorail#issueComment',
author=api_pb2_v1_helpers.convert_person(
comment.user_id, mar.cnxn, self._services),
content=comment.content,
published=datetime.datetime.fromtimestamp(comment.timestamp),
updates=api_pb2_v1_helpers.convert_amendments(
issue, comment.amendments, mar, self._services),
canDelete=can_delete)
@monorail_api_method(
api_pb2_v1.ISSUES_COMMENTS_LIST_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesCommentsListResponse,
path='projects/{projectId}/issues/{issueId}/comments',
http_method='GET',
name='issues.comments.list')
def issues_comments_list(self, request):
"""List all comments for an issue."""
mar = self.mar_factory(request)
issue = self._services.issue.GetIssueByLocalID(
mar.cnxn, mar.project_id, request.issueId)
comments = self._services.issue.GetCommentsForIssue(
mar.cnxn, issue.issue_id)
visible_comments = []
for comment in comments[
request.startIndex:(request.startIndex + request.maxResults)]:
visible_comments.append(
api_pb2_v1_helpers.convert_comment(
issue, comment, mar, self._services, mar.granted_perms))
return api_pb2_v1.IssuesCommentsListResponse(
kind='monorail#issueCommentList',
totalResults=len(comments),
items=visible_comments)
@monorail_api_method(
api_pb2_v1.ISSUES_COMMENTS_DELETE_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesCommentsDeleteResponse,
path='projects/{projectId}/issues/{issueId}/comments/{commentId}',
http_method='POST',
name='issues.comments.undelete')
def issues_comments_undelete(self, request):
"""Restore a deleted comment."""
return self.aux_delete_comment(request, False)
@monorail_api_method(
api_pb2_v1.USERS_GET_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.UsersGetResponse,
path='users/{userId}',
http_method='GET',
name='users.get')
def users_get(self, request):
"""Get a user."""
owner_project_only = request.ownerProjectsOnly
mar = self.mar_factory(request)
(visible_ownership, visible_deleted, visible_membership,
visible_contrib) = sitewide_helpers.GetUserProjects(
mar.cnxn, self._services, mar.auth.user_pb, mar.auth.effective_ids,
mar.viewed_user_auth.effective_ids)
project_list = []
for proj in (visible_ownership + visible_deleted):
config = self._services.config.GetProjectConfig(
mar.cnxn, proj.project_id)
proj_result = api_pb2_v1_helpers.convert_project(
proj, config, api_pb2_v1.Role.owner)
project_list.append(proj_result)
if not owner_project_only:
for proj in visible_membership:
config = self._services.config.GetProjectConfig(
mar.cnxn, proj.project_id)
proj_result = api_pb2_v1_helpers.convert_project(
proj, config, api_pb2_v1.Role.member)
project_list.append(proj_result)
for proj in visible_contrib:
config = self._services.config.GetProjectConfig(
mar.cnxn, proj.project_id)
proj_result = api_pb2_v1_helpers.convert_project(
proj, config, api_pb2_v1.Role.contributor)
project_list.append(proj_result)
return api_pb2_v1.UsersGetResponse(
id=str(mar.viewed_user_auth.user_id),
kind='monorail#user',
projects=project_list,
)
@monorail_api_method(
api_pb2_v1.ISSUES_GET_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesGetInsertResponse,
path='projects/{projectId}/issues/{issueId}',
http_method='GET',
name='issues.get')
def issues_get(self, request):
"""Get an issue."""
mar = self.mar_factory(request)
issue = self._services.issue.GetIssueByLocalID(
mar.cnxn, mar.project_id, request.issueId)
return api_pb2_v1_helpers.convert_issue(
api_pb2_v1.IssuesGetInsertResponse, issue, mar, self._services)
@monorail_api_method(
api_pb2_v1.ISSUES_INSERT_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesGetInsertResponse,
path='projects/{projectId}/issues',
http_method='POST',
name='issues.insert')
def issues_insert(self, request):
"""Add a new issue."""
mar = self.mar_factory(request)
if not mar.perms.CanUsePerm(
permissions.CREATE_ISSUE, mar.auth.effective_ids, mar.project, []):
raise permissions.PermissionException(
'The requester %s is not allowed to create issues for project %s.' %
(mar.auth.email, mar.project_name))
owner_id = None
if request.owner:
try:
owner_id = self._services.user.LookupUserID(
mar.cnxn, request.owner.name)
except user_svc.NoSuchUserException:
raise endpoints.BadRequestException(
'The specified owner %s does not exist.' % request.owner.name)
cc_ids = []
if request.cc:
cc_ids = self._services.user.LookupUserIDs(
mar.cnxn, [ap.name for ap in request.cc],
autocreate=True).values()
comp_ids = api_pb2_v1_helpers.convert_component_ids(
mar.config, request.components)
fields_add, _, _, fields_labels, _ = (
api_pb2_v1_helpers.convert_field_values(
request.fieldValues, mar, self._services))
field_helpers.ValidateCustomFields(
mar, self._services, fields_add, mar.config, mar.errors)
if mar.errors.AnyErrors():
raise endpoints.BadRequestException(
'Invalid field values: %s' % mar.errors.custom_fields)
local_id = self._services.issue.CreateIssue(
mar.cnxn, self._services, mar.project_id,
request.summary, request.status, owner_id,
cc_ids, request.labels + fields_labels, fields_add,
comp_ids, mar.auth.user_id, request.description,
blocked_on=api_pb2_v1_helpers.convert_issueref_pbs(
request.blockedOn, mar, self._services),
blocking=api_pb2_v1_helpers.convert_issueref_pbs(
request.blocking, mar, self._services))
new_issue = self._services.issue.GetIssueByLocalID(
mar.cnxn, mar.project_id, local_id)
self._services.issue_star.SetStar(
mar.cnxn, self._services, mar.config, new_issue.issue_id,
mar.auth.user_id, True)
if request.sendEmail:
notify.PrepareAndSendIssueChangeNotification(
new_issue.issue_id, framework_helpers.GetHostPort(),
new_issue.reporter_id, 0)
return api_pb2_v1_helpers.convert_issue(
api_pb2_v1.IssuesGetInsertResponse, new_issue, mar, self._services)
@monorail_api_method(
api_pb2_v1.ISSUES_LIST_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.IssuesListResponse,
path='projects/{projectId}/issues',
http_method='GET',
name='issues.list')
def issues_list(self, request):
"""List issues for projects."""
mar = self.mar_factory(request)
if request.additionalProject:
for project_name in request.additionalProject:
project = self._services.project.GetProjectByName(
mar.cnxn, project_name)
if project and not permissions.UserCanViewProject(
mar.auth.user_pb, mar.auth.effective_ids, project):
raise permissions.PermissionException(
'The user %s has no permission for project %s' %
(mar.auth.email, project_name))
prof = profiler.Profiler()
pipeline = frontendsearchpipeline.FrontendSearchPipeline(
mar, self._services, prof, mar.num)
if not mar.errors.AnyErrors():
pipeline.SearchForIIDs()
pipeline.MergeAndSortIssues()
pipeline.Paginate()
else:
raise endpoints.BadRequestException(mar.errors.query)
issue_list = [
api_pb2_v1_helpers.convert_issue(
api_pb2_v1.IssueWrapper, r, mar, self._services)
for r in pipeline.visible_results]
return api_pb2_v1.IssuesListResponse(
kind='monorail#issueList',
totalResults=pipeline.total_count,
items=issue_list)
@monorail_api_method(
api_pb2_v1.GROUPS_SETTINGS_LIST_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.GroupsSettingsListResponse,
path='groups/settings',
http_method='GET',
name='groups.settings.list')
def groups_settings_list(self, request):
"""List all group settings."""
mar = self.mar_factory(request)
all_groups = self._services.usergroup.GetAllUserGroupsInfo(mar.cnxn)
group_settings = []
for g in all_groups:
setting = g[2]
wrapper = api_pb2_v1_helpers.convert_group_settings(g[0], setting)
if not request.importedGroupsOnly or wrapper.ext_group_type:
group_settings.append(wrapper)
return api_pb2_v1.GroupsSettingsListResponse(
groupSettings=group_settings)
@monorail_api_method(
api_pb2_v1.GROUPS_CREATE_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.GroupsCreateResponse,
path='groups',
http_method='POST',
name='groups.create')
def groups_create(self, request):
"""Create a new user group."""
mar = self.mar_factory(request)
if not permissions.CanCreateGroup(mar.perms):
raise permissions.PermissionException(
'The user is not allowed to create groups.')
user_dict = self._services.user.LookupExistingUserIDs(
mar.cnxn, [request.groupName])
if request.groupName.lower() in user_dict:
raise usergroup_svc.GroupExistsException(
'group %s already exists' % request.groupName)
if request.ext_group_type:
ext_group_type = str(request.ext_group_type).lower()
else:
ext_group_type = None
group_id = self._services.usergroup.CreateGroup(
mar.cnxn, self._services, request.groupName,
str(request.who_can_view_members).lower(),
ext_group_type)
return api_pb2_v1.GroupsCreateResponse(
groupID=group_id)
@monorail_api_method(
api_pb2_v1.GROUPS_GET_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.GroupsGetResponse,
path='groups/{groupName}',
http_method='GET',
name='groups.get')
def groups_get(self, request):
"""Get a group's settings and users."""
mar = self.mar_factory(request)
if not mar.viewed_user_auth:
raise user_svc.NoSuchUserException(request.groupName)
group_id = mar.viewed_user_auth.user_id
group_settings = self._services.usergroup.GetGroupSettings(
mar.cnxn, group_id)
member_ids, owner_ids = self._services.usergroup.LookupAllMembers(
mar.cnxn, [group_id])
(owned_project_ids, membered_project_ids,
contrib_project_ids) = self._services.project.GetUserRolesInAllProjects(
mar.cnxn, mar.auth.effective_ids)
project_ids = owned_project_ids.union(
membered_project_ids).union(contrib_project_ids)
if not permissions.CanViewGroup(
mar.perms, mar.auth.effective_ids, group_settings, member_ids[group_id],
owner_ids[group_id], project_ids):
raise permissions.PermissionException(
'The user is not allowed to view this group.')
member_ids, owner_ids = self._services.usergroup.LookupMembers(
mar.cnxn, [group_id])
member_emails = self._services.user.LookupUserEmails(
mar.cnxn, member_ids[group_id]).values()
owner_emails = self._services.user.LookupUserEmails(
mar.cnxn, owner_ids[group_id]).values()
return api_pb2_v1.GroupsGetResponse(
groupID=group_id,
groupSettings=api_pb2_v1_helpers.convert_group_settings(
request.groupName, group_settings),
groupOwners=owner_emails,
groupMembers=member_emails)
@monorail_api_method(
api_pb2_v1.GROUPS_UPDATE_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.GroupsUpdateResponse,
path='groups/{groupName}',
http_method='POST',
name='groups.update')
def groups_update(self, request):
"""Update a group's settings and users."""
mar = self.mar_factory(request)
group_id = mar.viewed_user_auth.user_id
member_ids_dict, owner_ids_dict = self._services.usergroup.LookupMembers(
mar.cnxn, [group_id])
owner_ids = owner_ids_dict.get(group_id, [])
member_ids = member_ids_dict.get(group_id, [])
if not permissions.CanEditGroup(
mar.perms, mar.auth.effective_ids, owner_ids):
raise permissions.PermissionException(
'The user is not allowed to edit this group.')
group_settings = self._services.usergroup.GetGroupSettings(
mar.cnxn, group_id)
if (request.who_can_view_members or request.ext_group_type
or request.last_sync_time or request.friend_projects):
group_settings.who_can_view_members = (
request.who_can_view_members or group_settings.who_can_view_members)
group_settings.ext_group_type = (
request.ext_group_type or group_settings.ext_group_type)
group_settings.last_sync_time = (
request.last_sync_time or group_settings.last_sync_time)
if framework_constants.NO_VALUES in request.friend_projects:
group_settings.friend_projects = []
else:
id_dict = self._services.project.LookupProjectIDs(
mar.cnxn, request.friend_projects)
group_settings.friend_projects = (
id_dict.values() or group_settings.friend_projects)
self._services.usergroup.UpdateSettings(
mar.cnxn, group_id, group_settings)
if request.groupOwners or request.groupMembers:
self._services.usergroup.RemoveMembers(
mar.cnxn, group_id, owner_ids + member_ids)
owners_dict = self._services.user.LookupUserIDs(
mar.cnxn, request.groupOwners, autocreate=True)
self._services.usergroup.UpdateMembers(
mar.cnxn, group_id, owners_dict.values(), 'owner')
members_dict = self._services.user.LookupUserIDs(
mar.cnxn, request.groupMembers, autocreate=True)
self._services.usergroup.UpdateMembers(
mar.cnxn, group_id, members_dict.values(), 'member')
return api_pb2_v1.GroupsUpdateResponse()
@monorail_api_method(
api_pb2_v1.COMPONENTS_LIST_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.ComponentsListResponse,
path='projects/{projectId}/components',
http_method='GET',
name='components.list')
def components_list(self, request):
"""List all components of a given project."""
mar = self.mar_factory(request)
config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id)
components = [api_pb2_v1_helpers.convert_component_def(
cd, mar, self._services) for cd in config.component_defs]
return api_pb2_v1.ComponentsListResponse(
components=components)
@monorail_api_method(
api_pb2_v1.COMPONENTS_CREATE_REQUEST_RESOURCE_CONTAINER,
api_pb2_v1.Component,
path='projects/{projectId}/components',
http_method='POST',
name='components.create')
def components_create(self, request):
"""Create a component."""
mar = self.mar_factory(request)
if not mar.perms.CanUsePerm(
permissions.EDIT_PROJECT, mar.auth.effective_ids, mar.project, []):
raise permissions.PermissionException(
'User is not allowed to create components for this project')
config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id)
leaf_name = request.componentName
if not tracker_constants.COMPONENT_NAME_RE.match(leaf_name):
raise config_svc.InvalidComponentNameException(
'The component name %s is invalid.' % leaf_name)
parent_path = request.parentPath
if parent_path:
parent_def = tracker_bizobj.FindComponentDef(parent_path, config)
if not parent_def:
raise config_svc.NoSuchComponentException(
'Parent component %s does not exist.' % parent_path)
if not permissions.CanEditComponentDef(
mar.auth.effective_ids, mar.perms, mar.project, parent_def, config):
raise permissions.PermissionException(
'User is not allowed to add a subcomponent to component %s' %
parent_path)
path = '%s>%s' % (parent_path, leaf_name)
else:
path = leaf_name
if tracker_bizobj.FindComponentDef(path, config):
raise config_svc.InvalidComponentNameException(
'The name %s is already in use.' % path)
created = int(time.time())
user_emails = set()
user_emails.update([mar.auth.email] + request.admin + request.cc)
user_ids_dict = self._services.user.LookupUserIDs(
mar.cnxn, list(user_emails), autocreate=False)
admin_ids = [user_ids_dict[uname] for uname in request.admin]
cc_ids = [user_ids_dict[uname] for uname in request.cc]
label_ids = [] # TODO(jrobbins): allow API clients to specify this too.
component_id = self._services.config.CreateComponentDef(
mar.cnxn, mar.project_id, path, request.description, request.deprecated,
admin_ids, cc_ids, created, user_ids_dict[mar.auth.email], label_ids)
return api_pb2_v1.Component(
componentId=component_id,
projectName=request.projectId,
componentPath=path,
description=request.description,
admin=request.admin,
cc=request.cc,
deprecated=request.deprecated,
created=datetime.datetime.fromtimestamp(created),
creator=mar.auth.email)
@monorail_api_method(
api_pb2_v1.COMPONENTS_DELETE_REQUEST_RESOURCE_CONTAINER,
message_types.VoidMessage,
path='projects/{projectId}/components/{componentPath}',
http_method='DELETE',
name='components.delete')
def components_delete(self, request):
"""Delete a component."""
mar = self.mar_factory(request)
config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id)
component_path = request.componentPath
component_def = tracker_bizobj.FindComponentDef(
component_path, config)
if not component_def:
raise config_svc.NoSuchComponentException(
'The component %s does not exist.' % component_path)
if not permissions.CanViewComponentDef(
mar.auth.effective_ids, mar.perms, mar.project, component_def):
raise permissions.PermissionException(
'User is not allowed to view this component %s' % component_path)
if not permissions.CanEditComponentDef(
mar.auth.effective_ids, mar.perms, mar.project, component_def, config):
raise permissions.PermissionException(
'User is not allowed to delete this component %s' % component_path)
allow_delete = not tracker_bizobj.FindDescendantComponents(
config, component_def)
if not allow_delete:
raise permissions.PermissionException(
'User tried to delete component that had subcomponents')
self._services.issue.DeleteComponentReferences(
mar.cnxn, component_def.component_id)
self._services.config.DeleteComponentDef(
mar.cnxn, mar.project_id, component_def.component_id)
return message_types.VoidMessage()
@monorail_api_method(
api_pb2_v1.COMPONENTS_UPDATE_REQUEST_RESOURCE_CONTAINER,
message_types.VoidMessage,
path='projects/{projectId}/components/{componentPath}',
http_method='POST',
name='components.update')
def components_update(self, request):
"""Update a component."""
mar = self.mar_factory(request)
config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id)
component_path = request.componentPath
component_def = tracker_bizobj.FindComponentDef(
component_path, config)
if not component_def:
raise config_svc.NoSuchComponentException(
'The component %s does not exist.' % component_path)
if not permissions.CanViewComponentDef(
mar.auth.effective_ids, mar.perms, mar.project, component_def):
raise permissions.PermissionException(
'User is not allowed to view this component %s' % component_path)
if not permissions.CanEditComponentDef(
mar.auth.effective_ids, mar.perms, mar.project, component_def, config):
raise permissions.PermissionException(
'User is not allowed to edit this component %s' % component_path)
original_path = component_def.path
new_path = component_def.path
new_docstring = component_def.docstring
new_deprecated = component_def.deprecated
new_admin_ids = component_def.admin_ids
new_cc_ids = component_def.cc_ids
update_filterrule = False
for update in request.updates:
if update.field == api_pb2_v1.ComponentUpdateFieldID.LEAF_NAME:
leaf_name = update.leafName
if not tracker_constants.COMPONENT_NAME_RE.match(leaf_name):
raise config_svc.InvalidComponentNameException(
'The component name %s is invalid.' % leaf_name)
if '>' in original_path:
parent_path = original_path[:original_path.rindex('>')]
new_path = '%s>%s' % (parent_path, leaf_name)
else:
new_path = leaf_name
conflict = tracker_bizobj.FindComponentDef(new_path, config)
if conflict and conflict.component_id != component_def.component_id:
raise config_svc.InvalidComponentNameException(
'The name %s is already in use.' % new_path)
update_filterrule = True
elif update.field == api_pb2_v1.ComponentUpdateFieldID.DESCRIPTION:
new_docstring = update.description
elif update.field == api_pb2_v1.ComponentUpdateFieldID.ADMIN:
user_ids_dict = self._services.user.LookupUserIDs(
mar.cnxn, list(update.admin), autocreate=True)
new_admin_ids = [user_ids_dict[email] for email in update.admin]
elif update.field == api_pb2_v1.ComponentUpdateFieldID.CC:
user_ids_dict = self._services.user.LookupUserIDs(
mar.cnxn, list(update.cc), autocreate=True)
new_cc_ids = [user_ids_dict[email] for email in update.cc]
update_filterrule = True
elif update.field == api_pb2_v1.ComponentUpdateFieldID.DEPRECATED:
new_deprecated = update.deprecated
else:
logging.error('Unknown component field %r', update.field)
new_modified = int(time.time())
new_modifier_id = self._services.user.LookupUserID(
mar.cnxn, mar.auth.email, autocreate=False)
logging.info(
'Updating component id %d: path-%s, docstring-%s, deprecated-%s,'
' admin_ids-%s, cc_ids-%s modified by %s', component_def.component_id,
new_path, new_docstring, new_deprecated, new_admin_ids, new_cc_ids,
new_modifier_id)
self._services.config.UpdateComponentDef(
mar.cnxn, mar.project_id, component_def.component_id,
path=new_path, docstring=new_docstring, deprecated=new_deprecated,
admin_ids=new_admin_ids, cc_ids=new_cc_ids, modified=new_modified,
modifier_id=new_modifier_id)
# TODO(sheyang): reuse the code in componentdetails
if original_path != new_path:
# If the name changed then update all of its subcomponents as well.
subcomponent_ids = tracker_bizobj.FindMatchingComponentIDs(
original_path, config, exact=False)
for subcomponent_id in subcomponent_ids:
if subcomponent_id == component_def.component_id:
continue
subcomponent_def = tracker_bizobj.FindComponentDefByID(
subcomponent_id, config)
subcomponent_new_path = subcomponent_def.path.replace(
original_path, new_path, 1)
self._services.config.UpdateComponentDef(
mar.cnxn, mar.project_id, subcomponent_def.component_id,
path=subcomponent_new_path)
if update_filterrule:
filterrules_helpers.RecomputeAllDerivedFields(
mar.cnxn, self._services, mar.project, config)
return message_types.VoidMessage()
from twisted.cred import credentials
import buildslave
from buildslave.pbutil import ReconnectingPBClientFactory
from buildslave.commands import registry, base
from buildslave import monkeypatches
from infra_libs import ts_mon
connected_metric = ts_mon.BooleanMetric(
'buildbot/slave/connected',
'Whether the slave is currently connected to its master.', None)
connection_failures_metric = ts_mon.CounterMetric(
'buildbot/slave/connection_failures',
'Count of failures connecting to the buildbot master.',
[ts_mon.StringField('reason')])
running_metric = ts_mon.BooleanMetric(
'buildbot/slave/is_building',
'Whether a build step is currently in progress.',
[ts_mon.StringField('builder')])
steps_metric = ts_mon.CounterMetric(
'buildbot/slave/steps',
'Count of build steps run by each builder on this slave.',
[ts_mon.StringField('builder'),
ts_mon.BooleanField('success')])
class UnknownCommand(pb.Error):
pass
def PerformStage(self):
"""Perform the actual work for this stage.
This includes final metadata archival, and update CIDB with our final status
as well as producting a logged build result summary.
"""
build_identifier, _ = self._run.GetCIDBHandle()
build_id = build_identifier.cidb_id
buildbucket_id = build_identifier.buildbucket_id
if results_lib.Results.BuildSucceededSoFar(self.buildstore,
buildbucket_id, self.name):
final_status = constants.BUILDER_STATUS_PASSED
else:
final_status = constants.BUILDER_STATUS_FAILED
if not hasattr(self._run.attrs, 'release_tag'):
# If, for some reason, sync stage was not completed and
# release_tag was not set. Set it to None here because
# ArchiveResults() depends the existence of this attr.
self._run.attrs.release_tag = None
# Set up our report metadata.
self._run.attrs.metadata.UpdateWithDict(
self.GetReportMetadata(
final_status=final_status,
completion_instance=self._completion_instance))
src_root = self._build_root
# Workspace builders use a different buildroot for overlays.
if self._run.config.workspace_branch and self._run.options.workspace:
src_root = self._run.options.workspace
# Add tags for the arches and statuses of the build.
# arches requires crossdev which isn't available at the early part of the
# build.
arches = []
for board in self._run.config['boards']:
toolchains = toolchain.GetToolchainsForBoard(board,
buildroot=src_root)
default = list(
toolchain.FilterToolchains(toolchains, 'default', True))
if default:
try:
arches.append(toolchain.GetArchForTarget(default[0]))
except cros_build_lib.RunCommandError as e:
logging.warning(
'Unable to retrieve arch for board %s default toolchain %s: %s',
board, default, e)
tags = {
'arches': arches,
'status': final_status,
}
results = self._run.attrs.metadata.GetValue('results')
for stage in results:
tags['stage_status:%s' % stage['name']] = stage['status']
tags['stage_summary:%s' % stage['name']] = stage['summary']
self._run.attrs.metadata.UpdateKeyDictWithDict(constants.METADATA_TAGS,
tags)
# Some operations can only be performed if a valid version is available.
try:
self._run.GetVersionInfo()
self.ArchiveResults(final_status)
metadata_url = os.path.join(self.upload_url,
constants.METADATA_JSON)
except cbuildbot_run.VersionNotSetError:
logging.error('A valid version was never set for this run. '
'Can not archive results.')
metadata_url = ''
results_lib.Results.Report(sys.stdout,
current_version=(self._run.attrs.release_tag
or ''))
# Upload goma log if used for BuildPackage and TestSimpleChrome.
_UploadAndLinkGomaLogIfNecessary(
'BuildPackages', self._run.config.name, self._run.options.goma_dir,
self._run.options.goma_client_json,
self._run.attrs.metadata.GetValueWithDefault('goma_tmp_dir'))
_UploadAndLinkGomaLogIfNecessary(
'TestSimpleChromeWorkflow', self._run.config.name,
self._run.options.goma_dir, self._run.options.goma_client_json,
self._run.attrs.metadata.GetValueWithDefault(
'goma_tmp_dir_for_simple_chrome'))
if self.buildstore.AreClientsReady():
status_for_db = final_status
# TODO(pprabhu): After BuildData and CBuildbotMetdata are merged, remove
# this extra temporary object creation.
# XXX:HACK We're creating a BuildData with an empty URL. Don't try to
# MarkGathered this object.
build_data = metadata_lib.BuildData(
'', self._run.attrs.metadata.GetDict())
# TODO(akeshet): Find a clearer way to get the "primary upload url" for
# the metadata.json file. One alternative is _GetUploadUrls(...)[0].
# Today it seems that element 0 of its return list is the primary upload
# url, but there is no guarantee or unit test coverage of that.
self.buildstore.FinishBuild(build_id,
status=status_for_db,
summary=build_data.failure_message,
metadata_url=metadata_url)
duration = self._GetBuildDuration()
mon_fields = {
'status': status_for_db,
'build_config': self._run.config.name,
'important': self._run.config.important
}
metrics.Counter(
constants.MON_BUILD_COMP_COUNT).increment(fields=mon_fields)
metrics.CumulativeSecondsDistribution(
constants.MON_BUILD_DURATION).add(duration, fields=mon_fields)
if self._run.options.sanity_check_build:
metrics.Counter(
constants.MON_BUILD_SANITY_COMP_COUNT).increment(
fields=mon_fields)
metrics.Gauge(
constants.MON_BUILD_SANITY_ID,
description=
'The build number of the latest sanity build. Used '
'for recovering the link to the latest failing build '
'in the alert when a sanity build fails.',
field_spec=[
ts_mon.StringField('status'),
ts_mon.StringField('build_config'),
ts_mon.StringField('builder_name'),
ts_mon.BooleanField('important')
]).set(self._run.buildnumber,
fields=dict(
mon_fields,
builder_name=self._run.GetBuilderName()))
if config_lib.IsMasterCQ(self._run.config):
self_destructed = self._run.attrs.metadata.GetValueWithDefault(
constants.SELF_DESTRUCTED_BUILD, False)
mon_fields = {
'status': status_for_db,
'self_destructed': self_destructed
}
metrics.CumulativeSecondsDistribution(
constants.MON_CQ_BUILD_DURATION).add(duration,
fields=mon_fields)
annotator_link = uri_lib.ConstructAnnotatorUri(build_id)
logging.PrintBuildbotLink('Build annotator', annotator_link)
# From this point forward, treat all exceptions as warnings.
self._post_completion = True
# Dump report about things we retry.
retry_stats.ReportStats(sys.stdout)
# Each is identified by an int shard ID.
# And there is one connection to the master DB identified by key MASTER_CNXN.
MASTER_CNXN = 'master_cnxn'
CONNECTION_COUNT = ts_mon.CounterMetric(
'monorail/sql/connection_count',
'Count of connections made to the SQL database.',
[ts_mon.BooleanField('success')])
DB_CNXN_LATENCY = ts_mon.CumulativeDistributionMetric(
'monorail/sql/db_cnxn_latency',
'Time needed to establish a DB connection.', None)
DB_QUERY_LATENCY = ts_mon.CumulativeDistributionMetric(
'monorail/sql/db_query_latency', 'Time needed to make a DB query.',
[ts_mon.StringField('type')])
DB_COMMIT_LATENCY = ts_mon.CumulativeDistributionMetric(
'monorail/sql/db_commit_latency', 'Time needed to make a DB commit.', None)
DB_ROLLBACK_LATENCY = ts_mon.CumulativeDistributionMetric(
'monorail/sql/db_rollback_latency', 'Time needed to make a DB rollback.',
None)
DB_RETRY_COUNT = ts_mon.CounterMetric('monorail/sql/db_retry_count',
'Count of queries retried.', None)
DB_QUERY_COUNT = ts_mon.CounterMetric('monorail/sql/db_query_count',
'Count of queries sent to the DB.',
[ts_mon.StringField('type')])
class LoadApiClientConfigs(webapp2.RequestHandler):
config_loads = ts_mon.CounterMetric(
'monorail/client_config_svc/loads',
'Results of fetches from luci-config.',
[ts_mon.BooleanField('success'),
ts_mon.StringField('type')])
def get(self):
authorization_token, _ = app_identity.get_access_token(
framework_constants.OAUTH_SCOPE)
response = urlfetch.fetch(LUCI_CONFIG_URL,
method=urlfetch.GET,
follow_redirects=False,
headers={
'Content-Type':
'application/json; charset=UTF-8',
'Authorization':
'Bearer ' + authorization_token
})
if response.status_code != 200:
logging.error('Invalid response from luci-config: %r', response)
self.config_loads.increment({
'success': False,
'type': 'luci-cfg-error'
})
self.abort(500, 'Invalid response from luci-config')
try:
content_text = self._process_response(response)
except Exception as e:
self.abort(500, str(e))
logging.info('luci-config content decoded: %r.', content_text)
configs = ClientConfig(configs=content_text,
key_name='api_client_configs')
configs.put()
self.config_loads.increment({'success': True, 'type': 'success'})
def _process_response(self, response):
try:
content = json.loads(response.content)
except ValueError:
logging.error('Response was not JSON: %r', response.content)
self.config_loads.increment({
'success': False,
'type': 'json-load-error'
})
raise
try:
config_content = content['content']
except KeyError:
logging.error('JSON contained no content: %r', content)
self.config_loads.increment({
'success': False,
'type': 'json-key-error'
})
raise
try:
content_text = base64.b64decode(config_content)
except TypeError:
logging.error('Content was not b64: %r', config_content)
self.config_loads.increment({
'success': False,
'type': 'b64-decode-error'
})
raise
try:
cfg = api_clients_config_pb2.ClientCfg()
protobuf.text_format.Merge(content_text, cfg)
except:
logging.error('Content was not a valid ClientCfg proto: %r',
content_text)
self.config_loads.increment({
'success': False,
'type': 'proto-load-error'
})
raise
return content_text
class EventMonUploader(webapp2.RequestHandler):
num_test_results = ts_mon.CounterMetric(
'test_results/num_test_results',
'Number of reported test results',
[ts_mon.StringField('result_type'),
ts_mon.StringField('master'),
ts_mon.StringField('builder'),
ts_mon.StringField('test_type')])
def post(self):
if not self.request.body:
logging.error('Missing request payload')
self.response.set_status(400)
return
try:
payload = json.loads(self.request.body)
except ValueError:
logging.error('Failed to parse request payload as JSON')
self.response.set_status(400)
return
# Retrieve test json from datastore based on task parameters.
master = payload.get('master')
builder = payload.get('builder')
build_number = payload.get('build_number')
test_type = payload.get('test_type')
step_name = payload.get('step_name')
if (not master or not builder or build_number is None or not test_type or
not step_name):
logging.error(
'Missing required parameters: (master=%s, builder=%s, '
'build_number=%s, test_type=%s, step_name=%s)' %
(master, builder, build_number, test_type, step_name))
self.response.set_status(400)
return
files = TestFile.get_files(
master, builder, test_type, build_number, 'full_results.json',
load_data=True, limit=1)
if not files:
logging.error('Failed to find full_results.json for (%s, %s, %s, %s)' % (
master, builder, build_number, test_type))
self.response.set_status(404)
return
file_json = JsonResults.load_json(files[0].data)
# Create a proto event and send it to event_mon.
event = event_mon.Event('POINT')
test_results = event.proto.test_results
test_results.master_name = master
test_results.builder_name = builder
test_results.build_number = int(build_number)
test_results.test_type = test_type
test_results.step_name = step_name
if 'interrupted' in file_json:
test_results.interrupted = file_json['interrupted']
if 'version' in file_json:
test_results.version = file_json['version']
if 'seconds_since_epoch' in file_json:
test_results.usec_since_epoch = long(
float(file_json['seconds_since_epoch']) * 1000 * 1000)
def convert_test_result_type(json_val):
self.num_test_results.increment({
'result_type': json_val, 'master': master, 'builder': builder,
'test_type': test_type})
try:
return (event_mon.protos.chrome_infra_log_pb2.TestResultsEvent.
TestResultType.Value(json_val.upper().replace('+', '_')))
except ValueError:
return event_mon.protos.chrome_infra_log_pb2.TestResultsEvent.UNKNOWN
tests = util.flatten_tests_trie(
file_json.get('tests', {}), file_json.get('path_delimiter', '/'))
for name, test in tests.iteritems():
test_result = test_results.tests.add()
test_result.test_name = name
test_result.actual.extend(
convert_test_result_type(res) for res in test['actual'])
test_result.expected.extend(
convert_test_result_type(res) for res in test['expected'])
event.send()
class ServiceThread(threading.Thread):
"""Thread that controls a single Service object.
The methods on this object (start_service(), stop_service(), etc.) can be
called from any thread and are asynchronous - they just instruct the thread to
perform the given action on the Service.
This thread also polls the service occasionally and restarts it if it crashed.
"""
failures = ts_mon.CounterMetric('service_manager/failures',
'Number of times each service unexpectedly exited and was restarted by '
'service manager',
[ts_mon.StringField('service')])
reconfigs = ts_mon.CounterMetric('service_manager/reconfigs',
'Number of times each service was restarted because its configuration '
'changed',
[ts_mon.StringField('service')])
upgrades = ts_mon.CounterMetric('service_manager/upgrades',
'Number of times each service was restarted because its version '
'changed',
[ts_mon.StringField('service')])
def __init__(self, poll_interval, state_directory, service_config,
cloudtail,
wait_condition=None):
"""
Args:
poll_interval: How often (in seconds) to restart failed services.
state_directory: A file will be created in this directory (with the same
name as the service) when it is running containing its PID and
starttime.
service_config: A dictionary containing the service's config. See README
for a description of the fields.
cloudtail: An object that knows how to start cloudtail.
"""
super(ServiceThread, self).__init__()
if wait_condition is None: # pragma: no cover
wait_condition = threading.Condition()
self._poll_interval = poll_interval
self._state_directory = state_directory
self._cloudtail = cloudtail
self._service = service.Service(self._state_directory,
service_config,
self._cloudtail)
self._condition = wait_condition # Protects _state.
self._state = _State() # _condition must be held.
self._state_changed = False
self._started = False # Whether we started the service already.
def _wait(self):
with self._condition:
if not self._state_changed: # pragma: no cover
self._condition.wait(self._poll_interval)
# Clone the state object so we can release the lock.
ret = self._state.clone()
self._state.new_config = None
self._state_changed = False
return ret
@contextlib.contextmanager
def _change_state(self):
with self._condition:
yield
self._state_changed = True
self._condition.notify()
def run(self):
while True:
try:
state = self._wait()
if state.exit:
return
elif state.new_config is not None:
# Stop the service if it's currently running.
self._service.stop()
# Recreate it with the new config and start it.
self.reconfigs.increment(fields={'service': self._service.name})
self._service = service.Service(self._state_directory,
state.new_config,
self._cloudtail)
self._service.start()
self._started = True
elif state.should_run == False:
# Ensure the service is stopped.
self._service.stop()
self._started = False
elif state.should_run == True:
try:
proc_state = self._service.get_running_process_state()
except service.UnexpectedProcessStateError:
self.failures.increment(fields={'service': self._service.name})
logging.exception('Unexpected error getting state for service %s',
self._service.name)
except service.ProcessNotRunning as ex:
if self._started:
# We started it last time but it's not running any more.
self.failures.increment(fields={'service': self._service.name})
LOGGER.warning('Service %s failed (%r), restarting',
self._service.name, ex)
else:
# We're about to start it for the first time.
LOGGER.info('Starting service %s for the first time (%r)',
self._service.name, ex)
else:
if self._service.has_version_changed(proc_state):
self.upgrades.increment(fields={'service': self._service.name})
LOGGER.info('Service %s has a new package version, restarting',
self._service.name)
self._service.stop()
elif self._service.has_cmd_changed(proc_state):
self.reconfigs.increment(fields={'service': self._service.name})
LOGGER.info(
'Service %s has new command: was %s, restarting with %s',
self._service.name, proc_state.cmd, self._service.cmd)
self._service.stop()
# Ensure the service is running.
self._service.start()
self._started = True
except Exception:
LOGGER.exception('Service thread failed for service %s',
self._service.name)
def start_service(self):
with self._change_state():
self._state.should_run = True
def stop_service(self):
with self._change_state():
self._state.should_run = False
def stop(self, join=True):
with self._change_state():
self._state.exit = True
if join: # pragma: no cover
self.join()
def restart_with_new_config(self, new_config):
with self._change_state():
self._state.new_config = new_config
self._state.should_run = True
class SpamService(object):
"""The persistence layer for spam reports."""
issue_actions = ts_mon.CounterMetric(
'monorail/spam_svc/issue', 'Count of things that happen to issues.',
[ts_mon.StringField('type')])
comment_actions = ts_mon.CounterMetric(
'monorail/spam_svc/comment',
'Count of things that happen to comments.',
[ts_mon.StringField('type')])
prediction_api_failures = ts_mon.CounterMetric(
'mononrail/spam_svc/prediction_api_failure',
'Failures calling the prediction API', None)
def __init__(self):
self.report_tbl = sql.SQLTableManager(SPAMREPORT_TABLE_NAME)
self.verdict_tbl = sql.SQLTableManager(SPAMVERDICT_TABLE_NAME)
self.issue_tbl = sql.SQLTableManager(ISSUE_TABLE)
self.prediction_service = None
try:
credentials = GoogleCredentials.get_application_default()
self.prediction_service = build('prediction',
'v1.6',
http=httplib2.Http(),
credentials=credentials)
except (Oauth2ClientError, ApiClientError):
logging.error("Error getting GoogleCredentials: %s" %
sys.exc_info()[0])
def LookupIssueFlaggers(self, cnxn, issue_id):
"""Returns users who've reported the issue or its comments as spam.
Returns a tuple. First element is a list of users who flagged the issue;
second element is a dictionary of comment id to a list of users who flagged
that comment.
"""
rows = self.report_tbl.Select(cnxn,
cols=['user_id', 'comment_id'],
issue_id=issue_id)
issue_reporters = []
comment_reporters = collections.defaultdict(list)
for row in rows:
if row[1]:
comment_reporters[row[1]].append(row[0])
else:
issue_reporters.append(row[0])
return issue_reporters, comment_reporters
def LookupIssueFlagCounts(self, cnxn, issue_ids):
"""Returns a map of issue_id to flag counts"""
rows = self.report_tbl.Select(cnxn,
cols=['issue_id', 'COUNT(*)'],
issue_id=issue_ids,
group_by=['issue_id'])
counts = {}
for row in rows:
counts[long(row[0])] = row[1]
return counts
def LookupIssueVerdicts(self, cnxn, issue_ids):
"""Returns a map of issue_id to most recent spam verdicts"""
rows = self.verdict_tbl.Select(
cnxn,
cols=['issue_id', 'reason', 'MAX(created)'],
issue_id=issue_ids,
group_by=['issue_id'])
counts = {}
for row in rows:
counts[long(row[0])] = row[1]
return counts
def LookupIssueVerdictHistory(self, cnxn, issue_ids):
"""Returns a map of issue_id to most recent spam verdicts"""
rows = self.verdict_tbl.Select(cnxn,
cols=[
'issue_id', 'reason', 'created',
'is_spam', 'classifier_confidence',
'user_id', 'overruled'
],
issue_id=issue_ids,
order_by=[('issue_id', []),
('created', [])])
# TODO: group by issue_id, make class instead of dict for verdict.
verdicts = []
for row in rows:
verdicts.append({
'issue_id': row[0],
'reason': row[1],
'created': row[2],
'is_spam': row[3],
'classifier_confidence': row[4],
'user_id': row[5],
'overruled': row[6],
})
return verdicts
def LookupCommentVerdictHistory(self, cnxn, comment_ids):
"""Returns a map of issue_id to most recent spam verdicts"""
rows = self.verdict_tbl.Select(cnxn,
cols=[
'comment_id', 'reason', 'created',
'is_spam', 'classifier_confidence',
'user_id', 'overruled'
],
comment_id=comment_ids,
order_by=[('comment_id', []),
('created', [])])
# TODO: group by comment_id, make class instead of dict for verdict.
verdicts = []
for row in rows:
verdicts.append({
'comment_id': row[0],
'reason': row[1],
'created': row[2],
'is_spam': row[3],
'classifier_confidence': row[4],
'user_id': row[5],
'overruled': row[6],
})
return verdicts
def FlagIssues(self, cnxn, issue_service, issues, reporting_user_id,
flagged_spam):
"""Creates or deletes a spam report on an issue."""
verdict_updates = []
if flagged_spam:
rows = [(issue.issue_id, issue.reporter_id, reporting_user_id)
for issue in issues]
self.report_tbl.InsertRows(cnxn,
SPAMREPORT_ISSUE_COLS,
rows,
ignore=True)
else:
issue_ids = [issue.issue_id for issue in issues]
self.report_tbl.Delete(cnxn,
issue_id=issue_ids,
user_id=reporting_user_id,
comment_id=None)
project_id = issues[0].project_id
# Now record new verdicts and update issue.is_spam, if they've changed.
ids = [issue.issue_id for issue in issues]
counts = self.LookupIssueFlagCounts(cnxn, ids)
previous_verdicts = self.LookupIssueVerdicts(cnxn, ids)
for issue_id in counts:
# If the flag counts changed enough to toggle the is_spam bit, need to
# record a new verdict and update the Issue.
if ((flagged_spam and counts[issue_id] >= settings.spam_flag_thresh
or not flagged_spam
and counts[issue_id] < settings.spam_flag_thresh)
and (previous_verdicts[issue_id] != REASON_MANUAL
if issue_id in previous_verdicts else True)):
verdict_updates.append(issue_id)
if len(verdict_updates) == 0:
return
# Some of the issues may have exceed the flag threshold, so issue verdicts
# and mark as spam in those cases.
rows = [(issue_id, flagged_spam, REASON_THRESHOLD, project_id)
for issue_id in verdict_updates]
self.verdict_tbl.InsertRows(cnxn,
THRESHVERDICT_ISSUE_COLS,
rows,
ignore=True)
update_issues = []
for issue in issues:
if issue.issue_id in verdict_updates:
issue.is_spam = flagged_spam
update_issues.append(issue)
if flagged_spam:
self.issue_actions.increment_by(len(update_issues),
{'type': 'flag'})
issue_service.UpdateIssues(cnxn,
update_issues,
update_cols=['is_spam'])
def FlagComment(self, cnxn, issue_id, comment_id, reported_user_id,
reporting_user_id, flagged_spam):
"""Creates or deletes a spam report on a comment."""
# TODO(seanmccullough): Bulk comment flagging? There's no UI for that.
if flagged_spam:
self.report_tbl.InsertRow(cnxn,
ignore=True,
issue_id=issue_id,
comment_id=comment_id,
reported_user_id=reported_user_id,
user_id=reporting_user_id)
self.comment_actions.increment({'type': 'flag'})
else:
self.report_tbl.Delete(cnxn,
issue_id=issue_id,
comment_id=comment_id,
user_id=reporting_user_id)
def RecordClassifierIssueVerdict(self, cnxn, issue, is_spam, confidence,
fail_open):
reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER
self.verdict_tbl.InsertRow(cnxn,
issue_id=issue.issue_id,
is_spam=is_spam,
reason=reason,
classifier_confidence=confidence,
project_id=issue.project_id)
if is_spam:
self.issue_actions.increment({'type': 'classifier'})
# This is called at issue creation time, so there's nothing else to do here.
def RecordManualIssueVerdicts(self, cnxn, issue_service, issues, user_id,
is_spam):
rows = [(user_id, issue.issue_id, is_spam, REASON_MANUAL,
issue.project_id) for issue in issues]
issue_ids = [issue.issue_id for issue in issues]
# Overrule all previous verdicts.
self.verdict_tbl.Update(
cnxn, {'overruled': True},
[('issue_id IN (%s)' % sql.PlaceHolders(issue_ids), issue_ids)],
commit=False)
self.verdict_tbl.InsertRows(cnxn,
MANUALVERDICT_ISSUE_COLS,
rows,
ignore=True)
for issue in issues:
issue.is_spam = is_spam
if is_spam:
self.issue_actions.increment_by(len(issues), {'type': 'manual'})
else:
issue_service.AllocateNewLocalIDs(cnxn, issues)
# This will commit the transaction.
issue_service.UpdateIssues(cnxn, issues, update_cols=['is_spam'])
def RecordManualCommentVerdict(self, cnxn, issue_service, user_service,
comment_id, sequence_num, user_id, is_spam):
# TODO(seanmccullough): Bulk comment verdicts? There's no UI for that.
self.verdict_tbl.InsertRow(cnxn,
ignore=True,
user_id=user_id,
comment_id=comment_id,
is_spam=is_spam,
reason=REASON_MANUAL)
comment = issue_service.GetComment(cnxn, comment_id)
comment.is_spam = is_spam
issue = issue_service.GetIssue(cnxn, comment.issue_id)
issue_service.SoftDeleteComment(cnxn, comment.project_id,
issue.local_id, sequence_num, user_id,
user_service, is_spam, True, is_spam)
if is_spam:
self.comment_actions.increment({'type': 'manual'})
def RecordClassifierCommentVerdict(self, cnxn, comment, is_spam,
confidence, fail_open):
reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER
self.verdict_tbl.InsertRow(cnxn,
comment_id=comment.id,
is_spam=is_spam,
reason=reason,
classifier_confidence=confidence,
project_id=comment.project_id)
if is_spam:
self.comment_actions.increment({'type': 'classifier'})
def _predict(self, body):
return self.prediction_service.trainedmodels().predict(
project=settings.classifier_project_id,
id=settings.classifier_model_id,
body=body).execute()
def _IsExempt(self, author, is_project_member):
"""Return True if the user is exempt from spam checking."""
if author.email is not None and author.email.endswith(
settings.spam_whitelisted_suffixes):
logging.info('%s whitelisted from spam filtering', author.email)
return True
if author.ignore_action_limits:
logging.info('%s trusted not to spam', author.email)
return True
if is_project_member:
logging.info('%s is a project member, assuming ham', author.email)
return True
return False
def ClassifyIssue(self, issue, firstComment, reporter, is_project_member):
"""Classify an issue as either spam or ham.
Args:
issue: the Issue.
firstComment: the first Comment on issue.
reporter: User PB for the Issue reporter.
is_project_member: True if reporter is a member of issue's project.
Returns a JSON dict of classifier prediction results from
the Cloud Prediction API.
"""
# Fail-safe: not spam.
result = {
'outputLabel': 'ham',
'outputMulti': [{
'label': 'ham',
'score': '1.0'
}],
'failed_open': False
}
if self._IsExempt(reporter, is_project_member):
return result
if not self.prediction_service:
logging.error("prediction_service not initialized.")
return result
features = spam_helpers.GenerateFeatures(issue.summary,
firstComment.content,
settings.spam_feature_hashes)
remaining_retries = 3
while remaining_retries > 0:
try:
result = self._predict({'input': {
'csvInstance': features,
}})
result['failed_open'] = False
return result
except Exception as ex:
remaining_retries = remaining_retries - 1
self.prediction_api_failures.increment()
logging.error('Error calling prediction API: %s' % ex)
result['failed_open'] = True
return result
def ClassifyComment(self,
comment_content,
commenter,
is_project_member=True):
"""Classify a comment as either spam or ham.
Args:
comment: the comment text.
commenter: User PB for the user who authored the comment.
Returns a JSON dict of classifier prediction results from
the Cloud Prediction API.
"""
# Fail-safe: not spam.
result = {
'outputLabel': 'ham',
'outputMulti': [{
'label': 'ham',
'score': '1.0'
}],
'failed_open': False
}
if self._IsExempt(commenter, is_project_member):
return result
if not self.prediction_service:
logging.error("prediction_service not initialized.")
self.prediction_api_failures.increment()
result['failed_open'] = True
return result
features = spam_helpers.GenerateFeatures('', comment_content,
settings.spam_feature_hashes)
remaining_retries = 3
while remaining_retries > 0:
try:
result = self._predict({'input': {
'csvInstance': features,
}})
result['failed_open'] = False
return result
except Exception as ex:
remaining_retries = remaining_retries - 1
self.prediction_api_failures.increment()
logging.error('Error calling prediction API: %s' % ex)
result['failed_open'] = True
return result
def GetIssueClassifierQueue(self,
cnxn,
_issue_service,
project_id,
offset=0,
limit=10):
"""Returns list of recent issues with spam verdicts,
ranked in ascending order of confidence (so uncertain items are first).
"""
# TODO(seanmccullough): Optimize pagination. This query probably gets
# slower as the number of SpamVerdicts grows, regardless of offset
# and limit values used here. Using offset,limit in general may not
# be the best way to do this.
issue_results = self.verdict_tbl.Select(
cnxn,
cols=[
'issue_id', 'is_spam', 'reason', 'classifier_confidence',
'created'
],
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('issue_id IS NOT NULL', []),
],
order_by=[
('classifier_confidence ASC', []),
('created ASC', []),
],
group_by=['issue_id'],
offset=offset,
limit=limit,
)
ret = []
for row in issue_results:
ret.append(
ModerationItem(
issue_id=long(row[0]),
is_spam=row[1] == 1,
reason=row[2],
classifier_confidence=row[3],
verdict_time='%s' % row[4],
))
count = self.verdict_tbl.SelectValue(
cnxn,
col='COUNT(*)',
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('issue_id IS NOT NULL', []),
])
return ret, count
def GetIssueFlagQueue(self,
cnxn,
_issue_service,
project_id,
offset=0,
limit=10):
"""Returns list of recent issues that have been flagged by users"""
issue_flags = self.report_tbl.Select(
cnxn,
cols=[
"Issue.project_id", "Report.issue_id", "count(*) as count",
"max(Report.created) as latest",
"count(distinct Report.user_id) as users"
],
left_joins=["Issue ON Issue.id = Report.issue_id"],
where=[('Report.issue_id IS NOT NULL', []),
("Issue.project_id == %v", [project_id])],
order_by=[('count DESC', [])],
group_by=['Report.issue_id'],
offset=offset,
limit=limit)
ret = []
for row in issue_flags:
ret.append(
ModerationItem(
project_id=row[0],
issue_id=row[1],
count=row[2],
latest_report=row[3],
num_users=row[4],
))
count = self.verdict_tbl.SelectValue(
cnxn,
col='COUNT(DISTINCT Report.issue_id)',
where=[('Issue.project_id = %s', [project_id])],
left_joins=["Issue ON Issue.id = SpamReport.issue_id"])
return ret, count
def GetCommentClassifierQueue(self,
cnxn,
_issue_service,
project_id,
offset=0,
limit=10):
"""Returns list of recent comments with spam verdicts,
ranked in ascending order of confidence (so uncertain items are first).
"""
# TODO(seanmccullough): Optimize pagination. This query probably gets
# slower as the number of SpamVerdicts grows, regardless of offset
# and limit values used here. Using offset,limit in general may not
# be the best way to do this.
comment_results = self.verdict_tbl.Select(
cnxn,
cols=[
'issue_id', 'is_spam', 'reason', 'classifier_confidence',
'created'
],
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('comment_id IS NOT NULL', []),
],
order_by=[
('classifier_confidence ASC', []),
('created ASC', []),
],
group_by=['comment_id'],
offset=offset,
limit=limit,
)
ret = []
for row in comment_results:
ret.append(
ModerationItem(
comment_id=long(row[0]),
is_spam=row[1] == 1,
reason=row[2],
classifier_confidence=row[3],
verdict_time='%s' % row[4],
))
count = self.verdict_tbl.SelectValue(
cnxn,
col='COUNT(*)',
where=[
('project_id = %s', [project_id]),
('classifier_confidence <= %s',
[settings.classifier_moderation_thresh]),
('overruled = %s', [False]),
('comment_id IS NOT NULL', []),
])
return ret, count
def GetTrainingIssues(self,
cnxn,
issue_service,
since,
offset=0,
limit=100):
"""Returns list of recent issues with human-labeled spam/ham verdicts.
"""
# get all of the manual verdicts in the past day.
results = self.verdict_tbl.Select(
cnxn,
cols=['issue_id'],
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('issue_id IS NOT NULL', []),
('created > %s', [since.isoformat()]),
],
offset=offset,
limit=limit,
)
issue_ids = [long(row[0]) for row in results if row[0]]
issues = issue_service.GetIssues(cnxn, issue_ids)
comments = issue_service.GetCommentsForIssues(cnxn, issue_ids)
first_comments = {}
for issue in issues:
first_comments[issue.issue_id] = (
comments[issue.issue_id][0].content
if issue.issue_id in comments else "[Empty]")
count = self.verdict_tbl.SelectValue(cnxn,
col='COUNT(*)',
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('issue_id IS NOT NULL', []),
('created > %s',
[since.isoformat()]),
])
return issues, first_comments, count
def GetTrainingComments(self,
cnxn,
issue_service,
since,
offset=0,
limit=100):
"""Returns list of recent comments with human-labeled spam/ham verdicts.
"""
# get all of the manual verdicts in the past day.
results = self.verdict_tbl.Select(
cnxn,
cols=['comment_id'],
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('comment_id IS NOT NULL', []),
('created > %s', [since.isoformat()]),
],
offset=offset,
limit=limit,
)
comment_ids = [long(row[0]) for row in results if row[0]]
# Don't care about sequence numbers in this context yet.
comments = issue_service.GetCommentsByID(cnxn, comment_ids,
defaultdict(int))
count = self.verdict_tbl.SelectValue(cnxn,
col='COUNT(*)',
where=[
('overruled = %s', [False]),
('reason = %s', ['manual']),
('comment_id IS NOT NULL',
[]),
('created > %s',
[since.isoformat()]),
])
return comments, count
import re
import time
from logging.handlers import TimedRotatingFileHandler
import buildbot.status.results
from buildbot.status.base import StatusReceiverMultiService
from twisted.python import log as twisted_log
from common import chromium_utils
from infra_libs import ts_mon
step_field_spec = [
ts_mon.StringField('builder'),
ts_mon.StringField('master'),
ts_mon.StringField('project_id'),
ts_mon.StringField('result'),
ts_mon.StringField('slave'),
ts_mon.StringField('step_name'),
ts_mon.StringField('subproject_tag'),
]
step_durations = ts_mon.CumulativeDistributionMetric(
'buildbot/master/builders/steps/durations',
'Time (in seconds) from step start to step end',
step_field_spec,
units=ts_mon.MetricsDataUnits.SECONDS,
# Use fixed-width bucketer up to 2.7 hours with 10-second precision.
bucketer=ts_mon.FixedWidthBucketer(10, 1000))
