def _ignored_metric(self): return metrics.Boolean( self._metric_root_path + 'ignored', description=( 'A boolean, for servers ignored for test infra prod alerts.'), field_spec=[ts_mon.StringField('target_data_center'), ts_mon.StringField('target_hostname'),])
def _presence_metric(self): return metrics.Boolean( self._metric_root_path + 'presence', description=( 'A boolean indicating whether a server is in the machines db.'), field_spec=[ts_mon.StringField('target_data_center'), ts_mon.StringField('target_hostname'),])
def _roles_metric(self): return metrics.String( self._metric_root_path + 'roles', description=( 'A string indicating the role of a server in the machines db.'), field_spec=[ts_mon.StringField('target_data_center'), ts_mon.StringField('target_hostname'),])
def calculate_spares_needed(self, target_total): """Calculate and log the spares needed to achieve a target. Return how many working spares are needed to achieve the given `target_total` with all DUTs working. The spares count may be positive or negative. Positive values indicate spares are needed to replace broken DUTs in order to reach the target; negative numbers indicate that no spares are needed, and that a corresponding number of working devices can be returned. If the new target total would require returning ineligible DUTs, an error is logged, and the target total is adjusted so that those DUTs are not exchanged. @param target_total The new target pool size. @return The number of spares needed. """ num_ineligible = len(self.ineligible_hosts) spares_needed = target_total >= num_ineligible metrics.Boolean( 'chromeos/autotest/balance_pools/exhausted_pools', 'True for each pool/model which requests more DUTs than supplied', # TODO(jrbarnette) The 'board' field is a legacy. We need # to leave it here until we do the extra work Monarch # requires to delete a field. field_spec=[ ts_mon.StringField('pool'), ts_mon.StringField('board'), ts_mon.StringField('model'), ]).set( not spares_needed, fields={ 'pool': self.pool, 'board': self.labels.get('model', ''), 'model': self.labels.get('model', ''), }, ) if not spares_needed: _log_error( '%s pool (%s): Target of %d is below minimum of %d DUTs.', self.pool, self.labels, target_total, num_ineligible, ) _log_error('Adjusting target to %d DUTs.', num_ineligible) target_total = num_ineligible else: _log_message('%s %s pool: Target of %d is above minimum.', self.labels.get('model', ''), self.pool, target_total) adjustment = target_total - self.total_hosts return len(self.broken_hosts) + adjustment
def _exchange_labels(dry_run, hosts, target_pool, spare_pool): """Reassign a list of DUTs from one pool to another. For all the given hosts, remove all labels associated with `spare_pool`, and add the labels for `target_pool`. If `dry_run` is true, perform no changes, but log the `atest` commands needed to accomplish the necessary label changes. @param dry_run Whether the logging is for a dry run or for actual execution. @param hosts List of DUTs (AFE hosts) to be reassigned. @param target_pool The `_DUTPool` object from which the hosts are drawn. @param spare_pool The `_DUTPool` object to which the hosts will be added. """ _log_info(dry_run, 'Transferring %d DUTs from %s to %s.', len(hosts), spare_pool.pool, target_pool.pool) metrics.Counter( 'chromeos/autotest/balance_pools/duts_moved', 'DUTs transferred between pools', # TODO(jrbarnette) The 'board' field is a legacy. We need to # leave it here until we do the extra work Monarch requires to # delete a field. field_spec=[ ts_mon.StringField('board'), ts_mon.StringField('model'), ts_mon.StringField('source_pool'), ts_mon.StringField('target_pool'), ] ).increment_by( len(hosts), fields={ 'board': target_pool.labels.get('model', ''), 'model': target_pool.labels.get('model', ''), 'source_pool': spare_pool.pool, 'target_pool': target_pool.pool, }, ) if not hosts: return additions = target_pool.pool_labels removals = spare_pool.pool_labels for host in hosts: if not dry_run: _log_message('Updating host: %s.', host.hostname) host.remove_labels(removals) host.add_labels(additions) else: _log_message('atest label remove -m %s %s', host.hostname, ' '.join(removals)) _log_message('atest label add -m %s %s', host.hostname, ' '.join(additions))
class Poller(threading.Thread): commits_metric = ts_mon.CounterMetric( 'bugdroid/commits', 'Counter of commits processed by bugdroid', [ ts_mon.StringField('poller'), ts_mon.StringField('project'), ts_mon.StringField('status') ]) def __init__(self, interval_in_minutes=15, setup_refresh_interval_minutes=0, run_once=False): threading.Thread.__init__(self, name=str(hash(self))) self.interval = interval_in_minutes * 60 self.refresh_interval = setup_refresh_interval_minutes self.run_once = run_once if setup_refresh_interval_minutes: self.setup_refresh = ( datetime.datetime.now() + datetime.timedelta(minutes=setup_refresh_interval_minutes)) else: self.setup_refresh = None def execute(self): raise NotImplementedError() def setup(self): # pylint: disable=R0201 return True def run(self): try: while True: if self.setup_refresh and self.setup_refresh < datetime.datetime.now( ): LOGGER.info('Re-running Poller setup') self.setup() self.setup_refresh = ( datetime.datetime.now() + datetime.timedelta(minutes=self.refresh_interval)) self.execute() if self.run_once: return time.sleep(self.interval) except Exception: LOGGER.exception('Unhandled Poller exception.') def start(self): if self.setup(): super(Poller, self).start()
def _RecordDurationMetric(batches): """Records a span duration metric for each span. Args: batches: A sequence of span batches (lists) Yields: Re-yields the same batches """ m = metrics.CumulativeSecondsDistribution( _SPAN_DURATION_METRIC, description="The durations of Spans consumed by export_to_cloud_trace", field_spec=[ts_mon.StringField('name')]) for batch in batches: batch = tuple(batch) # Needed because we will consume the iterator. for span in batch: try: time_delta = (_ParseDatetime(span['endTime']) - _ParseDatetime(span['startTime'])) m.add(time_delta.total_seconds(), fields={'name': span['name']}) except KeyError: log.error( "Span %s did not have required fields 'endTime', " "'startTime', and 'name'.", json.dumps(span)) yield batch
def main(argv): """Entry point for dut_mon.""" logging.getLogger().setLevel(logging.INFO) with ts_mon_config.SetupTsMonGlobalState('dut_mon', indirect=True): afe = frontend.AFE() counters = collections.defaultdict(lambda: 0) field_spec = [ts_mon.StringField('board'), ts_mon.StringField('model'), ts_mon.StringField('pool'), ts_mon.BooleanField('is_locked'), ts_mon.StringField('status'), ] dut_count = metrics.Gauge('chromeos/autotest/dut_mon/dut_count', description='The number of duts in a given ' 'state and bucket.', field_spec=field_spec) tick_count = metrics.Counter('chromeos/autotest/dut_mon/tick', description='Tick counter of dut_mon.') while True: # Note: We reset all counters to zero in each loop rather than # creating a new defaultdict, because we want to ensure that any # gauges that were previously set to a nonzero value by this process # get set back to zero if necessary. for k in counters: counters[k] = 0 logging.info('Fetching all hosts.') hosts = afe.get_hosts() logging.info('Fetched %s hosts.', len(hosts)) for host in hosts: fields = _get_bucket_for_host(host) counters[fields] += 1 for field, value in counters.iteritems(): logging.info('%s %s', field, value) dut_count.set(value, fields=field.__dict__) tick_count.increment() logging.info('Sleeping for 2 minutes.') time.sleep(120)
def _RecordSpanMetrics(span): """Increments the count of spans logged. Args: span: The span to record. """ m = metrics.Counter( _SPAN_COUNT_METRIC, description="A count of spans logged by a client.", field_spec=[ts_mon.StringField('name')]) m.increment(fields={'name': span.name})
def _EmitImportanceMetric(master_config, important_configs, experimental_configs): """Emit monarch metrics about which slave configs were important.""" importance = {build_config: True for build_config in important_configs} for build_config in experimental_configs: importance[build_config] = False m = metrics.BooleanMetric( 'chromeos/cbuildbot/master/has_important_slave', description='Slaves that were considered ' 'important by master.', field_spec=[ ts_mon.StringField('master_config'), ts_mon.StringField('slave_config') ]) for slave_config, is_important in importance.items(): m.set(is_important, fields={ 'master_config': master_config, 'slave_config': slave_config })
def testGetMetricFieldSpec(self): """Test each field type gets its FieldSpec.""" fields = { 'int': 12, 'bool': True, 'str': 'string', } expected_fieldspec = [ts_mon.IntegerField('int'), ts_mon.BooleanField('bool'), ts_mon.StringField('str')] self.assertEqual(ts_mon_config.GetMetricFieldSpec(fields), expected_fieldspec)
class ApiRateLimiter(RateLimiter): blocked_requests = ts_mon.CounterMetric( 'monorail/apiratelimiter/blocked_request', 'Count of requests that exceeded the rate limit and were blocked.', None) limit_exceeded = ts_mon.CounterMetric( 'monorail/apiratelimiter/rate_exceeded', 'Count of requests that exceeded the rate limit.', None) cost_thresh_exceeded = ts_mon.CounterMetric( 'monorail/apiratelimiter/cost_thresh_exceeded', 'Count of requests that were expensive to process', None) checks = ts_mon.CounterMetric( 'monorail/apiratelimiter/check', 'Count of checks done, by fail/success type.', [ts_mon.StringField('type')]) #pylint: disable=arguments-differ def CheckStart(self, client_id, client_email, now=None): if now is None: now = time.time() keysets = _CreateApiCacheKeys(client_id, client_email, now) qpm_limit = client_config_svc.GetQPMDict().get(client_email, DEFAULT_API_QPM) window_limit = qpm_limit * N_MINUTES self._AuxCheckStart(keysets, window_limit, settings.api_ratelimiting_enabled, ApiRateLimitExceeded(client_id, client_email)) #pylint: disable=arguments-differ def CheckEnd(self, client_id, client_email, now, start_time): if not settings.ratelimiting_cost_enabled: return elapsed_ms = (now - start_time) * 1000 if elapsed_ms < settings.api_ratelimiting_cost_thresh_ms: return keysets = _CreateApiCacheKeys(client_id, client_email, start_time) self._AuxCheckEnd( keysets, 'API Rate Limit Cost Threshold Exceeded: %s, %s' % (client_id, client_email), settings.api_ratelimiting_cost_penalty)
def testEnqueue(self): """Test that _Indirect enqueues messages correctly.""" metric = metrics.Boolean with parallel.Manager() as manager: q = manager.Queue() self.PatchObject(metrics, 'MESSAGE_QUEUE', q) proxy_metric = metric('foo') proxy_metric.example('arg1', {'field_name': 'value'}) message = q.get(timeout=10) expected_metric_kwargs = { 'field_spec': [ts_mon.StringField('field_name')], 'description': 'No description.', } self.assertEqual( message, metrics.MetricCall(metric.__name__, ('foo',), expected_metric_kwargs, 'example', ('arg1', {'field_name': 'value'}), {}, False))
class SpamService(object): """The persistence layer for spam reports.""" issue_actions = ts_mon.CounterMetric( 'monorail/spam_svc/issue', 'Count of things that happen to issues.', [ts_mon.StringField('type')]) comment_actions = ts_mon.CounterMetric( 'monorail/spam_svc/comment', 'Count of things that happen to comments.', [ts_mon.StringField('type')]) ml_engine_failures = ts_mon.CounterMetric( 'monorail/spam_svc/ml_engine_failure', 'Failures calling the ML Engine API', None) def __init__(self): self.report_tbl = sql.SQLTableManager(SPAMREPORT_TABLE_NAME) self.verdict_tbl = sql.SQLTableManager(SPAMVERDICT_TABLE_NAME) self.issue_tbl = sql.SQLTableManager(ISSUE_TABLE) # ML Engine library is lazy loaded below. self.ml_engine = None def LookupIssuesFlaggers(self, cnxn, issue_ids): """Returns users who've reported the issues or their comments as spam. Returns a dictionary {issue_id: (issue_reporters, comment_reporters)} issue_reportes is a list of users who flagged the issue; comment_reporters element is a dictionary {comment_id: [user_ids]} where user_ids are the users who flagged that comment. """ rows = self.report_tbl.Select( cnxn, cols=['issue_id', 'user_id', 'comment_id'], issue_id=issue_ids) reporters = collections.defaultdict( # Return a tuple of (issue_reporters, comment_reporters) as described # above. lambda: ([], collections.defaultdict(list))) for row in rows: issue_id = int(row[0]) user_id = row[1] if row[2]: comment_id = row[2] reporters[issue_id][1][comment_id].append(user_id) else: reporters[issue_id][0].append(user_id) return reporters def LookupIssueFlaggers(self, cnxn, issue_id): """Returns users who've reported the issue or its comments as spam. Returns a tuple. First element is a list of users who flagged the issue; second element is a dictionary of comment id to a list of users who flagged that comment. """ return self.LookupIssuesFlaggers(cnxn, [issue_id])[issue_id] def LookupIssueFlagCounts(self, cnxn, issue_ids): """Returns a map of issue_id to flag counts""" rows = self.report_tbl.Select(cnxn, cols=['issue_id', 'COUNT(*)'], issue_id=issue_ids, group_by=['issue_id']) counts = {} for row in rows: counts[int(row[0])] = row[1] return counts def LookupIssueVerdicts(self, cnxn, issue_ids): """Returns a map of issue_id to most recent spam verdicts""" rows = self.verdict_tbl.Select(cnxn, cols=['issue_id', 'reason', 'MAX(created)'], issue_id=issue_ids, comment_id=None, group_by=['issue_id']) counts = {} for row in rows: counts[int(row[0])] = row[1] return counts def LookupIssueVerdictHistory(self, cnxn, issue_ids): """Returns a map of issue_id to most recent spam verdicts""" rows = self.verdict_tbl.Select(cnxn, cols=[ 'issue_id', 'reason', 'created', 'is_spam', 'classifier_confidence', 'user_id', 'overruled'], issue_id=issue_ids, order_by=[('issue_id', []), ('created', [])]) # TODO: group by issue_id, make class instead of dict for verdict. verdicts = [] for row in rows: verdicts.append({ 'issue_id': row[0], 'reason': row[1], 'created': row[2], 'is_spam': row[3], 'classifier_confidence': row[4], 'user_id': row[5], 'overruled': row[6], }) return verdicts def LookupCommentVerdictHistory(self, cnxn, comment_ids): """Returns a map of issue_id to most recent spam verdicts""" rows = self.verdict_tbl.Select(cnxn, cols=[ 'comment_id', 'reason', 'created', 'is_spam', 'classifier_confidence', 'user_id', 'overruled'], comment_id=comment_ids, order_by=[('comment_id', []), ('created', [])]) # TODO: group by comment_id, make class instead of dict for verdict. verdicts = [] for row in rows: verdicts.append({ 'comment_id': row[0], 'reason': row[1], 'created': row[2], 'is_spam': row[3], 'classifier_confidence': row[4], 'user_id': row[5], 'overruled': row[6], }) return verdicts def FlagIssues(self, cnxn, issue_service, issues, reporting_user_id, flagged_spam): """Creates or deletes a spam report on an issue.""" verdict_updates = [] if flagged_spam: rows = [(issue.issue_id, issue.reporter_id, reporting_user_id) for issue in issues] self.report_tbl.InsertRows(cnxn, SPAMREPORT_ISSUE_COLS, rows, ignore=True) else: issue_ids = [issue.issue_id for issue in issues] self.report_tbl.Delete( cnxn, issue_id=issue_ids, user_id=reporting_user_id, comment_id=None) project_id = issues[0].project_id # Now record new verdicts and update issue.is_spam, if they've changed. ids = [issue.issue_id for issue in issues] counts = self.LookupIssueFlagCounts(cnxn, ids) previous_verdicts = self.LookupIssueVerdicts(cnxn, ids) for issue_id in counts: # If the flag counts changed enough to toggle the is_spam bit, need to # record a new verdict and update the Issue. # No number of user spam flags can overturn an admin's verdict. if previous_verdicts.get(issue_id) == REASON_MANUAL: continue # If enough spam flags come in, mark the issue as spam. if (flagged_spam and counts[issue_id] >= settings.spam_flag_thresh): verdict_updates.append(issue_id) if len(verdict_updates) == 0: return # Some of the issues may have exceed the flag threshold, so issue verdicts # and mark as spam in those cases. rows = [(issue_id, flagged_spam, REASON_THRESHOLD, project_id) for issue_id in verdict_updates] self.verdict_tbl.InsertRows(cnxn, THRESHVERDICT_ISSUE_COLS, rows, ignore=True) update_issues = [] for issue in issues: if issue.issue_id in verdict_updates: issue.is_spam = flagged_spam update_issues.append(issue) if flagged_spam: self.issue_actions.increment_by(len(update_issues), {'type': 'flag'}) issue_service.UpdateIssues(cnxn, update_issues, update_cols=['is_spam']) def FlagComment(self, cnxn, issue_id, comment_id, reported_user_id, reporting_user_id, flagged_spam): """Creates or deletes a spam report on a comment.""" # TODO(seanmccullough): Bulk comment flagging? There's no UI for that. if flagged_spam: self.report_tbl.InsertRow( cnxn, ignore=True, issue_id=issue_id, comment_id=comment_id, reported_user_id=reported_user_id, user_id=reporting_user_id) self.comment_actions.increment({'type': 'flag'}) else: self.report_tbl.Delete( cnxn, issue_id=issue_id, comment_id=comment_id, user_id=reporting_user_id) def RecordClassifierIssueVerdict(self, cnxn, issue, is_spam, confidence, fail_open): reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER self.verdict_tbl.InsertRow(cnxn, issue_id=issue.issue_id, is_spam=is_spam, reason=reason, classifier_confidence=confidence, project_id=issue.project_id) if is_spam: self.issue_actions.increment({'type': 'classifier'}) # This is called at issue creation time, so there's nothing else to do here. def RecordManualIssueVerdicts(self, cnxn, issue_service, issues, user_id, is_spam): rows = [(user_id, issue.issue_id, is_spam, REASON_MANUAL, issue.project_id) for issue in issues] issue_ids = [issue.issue_id for issue in issues] # Overrule all previous verdicts. self.verdict_tbl.Update(cnxn, {'overruled': True}, [ ('issue_id IN (%s)' % sql.PlaceHolders(issue_ids), issue_ids) ], commit=False) self.verdict_tbl.InsertRows(cnxn, MANUALVERDICT_ISSUE_COLS, rows, ignore=True) for issue in issues: issue.is_spam = is_spam if is_spam: self.issue_actions.increment_by(len(issues), {'type': 'manual'}) else: issue_service.AllocateNewLocalIDs(cnxn, issues) # This will commit the transaction. issue_service.UpdateIssues(cnxn, issues, update_cols=['is_spam']) def RecordManualCommentVerdict(self, cnxn, issue_service, user_service, comment_id, user_id, is_spam): # TODO(seanmccullough): Bulk comment verdicts? There's no UI for that. self.verdict_tbl.InsertRow(cnxn, ignore=True, user_id=user_id, comment_id=comment_id, is_spam=is_spam, reason=REASON_MANUAL) comment = issue_service.GetComment(cnxn, comment_id) comment.is_spam = is_spam issue = issue_service.GetIssue(cnxn, comment.issue_id, use_cache=False) issue_service.SoftDeleteComment( cnxn, issue, comment, user_id, user_service, is_spam, True, is_spam) if is_spam: self.comment_actions.increment({'type': 'manual'}) def RecordClassifierCommentVerdict(self, cnxn, comment, is_spam, confidence, fail_open): reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER self.verdict_tbl.InsertRow(cnxn, comment_id=comment.id, is_spam=is_spam, reason=reason, classifier_confidence=confidence, project_id=comment.project_id) if is_spam: self.comment_actions.increment({'type': 'classifier'}) def _predict(self, instance): """Requests a prediction from the ML Engine API. Sample API response: {'predictions': [{ 'classes': ['0', '1'], 'scores': [0.4986788034439087, 0.5013211965560913] }]} This hits the default model. Returns: A floating point number representing the confidence the instance is spam. """ model_name = 'projects/%s/models/%s' % ( settings.classifier_project_id, settings.spam_model_name) body = {'instances': [{"inputs": instance["word_hashes"]}]} if not self.ml_engine: self.ml_engine = ml_helpers.setup_ml_engine() request = self.ml_engine.projects().predict(name=model_name, body=body) response = request.execute() logging.info('ML Engine API response: %r' % response) prediction = response['predictions'][0] # Ensure the class confidence we return is for the spam, not the ham label. # The spam label, '1', is usually at index 1 but I'm not sure of any # guarantees around label order. if prediction['classes'][1] == SPAM_CLASS_LABEL: return prediction['scores'][1] elif prediction['classes'][0] == SPAM_CLASS_LABEL: return prediction['scores'][0] else: raise Exception('No predicted classes found.') def _IsExempt(self, author, is_project_member): """Return True if the user is exempt from spam checking.""" if author.email is not None and author.email.endswith( settings.spam_whitelisted_suffixes): logging.info('%s whitelisted from spam filtering', author.email) return True if is_project_member: logging.info('%s is a project member, assuming ham', author.email) return True return False def ClassifyIssue(self, issue, firstComment, reporter, is_project_member): """Classify an issue as either spam or ham. Args: issue: the Issue. firstComment: the first Comment on issue. reporter: User PB for the Issue reporter. is_project_member: True if reporter is a member of issue's project. Returns a JSON dict of classifier prediction results from the ML Engine API. """ instance = ml_helpers.GenerateFeaturesRaw( [issue.summary, firstComment.content], settings.spam_feature_hashes) return self._classify(instance, reporter, is_project_member) def ClassifyComment(self, comment_content, commenter, is_project_member=True): """Classify a comment as either spam or ham. Args: comment: the comment text. commenter: User PB for the user who authored the comment. Returns a JSON dict of classifier prediction results from the ML Engine API. """ instance = ml_helpers.GenerateFeaturesRaw( ['', comment_content], settings.spam_feature_hashes) return self._classify(instance, commenter, is_project_member) def _classify(self, instance, author, is_project_member): # Fail-safe: not spam. result = self.ham_classification() if self._IsExempt(author, is_project_member): return result if not self.ml_engine: self.ml_engine = ml_helpers.setup_ml_engine() # If setup_ml_engine returns None, it failed to init. if not self.ml_engine: logging.error("ML Engine not initialized.") self.ml_engine_failures.increment() result['failed_open'] = True return result remaining_retries = 3 while remaining_retries > 0: try: result['confidence_is_spam'] = self._predict(instance) result['failed_open'] = False return result except Exception as ex: remaining_retries = remaining_retries - 1 self.ml_engine_failures.increment() logging.error('Error calling ML Engine API: %s' % ex) result['failed_open'] = True return result def ham_classification(self): return {'confidence_is_spam': 0.0, 'failed_open': False} def GetIssueClassifierQueue( self, cnxn, _issue_service, project_id, offset=0, limit=10): """Returns list of recent issues with spam verdicts, ranked in ascending order of confidence (so uncertain items are first). """ # TODO(seanmccullough): Optimize pagination. This query probably gets # slower as the number of SpamVerdicts grows, regardless of offset # and limit values used here. Using offset,limit in general may not # be the best way to do this. issue_results = self.verdict_tbl.Select(cnxn, cols=['issue_id', 'is_spam', 'reason', 'classifier_confidence', 'created'], where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('issue_id IS NOT NULL', []), ], order_by=[ ('classifier_confidence ASC', []), ('created ASC', []), ], group_by=['issue_id'], offset=offset, limit=limit, ) ret = [] for row in issue_results: ret.append(ModerationItem( issue_id=int(row[0]), is_spam=row[1] == 1, reason=row[2], classifier_confidence=row[3], verdict_time='%s' % row[4], )) count = self.verdict_tbl.SelectValue(cnxn, col='COUNT(*)', where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('issue_id IS NOT NULL', []), ]) return ret, count def GetIssueFlagQueue( self, cnxn, _issue_service, project_id, offset=0, limit=10): """Returns list of recent issues that have been flagged by users""" issue_flags = self.report_tbl.Select(cnxn, cols = ["Issue.project_id", "Report.issue_id", "count(*) as count", "max(Report.created) as latest", "count(distinct Report.user_id) as users"], left_joins=["Issue ON Issue.id = Report.issue_id"], where=[('Report.issue_id IS NOT NULL', []), ("Issue.project_id == %v", [project_id])], order_by=[('count DESC', [])], group_by=['Report.issue_id'], offset=offset, limit=limit) ret = [] for row in issue_flags: ret.append(ModerationItem( project_id=row[0], issue_id=row[1], count=row[2], latest_report=row[3], num_users=row[4], )) count = self.verdict_tbl.SelectValue(cnxn, col='COUNT(DISTINCT Report.issue_id)', where=[('Issue.project_id = %s', [project_id])], left_joins=["Issue ON Issue.id = SpamReport.issue_id"]) return ret, count def GetCommentClassifierQueue( self, cnxn, _issue_service, project_id, offset=0, limit=10): """Returns list of recent comments with spam verdicts, ranked in ascending order of confidence (so uncertain items are first). """ # TODO(seanmccullough): Optimize pagination. This query probably gets # slower as the number of SpamVerdicts grows, regardless of offset # and limit values used here. Using offset,limit in general may not # be the best way to do this. comment_results = self.verdict_tbl.Select(cnxn, cols=['issue_id', 'is_spam', 'reason', 'classifier_confidence', 'created'], where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('comment_id IS NOT NULL', []), ], order_by=[ ('classifier_confidence ASC', []), ('created ASC', []), ], group_by=['comment_id'], offset=offset, limit=limit, ) ret = [] for row in comment_results: ret.append(ModerationItem( comment_id=int(row[0]), is_spam=row[1] == 1, reason=row[2], classifier_confidence=row[3], verdict_time='%s' % row[4], )) count = self.verdict_tbl.SelectValue(cnxn, col='COUNT(*)', where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('comment_id IS NOT NULL', []), ]) return ret, count def GetTrainingIssues(self, cnxn, issue_service, since, offset=0, limit=100): """Returns list of recent issues with human-labeled spam/ham verdicts. """ # get all of the manual verdicts in the past day. results = self.verdict_tbl.Select(cnxn, cols=['issue_id'], where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('issue_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ], offset=offset, limit=limit, ) issue_ids = [int(row[0]) for row in results if row[0]] issues = issue_service.GetIssues(cnxn, issue_ids) comments = issue_service.GetCommentsForIssues(cnxn, issue_ids) first_comments = {} for issue in issues: first_comments[issue.issue_id] = (comments[issue.issue_id][0].content if issue.issue_id in comments else "[Empty]") count = self.verdict_tbl.SelectValue(cnxn, col='COUNT(*)', where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('issue_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ]) return issues, first_comments, count def GetTrainingComments(self, cnxn, issue_service, since, offset=0, limit=100): """Returns list of recent comments with human-labeled spam/ham verdicts. """ # get all of the manual verdicts in the past day. results = self.verdict_tbl.Select( cnxn, distinct=True, cols=['comment_id'], where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('comment_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ], offset=offset, limit=limit, ) comment_ids = [int(row[0]) for row in results if row[0]] # Don't care about sequence numbers in this context yet. comments = issue_service.GetCommentsByID(cnxn, comment_ids, defaultdict(int)) return comments def ExpungeUsersInSpam(self, cnxn, user_ids): """Removes all references to given users from Spam DB tables. This method will not commit the operations. This method will not make changes to in-memory data. """ commit = False self.report_tbl.Delete(cnxn, reported_user_id=user_ids, commit=commit) delta = {'user_id': framework_constants.DELETED_USER_ID} self.report_tbl.Update(cnxn, delta, user_id=user_ids, commit=commit) self.verdict_tbl.Update(cnxn, delta, user_id=user_ids, commit=commit)
import datetime import logging import random import re import threading import time from infra_libs import ts_mon from contextlib import contextmanager from google.appengine.api import app_identity PHASE_TIME = ts_mon.CumulativeDistributionMetric( 'monorail/servlet/phase_time', 'Time spent in profiler phases, in ms', [ts_mon.StringField('phase')]) # trace_service does not like very long names. MAX_PHASE_NAME_LENGTH = 200 class Profiler(object): """Object to record and help display request processing profiling info. The Profiler class holds a list of phase objects, which can hold additional phase objects (which are subphases). Each phase or subphase represents some meaningful part of this application's HTTP request processing. """ _COLORS = [ '900', '090', '009', '360', '306', '036', '630', '630', '063', '333'
class BugdroidGitPollerHandler(poller_handlers.BasePollerHandler): """Handler for updating bugs with information from commits.""" bug_comments_metric = ts_mon.CounterMetric( 'bugdroid/bug_comments', 'Counter of comments added to bugs', [ts_mon.StringField('project'), ts_mon.StringField('status')]) def __init__(self, monorail, logger, default_project, no_merge=None, public_bugs=True, test_mode=False, issues_labels=None, *args, **kwargs): self.monorail_client = monorail self.logger = logger self.default_project = default_project self.no_merge = no_merge or [] self.public_bugs = public_bugs self.test_mode = test_mode if issues_labels: self.issues_labels = dict((p.key, p.value) for p in issues_labels) else: self.issues_labels = {} super(BugdroidGitPollerHandler, self).__init__(*args, **kwargs) def _ApplyMergeMergedLabel(self, issue, branch): if not branch or not issue: return label = '%s-%s' % (self.issues_labels.get('merge', 'merge-merged'), branch) issue.add_label(label) self.logger.debug('Adding %s', label) label = self.issues_labels.get('approved', 'merge-approved') if issue.has_label(label): issue.remove_label(label) self.logger.debug('Removing %s', label) mstone = branch_utils.get_mstone(branch, False) if mstone: label = 'merge-approved-%s' % mstone if issue.has_label(label): issue.remove_label(label) self.logger.debug('Removing %s', label) def ProcessLogEntry(self, log_entry): project_bugs = log_parser.get_issues( log_entry, default_project=self.default_project) self.logger.info('Processing commit %s : bugs %s', log_entry.revision, str(project_bugs)) if project_bugs: comment = self._CreateMessage(log_entry) self.logger.debug(comment) for project, bugs in project_bugs.iteritems(): for bug in bugs: try: issue = self.monorail_client.get_issue(project, bug) issue.set_comment(comment[:24 * 1024]) branch = scm_helper.GetBranch(log_entry) # Apply merge labels if this commit landed on a branch. if branch and not (log_entry.scm in ['git', 'gerrit'] and scm_helper.GetBranch(log_entry, full=True) in self.no_merge): self._ApplyMergeMergedLabel(issue, branch) self.logger.debug('Attempting to save issue: %d', issue.id) if not self.test_mode: self.monorail_client.update_issue( project, issue, log_parser.should_send_email(log_entry.msg)) else: self.logger.debug('Test mode, skipping') except Exception: self.bug_comments_metric.increment({ 'project': project, 'status': 'failure' }) raise else: self.bug_comments_metric.increment({ 'project': project, 'status': 'success' }) def _CreateMessage(self, log_entry): msg = '' msg += 'The following revision refers to this bug:\n' msg += ' %s\n\n' % log_entry.GetCommitUrl() msg += self._BuildLogSpecial(log_entry) return msg def _BuildLogSpecial(self, log_entry): """Generate git-log style message, with links to files in the Web UI.""" rtn = 'commit %s\n' % log_entry.commit rtn += 'Author: %s <%s>\n' % (log_entry.author_name, log_entry.author_email) rtn += 'Date: %s\n' % log_entry.committer_date if self.public_bugs: rtn += '\n%s\n' % log_entry.msg for path in log_entry.paths: if path.action == 'delete': # Use parent and copy_from_path for deletions, otherwise we get links # to https://.../<commit>//dev/null rtn += '[%s] %s\n' % ( path.action, log_entry.GetPathUrl( path.copy_from_path, parent=True, universal=True)) else: rtn += '[%s] %s\n' % (path.action, log_entry.GetPathUrl(path.filename, universal=True)) return rtn
from __future__ import print_function from __future__ import division from __future__ import absolute_import from framework import authdata from framework import sql from framework import xsrf from gae_ts_mon.handlers import TSMonJSHandler from google.appengine.api import users from infra_libs import ts_mon STANDARD_FIELDS = [ ts_mon.StringField('client_id'), ts_mon.StringField('host_name'), ts_mon.BooleanField('document_visible'), ] # User action metrics. ISSUE_CREATE_LATENCY_METRIC = ts_mon.CumulativeDistributionMetric( 'monorail/frontend/issue_create_latency', ('Latency between Issue Entry form submission and page load of ' 'the subsequent issue page.'), field_spec=STANDARD_FIELDS, units=ts_mon.MetricsDataUnits.MILLISECONDS) ISSUE_UPDATE_LATENCY_METRIC = ts_mon.CumulativeDistributionMetric( 'monorail/frontend/issue_update_latency', ('Latency between Issue Update form submission and page load of ' 'the subsequent issue page.'),
class RateLimiter(object): blocked_requests = ts_mon.CounterMetric( 'monorail/ratelimiter/blocked_request', 'Count of requests that exceeded the rate limit and were blocked.', None) limit_exceeded = ts_mon.CounterMetric( 'monorail/ratelimiter/rate_exceeded', 'Count of requests that exceeded the rate limit.', None) cost_thresh_exceeded = ts_mon.CounterMetric( 'monorail/ratelimiter/cost_thresh_exceeded', 'Count of requests that were expensive to process', None) checks = ts_mon.CounterMetric( 'monorail/ratelimiter/check', 'Count of checks done, by fail/success type.', [ts_mon.StringField('type')]) def __init__(self, _cache=memcache, fail_open=True, **_kwargs): self.fail_open = fail_open def CheckStart(self, request, now=None): if (modules.get_current_module_name() not in MODULE_WHITELIST or users.is_current_user_admin()): return logging.info('X-AppEngine-Country: %s' % request.headers.get(COUNTRY_HEADER, 'ZZ')) if now is None: now = time.time() keysets, country, ip, user_email = _CacheKeys(request, now) # There are either two or three sets of keys in keysets. # Three if the user's country is in COUNTRY_LIMITS, otherwise two. self._AuxCheckStart( keysets, COUNTRY_LIMITS.get(country, DEFAULT_LIMIT), settings.ratelimiting_enabled, RateLimitExceeded(country=country, ip=ip, user_email=user_email)) def _AuxCheckStart(self, keysets, limit, ratelimiting_enabled, exception_obj): for keys in keysets: count = 0 try: counters = memcache.get_multi(keys) count = sum(counters.values()) self.checks.increment({'type': 'success'}) except Exception as e: logging.error(e) if not self.fail_open: self.checks.increment({'type': 'fail_closed'}) raise exception_obj self.checks.increment({'type': 'fail_open'}) if count > limit: # Since webapp2 won't let us return a 429 error code # <http://tools.ietf.org/html/rfc6585#section-4>, we can't # monitor rate limit exceeded events with our standard tools. # We return a 400 with a custom error message to the client, # and this logging is so we can monitor it internally. logging.info('%s, %d' % (exception_obj.message, count)) self.limit_exceeded.increment() if ratelimiting_enabled: self.blocked_requests.increment() raise exception_obj k = keys[0] # Only update the latest *time* bucket for each prefix (reverse chron). memcache.add(k, 0, time=EXPIRE_AFTER_SECS) memcache.incr(k, initial_value=0) def CheckEnd(self, request, now, start_time): """If a request was expensive to process, charge some extra points against this set of buckets. We pass in both now and start_time so we can update the buckets based on keys created from start_time instead of now. now and start_time are float seconds. """ if (modules.get_current_module_name() not in MODULE_WHITELIST or not settings.ratelimiting_cost_enabled): return elapsed_ms = (now - start_time) * 1000 # Would it kill the python lib maintainers to have timedelta.total_ms()? if elapsed_ms < settings.ratelimiting_cost_thresh_ms: return # TODO: Look into caching the keys instead of generating them twice # for every request. Say, return them from CheckStart so they can # be bassed back in here later. keysets, country, ip, user_email = _CacheKeys(request, start_time) self._AuxCheckEnd( keysets, 'Rate Limit Cost Threshold Exceeded: %s, %s, %s' % (country, ip, user_email), settings.ratelimiting_cost_penalty) def _AuxCheckEnd(self, keysets, log_str, ratelimiting_cost_penalty): self.cost_thresh_exceeded.increment() for keys in keysets: logging.info(log_str) # Only update the latest *time* bucket for each prefix (reverse chron). k = keys[0] memcache.add(k, 0, time=EXPIRE_AFTER_SECS) memcache.incr(k, delta=ratelimiting_cost_penalty, initial_value=0)
def RecordSubmissionMetrics(action_history, submitted_change_strategies): """Record submission metrics in monarch. Args: action_history: A CLActionHistory instance for all cl actions for all changes in submitted_change_strategies. submitted_change_strategies: A dictionary from GerritPatchTuples to submission strategy strings. These changes will have their handling times recorded in monarch. """ # TODO(phobbs) move to top level after crbug.com/755415 handling_time_metric = metrics.CumulativeSecondsDistribution( constants.MON_CL_HANDLE_TIME) wall_clock_time_metric = metrics.CumulativeSecondsDistribution( constants.MON_CL_WALL_CLOCK_TIME) precq_time_metric = metrics.CumulativeSecondsDistribution( constants.MON_CL_PRECQ_TIME) wait_time_metric = metrics.CumulativeSecondsDistribution( constants.MON_CL_WAIT_TIME) cq_run_time_metric = metrics.CumulativeSecondsDistribution( constants.MON_CL_CQRUN_TIME) cq_tries_metric = metrics.CumulativeSmallIntegerDistribution( constants.MON_CL_CQ_TRIES) # These 3 false rejection metrics are different in subtle but important ways. # false_rejections: distribution of the number of times a CL was rejected, # broken down by what it was rejected by (cq vs. pre-cq). Every CL will emit # two data points to this distribution. false_rejection_metric = metrics.CumulativeSmallIntegerDistribution( constants.MON_CL_FALSE_REJ) # false_rejections_total: distribution of the total number of times a CL # was rejected (not broken down by phase). Note that there is no way to # independently calculate this from |false_rejections| distribution above, # since one cannot reconstruct after the fact which pre-cq and cq data points # (for the same underlying CL) belong together. false_rejection_total_metric = metrics.CumulativeSmallIntegerDistribution( constants.MON_CL_FALSE_REJ_TOTAL) # false_rejection_count: counter of the total number of false rejections that # have occurred (broken down by phase) false_rejection_count_metric = metrics.Counter( constants.MON_CL_FALSE_REJ_COUNT) # This metric excludes rejections which were forgiven by the CL-exonerator # service. false_rejections_minus_exonerations_metric = \ metrics.CumulativeSmallIntegerDistribution( constants.MON_CQ_FALSE_REJ_MINUS_EXONERATIONS, description='The number of rejections - exonerations per CL.', field_spec=[ts_mon.StringField('submission_strategy')], ) precq_false_rejections = action_history.GetFalseRejections( bot_type=constants.PRE_CQ) cq_false_rejections = action_history.GetFalseRejections( bot_type=constants.CQ) exonerations = action_history.GetExonerations() for change, strategy in submitted_change_strategies.iteritems(): strategy = strategy or '' handling_time = clactions.GetCLHandlingTime(change, action_history) wall_clock_time = clactions.GetCLWallClockTime(change, action_history) precq_time = clactions.GetPreCQTime(change, action_history) wait_time = clactions.GetCQWaitTime(change, action_history) run_time = clactions.GetCQRunTime(change, action_history) pickups = clactions.GetCQAttemptsCount(change, action_history) fields = {'submission_strategy': strategy} handling_time_metric.add(handling_time, fields=fields) wall_clock_time_metric.add(wall_clock_time, fields=fields) precq_time_metric.add(precq_time, fields=fields) wait_time_metric.add(wait_time, fields=fields) cq_run_time_metric.add(run_time, fields=fields) cq_tries_metric.add(pickups, fields=fields) rejection_types = ( (constants.PRE_CQ, precq_false_rejections), (constants.CQ, cq_false_rejections), ) total_rejections = 0 for by, rej in rejection_types: c = len(rej.get(change, [])) f = dict(fields, rejected_by=by) false_rejection_metric.add(c, fields=f) false_rejection_count_metric.increment_by(c, fields=f) total_rejections += c false_rejection_total_metric.add(total_rejections, fields=fields) n_exonerations = len(exonerations.get(change, [])) # TODO(crbug.com/804900) max(0, ...) is required because of an accounting # bug where sometimes this quantity is negative. net_rejections = total_rejections - n_exonerations if net_rejections < 0: logging.error( 'Exonerations is larger than total rejections for CL %s.' ' See crbug.com/804900', change) false_rejections_minus_exonerations_metric.add(max(0, net_rejections), fields=fields)
LOGGER = logging.getLogger(__name__) LoopResults = collections.namedtuple( 'LoopResults', [ # True on no errors or if all failed attempts were successfully retried. 'success', # Total number of errors seen (some may have been fixed with retries). 'error_count', ], ) count_metric = ts_mon.CounterMetric( 'proc/outer_loop/count', 'Counter of loop iterations for this process, by success or failure', [ts_mon.StringField('status')]) success_metric = ts_mon.BooleanMetric('proc/outer_loop/success', 'Set immediately before the loop exits', None) durations_metric = ts_mon.CumulativeDistributionMetric( 'proc/outer_loop/durations', 'Times (in seconds) taken to execute the task', None) def loop(task, sleep_timeout, duration=None, max_errors=None, time_mod=time): """Runs the task in a loop for a given duration. Handles and logs all uncaught exceptions. ``task`` callback should return True on success, and False (or raise an exception) in error. Doesn't leak any exceptions (including KeyboardInterrupt).
import os import sys import urlparse from infra.libs import git2 from infra.libs.service_utils import outer_loop from infra.services.gsubtreed import gsubtreed from infra_libs import logs from infra_libs import ts_mon # Return value of parse_args. Options = collections.namedtuple('Options', 'repo loop_opts json_output') commits_counter = ts_mon.CounterMetric( 'gsubtreed/commit_count', 'Number of commits processed by gsubtreed', [ts_mon.StringField('path')]) def parse_args(args): # pragma: no cover def check_url(s): parsed = urlparse.urlparse(s) if parsed.scheme not in ('https', 'git', 'file'): raise argparse.ArgumentTypeError( 'Repo URL must use https, git or file protocol.') if not parsed.path.strip('/'): raise argparse.ArgumentTypeError('URL is missing a path?') return git2.Repo(s) parser = argparse.ArgumentParser('./run.py %s' % __package__) parser.add_argument('--dry_run', action='store_true',
class MonorailApi(remote.Service): # Class variables. Handy to mock. _services = None _mar = None api_requests = ts_mon.CounterMetric( 'monorail/api_requests', 'Number of requests to Monorail api', [ts_mon.StringField('client_id'), ts_mon.StringField('client_email')]) ratelimiter = ratelimiter.ApiRateLimiter() @classmethod def _set_services(cls, services): cls._services = services def mar_factory(self, request): if not self._mar: self._mar = monorailrequest.MonorailApiRequest(request, self._services) return self._mar def aux_delete_comment(self, request, delete=True): mar = self.mar_factory(request) action_name = 'delete' if delete else 'undelete' issue = self._services.issue.GetIssueByLocalID( mar.cnxn, mar.project_id, request.issueId) all_comments = self._services.issue.GetCommentsForIssue( mar.cnxn, issue.issue_id) try: issue_comment = all_comments[request.commentId] except IndexError: raise issue_svc.NoSuchIssueException( 'The issue %s:%d does not have comment %d.' % (mar.project_name, request.issueId, request.commentId)) if not permissions.CanDelete( mar.auth.user_id, mar.auth.effective_ids, mar.perms, issue_comment.deleted_by, issue_comment.user_id, mar.project, permissions.GetRestrictions(issue), mar.granted_perms): raise permissions.PermissionException( 'User is not allowed to %s the comment %d of issue %s:%d' % (action_name, request.commentId, mar.project_name, request.issueId)) self._services.issue.SoftDeleteComment( mar.cnxn, mar.project_id, request.issueId, request.commentId, mar.auth.user_id, self._services.user, delete=delete) return api_pb2_v1.IssuesCommentsDeleteResponse() def increment_request_limit(self, request, client_id, client_email): """Check whether the requester has exceeded API quotas limit, and increment request count in DB and ts_mon. """ mar = self.mar_factory(request) # soft_limit == hard_limit for api_request, so this function either # returns False if under limit, or raise ExcessiveActivityException if not actionlimit.NeedCaptcha( mar.auth.user_pb, actionlimit.API_REQUEST, skip_lifetime_check=True): actionlimit.CountAction( mar.auth.user_pb, actionlimit.API_REQUEST, delta=1) self._services.user.UpdateUser( mar.cnxn, mar.auth.user_id, mar.auth.user_pb) # Avoid value explosision and protect PII info if not framework_helpers.IsServiceAccount(client_email): client_email = '*****@*****.**' self.api_requests.increment_by( 1, {'client_id': client_id, 'client_email': client_email}) @monorail_api_method( api_pb2_v1.ISSUES_COMMENTS_DELETE_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesCommentsDeleteResponse, path='projects/{projectId}/issues/{issueId}/comments/{commentId}', http_method='DELETE', name='issues.comments.delete') def issues_comments_delete(self, request): """Delete a comment.""" return self.aux_delete_comment(request, True) @monorail_api_method( api_pb2_v1.ISSUES_COMMENTS_INSERT_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesCommentsInsertResponse, path='projects/{projectId}/issues/{issueId}/comments', http_method='POST', name='issues.comments.insert') def issues_comments_insert(self, request): """Add a comment.""" mar = self.mar_factory(request) issue = self._services.issue.GetIssueByLocalID( mar.cnxn, mar.project_id, request.issueId) old_owner_id = tracker_bizobj.GetOwnerId(issue) if not permissions.CanCommentIssue( mar.auth.effective_ids, mar.perms, mar.project, issue, mar.granted_perms): raise permissions.PermissionException( 'User is not allowed to comment this issue (%s, %d)' % (request.projectId, request.issueId)) updates_dict = {} if request.updates: if request.updates.moveToProject: move_to = request.updates.moveToProject.lower() move_to_project = issuedetail.CheckMoveIssueRequest( self._services, mar, issue, True, move_to, mar.errors) if mar.errors.AnyErrors(): raise endpoints.BadRequestException(mar.errors.move_to) updates_dict['move_to_project'] = move_to_project updates_dict['summary'] = request.updates.summary updates_dict['status'] = request.updates.status if request.updates.owner: if request.updates.owner == framework_constants.NO_USER_NAME: updates_dict['owner'] = framework_constants.NO_USER_SPECIFIED else: updates_dict['owner'] = self._services.user.LookupUserID( mar.cnxn, request.updates.owner) updates_dict['cc_add'], updates_dict['cc_remove'] = ( api_pb2_v1_helpers.split_remove_add(request.updates.cc)) updates_dict['cc_add'] = self._services.user.LookupUserIDs( mar.cnxn, updates_dict['cc_add'], autocreate=True).values() updates_dict['cc_remove'] = self._services.user.LookupUserIDs( mar.cnxn, updates_dict['cc_remove']).values() updates_dict['labels_add'], updates_dict['labels_remove'] = ( api_pb2_v1_helpers.split_remove_add(request.updates.labels)) blocked_on_add_strs, blocked_on_remove_strs = ( api_pb2_v1_helpers.split_remove_add(request.updates.blockedOn)) updates_dict['blocked_on_add'] = api_pb2_v1_helpers.issue_global_ids( blocked_on_add_strs, issue.project_id, mar, self._services) updates_dict['blocked_on_remove'] = api_pb2_v1_helpers.issue_global_ids( blocked_on_remove_strs, issue.project_id, mar, self._services) blocking_add_strs, blocking_remove_strs = ( api_pb2_v1_helpers.split_remove_add(request.updates.blocking)) updates_dict['blocking_add'] = api_pb2_v1_helpers.issue_global_ids( blocking_add_strs, issue.project_id, mar, self._services) updates_dict['blocking_remove'] = api_pb2_v1_helpers.issue_global_ids( blocking_remove_strs, issue.project_id, mar, self._services) components_add_strs, components_remove_strs = ( api_pb2_v1_helpers.split_remove_add(request.updates.components)) updates_dict['components_add'] = ( api_pb2_v1_helpers.convert_component_ids( mar.config, components_add_strs)) updates_dict['components_remove'] = ( api_pb2_v1_helpers.convert_component_ids( mar.config, components_remove_strs)) if request.updates.mergedInto: merge_project_name, merge_local_id = tracker_bizobj.ParseIssueRef( request.updates.mergedInto) merge_into_project = self._services.project.GetProjectByName( mar.cnxn, merge_project_name or issue.project_name) merge_into_issue = self._services.issue.GetIssueByLocalID( mar.cnxn, merge_into_project.project_id, merge_local_id) merge_allowed = tracker_helpers.IsMergeAllowed( merge_into_issue, mar, self._services) if not merge_allowed: raise permissions.PermissionException( 'User is not allowed to merge into issue %s:%s' % (merge_into_issue.project_name, merge_into_issue.local_id)) updates_dict['merged_into'] = merge_into_issue.issue_id (updates_dict['field_vals_add'], updates_dict['field_vals_remove'], updates_dict['fields_clear'], updates_dict['fields_labels_add'], updates_dict['fields_labels_remove']) = ( api_pb2_v1_helpers.convert_field_values( request.updates.fieldValues, mar, self._services)) field_helpers.ValidateCustomFields( mar, self._services, (updates_dict.get('field_vals_add', []) + updates_dict.get('field_vals_remove', [])), mar.config, mar.errors) if mar.errors.AnyErrors(): raise endpoints.BadRequestException( 'Invalid field values: %s' % mar.errors.custom_fields) _, comment = self._services.issue.DeltaUpdateIssue( cnxn=mar.cnxn, services=self._services, reporter_id=mar.auth.user_id, project_id=mar.project_id, config=mar.config, issue=issue, status=updates_dict.get('status'), owner_id=updates_dict.get('owner'), cc_add=updates_dict.get('cc_add', []), cc_remove=updates_dict.get('cc_remove', []), comp_ids_add=updates_dict.get('components_add', []), comp_ids_remove=updates_dict.get('components_remove', []), labels_add=(updates_dict.get('labels_add', []) + updates_dict.get('fields_labels_add', [])), labels_remove=(updates_dict.get('labels_remove', []) + updates_dict.get('fields_labels_remove', [])), field_vals_add=updates_dict.get('field_vals_add', []), field_vals_remove=updates_dict.get('field_vals_remove', []), fields_clear=updates_dict.get('fields_clear', []), blocked_on_add=updates_dict.get('blocked_on_add', []), blocked_on_remove=updates_dict.get('blocked_on_remove', []), blocking_add=updates_dict.get('blocking_add', []), blocking_remove=updates_dict.get('blocking_remove', []), merged_into=updates_dict.get('merged_into'), index_now=False, comment=request.content, summary=updates_dict.get('summary'), ) move_comment = None if 'move_to_project' in updates_dict: move_to_project = updates_dict['move_to_project'] old_text_ref = 'issue %s:%s' % (issue.project_name, issue.local_id) tracker_fulltext.UnindexIssues([issue.issue_id]) moved_back_iids = self._services.issue.MoveIssues( mar.cnxn, move_to_project, [issue], self._services.user) new_text_ref = 'issue %s:%s' % (issue.project_name, issue.local_id) if issue.issue_id in moved_back_iids: content = 'Moved %s back to %s again.' % (old_text_ref, new_text_ref) else: content = 'Moved %s to now be %s.' % (old_text_ref, new_text_ref) move_comment = self._services.issue.CreateIssueComment( mar.cnxn, move_to_project.project_id, issue.local_id, mar.auth.user_id, content, amendments=[ tracker_bizobj.MakeProjectAmendment(move_to_project.project_name)]) if 'merged_into' in updates_dict: new_starrers = tracker_helpers.GetNewIssueStarrers( mar.cnxn, self._services, issue.issue_id, merge_into_issue.issue_id) tracker_helpers.AddIssueStarrers( mar.cnxn, self._services, mar, merge_into_issue.issue_id, merge_into_project, new_starrers) _merge_comment = tracker_helpers.MergeCCsAndAddComment( self._services, mar, issue, merge_into_project, merge_into_issue) merge_into_issue_cmnts = self._services.issue.GetCommentsForIssue( mar.cnxn, merge_into_issue.issue_id) notify.PrepareAndSendIssueChangeNotification( merge_into_issue.issue_id, framework_helpers.GetHostPort(), mar.auth.user_id, len(merge_into_issue_cmnts) - 1, send_email=True) tracker_fulltext.IndexIssues( mar.cnxn, [issue], self._services.user, self._services.issue, self._services.config) comment = comment or move_comment if comment is None: return api_pb2_v1.IssuesCommentsInsertResponse() cmnts = self._services.issue.GetCommentsForIssue(mar.cnxn, issue.issue_id) seq = len(cmnts) - 1 if request.sendEmail: notify.PrepareAndSendIssueChangeNotification( issue.issue_id, framework_helpers.GetHostPort(), comment.user_id, seq, send_email=True, old_owner_id=old_owner_id) can_delete = permissions.CanDelete( mar.auth.user_id, mar.auth.effective_ids, mar.perms, comment.deleted_by, comment.user_id, mar.project, permissions.GetRestrictions(issue), granted_perms=mar.granted_perms) return api_pb2_v1.IssuesCommentsInsertResponse( id=seq, kind='monorail#issueComment', author=api_pb2_v1_helpers.convert_person( comment.user_id, mar.cnxn, self._services), content=comment.content, published=datetime.datetime.fromtimestamp(comment.timestamp), updates=api_pb2_v1_helpers.convert_amendments( issue, comment.amendments, mar, self._services), canDelete=can_delete) @monorail_api_method( api_pb2_v1.ISSUES_COMMENTS_LIST_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesCommentsListResponse, path='projects/{projectId}/issues/{issueId}/comments', http_method='GET', name='issues.comments.list') def issues_comments_list(self, request): """List all comments for an issue.""" mar = self.mar_factory(request) issue = self._services.issue.GetIssueByLocalID( mar.cnxn, mar.project_id, request.issueId) comments = self._services.issue.GetCommentsForIssue( mar.cnxn, issue.issue_id) visible_comments = [] for comment in comments[ request.startIndex:(request.startIndex + request.maxResults)]: visible_comments.append( api_pb2_v1_helpers.convert_comment( issue, comment, mar, self._services, mar.granted_perms)) return api_pb2_v1.IssuesCommentsListResponse( kind='monorail#issueCommentList', totalResults=len(comments), items=visible_comments) @monorail_api_method( api_pb2_v1.ISSUES_COMMENTS_DELETE_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesCommentsDeleteResponse, path='projects/{projectId}/issues/{issueId}/comments/{commentId}', http_method='POST', name='issues.comments.undelete') def issues_comments_undelete(self, request): """Restore a deleted comment.""" return self.aux_delete_comment(request, False) @monorail_api_method( api_pb2_v1.USERS_GET_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.UsersGetResponse, path='users/{userId}', http_method='GET', name='users.get') def users_get(self, request): """Get a user.""" owner_project_only = request.ownerProjectsOnly mar = self.mar_factory(request) (visible_ownership, visible_deleted, visible_membership, visible_contrib) = sitewide_helpers.GetUserProjects( mar.cnxn, self._services, mar.auth.user_pb, mar.auth.effective_ids, mar.viewed_user_auth.effective_ids) project_list = [] for proj in (visible_ownership + visible_deleted): config = self._services.config.GetProjectConfig( mar.cnxn, proj.project_id) proj_result = api_pb2_v1_helpers.convert_project( proj, config, api_pb2_v1.Role.owner) project_list.append(proj_result) if not owner_project_only: for proj in visible_membership: config = self._services.config.GetProjectConfig( mar.cnxn, proj.project_id) proj_result = api_pb2_v1_helpers.convert_project( proj, config, api_pb2_v1.Role.member) project_list.append(proj_result) for proj in visible_contrib: config = self._services.config.GetProjectConfig( mar.cnxn, proj.project_id) proj_result = api_pb2_v1_helpers.convert_project( proj, config, api_pb2_v1.Role.contributor) project_list.append(proj_result) return api_pb2_v1.UsersGetResponse( id=str(mar.viewed_user_auth.user_id), kind='monorail#user', projects=project_list, ) @monorail_api_method( api_pb2_v1.ISSUES_GET_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesGetInsertResponse, path='projects/{projectId}/issues/{issueId}', http_method='GET', name='issues.get') def issues_get(self, request): """Get an issue.""" mar = self.mar_factory(request) issue = self._services.issue.GetIssueByLocalID( mar.cnxn, mar.project_id, request.issueId) return api_pb2_v1_helpers.convert_issue( api_pb2_v1.IssuesGetInsertResponse, issue, mar, self._services) @monorail_api_method( api_pb2_v1.ISSUES_INSERT_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesGetInsertResponse, path='projects/{projectId}/issues', http_method='POST', name='issues.insert') def issues_insert(self, request): """Add a new issue.""" mar = self.mar_factory(request) if not mar.perms.CanUsePerm( permissions.CREATE_ISSUE, mar.auth.effective_ids, mar.project, []): raise permissions.PermissionException( 'The requester %s is not allowed to create issues for project %s.' % (mar.auth.email, mar.project_name)) owner_id = None if request.owner: try: owner_id = self._services.user.LookupUserID( mar.cnxn, request.owner.name) except user_svc.NoSuchUserException: raise endpoints.BadRequestException( 'The specified owner %s does not exist.' % request.owner.name) cc_ids = [] if request.cc: cc_ids = self._services.user.LookupUserIDs( mar.cnxn, [ap.name for ap in request.cc], autocreate=True).values() comp_ids = api_pb2_v1_helpers.convert_component_ids( mar.config, request.components) fields_add, _, _, fields_labels, _ = ( api_pb2_v1_helpers.convert_field_values( request.fieldValues, mar, self._services)) field_helpers.ValidateCustomFields( mar, self._services, fields_add, mar.config, mar.errors) if mar.errors.AnyErrors(): raise endpoints.BadRequestException( 'Invalid field values: %s' % mar.errors.custom_fields) local_id = self._services.issue.CreateIssue( mar.cnxn, self._services, mar.project_id, request.summary, request.status, owner_id, cc_ids, request.labels + fields_labels, fields_add, comp_ids, mar.auth.user_id, request.description, blocked_on=api_pb2_v1_helpers.convert_issueref_pbs( request.blockedOn, mar, self._services), blocking=api_pb2_v1_helpers.convert_issueref_pbs( request.blocking, mar, self._services)) new_issue = self._services.issue.GetIssueByLocalID( mar.cnxn, mar.project_id, local_id) self._services.issue_star.SetStar( mar.cnxn, self._services, mar.config, new_issue.issue_id, mar.auth.user_id, True) if request.sendEmail: notify.PrepareAndSendIssueChangeNotification( new_issue.issue_id, framework_helpers.GetHostPort(), new_issue.reporter_id, 0) return api_pb2_v1_helpers.convert_issue( api_pb2_v1.IssuesGetInsertResponse, new_issue, mar, self._services) @monorail_api_method( api_pb2_v1.ISSUES_LIST_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.IssuesListResponse, path='projects/{projectId}/issues', http_method='GET', name='issues.list') def issues_list(self, request): """List issues for projects.""" mar = self.mar_factory(request) if request.additionalProject: for project_name in request.additionalProject: project = self._services.project.GetProjectByName( mar.cnxn, project_name) if project and not permissions.UserCanViewProject( mar.auth.user_pb, mar.auth.effective_ids, project): raise permissions.PermissionException( 'The user %s has no permission for project %s' % (mar.auth.email, project_name)) prof = profiler.Profiler() pipeline = frontendsearchpipeline.FrontendSearchPipeline( mar, self._services, prof, mar.num) if not mar.errors.AnyErrors(): pipeline.SearchForIIDs() pipeline.MergeAndSortIssues() pipeline.Paginate() else: raise endpoints.BadRequestException(mar.errors.query) issue_list = [ api_pb2_v1_helpers.convert_issue( api_pb2_v1.IssueWrapper, r, mar, self._services) for r in pipeline.visible_results] return api_pb2_v1.IssuesListResponse( kind='monorail#issueList', totalResults=pipeline.total_count, items=issue_list) @monorail_api_method( api_pb2_v1.GROUPS_SETTINGS_LIST_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.GroupsSettingsListResponse, path='groups/settings', http_method='GET', name='groups.settings.list') def groups_settings_list(self, request): """List all group settings.""" mar = self.mar_factory(request) all_groups = self._services.usergroup.GetAllUserGroupsInfo(mar.cnxn) group_settings = [] for g in all_groups: setting = g[2] wrapper = api_pb2_v1_helpers.convert_group_settings(g[0], setting) if not request.importedGroupsOnly or wrapper.ext_group_type: group_settings.append(wrapper) return api_pb2_v1.GroupsSettingsListResponse( groupSettings=group_settings) @monorail_api_method( api_pb2_v1.GROUPS_CREATE_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.GroupsCreateResponse, path='groups', http_method='POST', name='groups.create') def groups_create(self, request): """Create a new user group.""" mar = self.mar_factory(request) if not permissions.CanCreateGroup(mar.perms): raise permissions.PermissionException( 'The user is not allowed to create groups.') user_dict = self._services.user.LookupExistingUserIDs( mar.cnxn, [request.groupName]) if request.groupName.lower() in user_dict: raise usergroup_svc.GroupExistsException( 'group %s already exists' % request.groupName) if request.ext_group_type: ext_group_type = str(request.ext_group_type).lower() else: ext_group_type = None group_id = self._services.usergroup.CreateGroup( mar.cnxn, self._services, request.groupName, str(request.who_can_view_members).lower(), ext_group_type) return api_pb2_v1.GroupsCreateResponse( groupID=group_id) @monorail_api_method( api_pb2_v1.GROUPS_GET_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.GroupsGetResponse, path='groups/{groupName}', http_method='GET', name='groups.get') def groups_get(self, request): """Get a group's settings and users.""" mar = self.mar_factory(request) if not mar.viewed_user_auth: raise user_svc.NoSuchUserException(request.groupName) group_id = mar.viewed_user_auth.user_id group_settings = self._services.usergroup.GetGroupSettings( mar.cnxn, group_id) member_ids, owner_ids = self._services.usergroup.LookupAllMembers( mar.cnxn, [group_id]) (owned_project_ids, membered_project_ids, contrib_project_ids) = self._services.project.GetUserRolesInAllProjects( mar.cnxn, mar.auth.effective_ids) project_ids = owned_project_ids.union( membered_project_ids).union(contrib_project_ids) if not permissions.CanViewGroup( mar.perms, mar.auth.effective_ids, group_settings, member_ids[group_id], owner_ids[group_id], project_ids): raise permissions.PermissionException( 'The user is not allowed to view this group.') member_ids, owner_ids = self._services.usergroup.LookupMembers( mar.cnxn, [group_id]) member_emails = self._services.user.LookupUserEmails( mar.cnxn, member_ids[group_id]).values() owner_emails = self._services.user.LookupUserEmails( mar.cnxn, owner_ids[group_id]).values() return api_pb2_v1.GroupsGetResponse( groupID=group_id, groupSettings=api_pb2_v1_helpers.convert_group_settings( request.groupName, group_settings), groupOwners=owner_emails, groupMembers=member_emails) @monorail_api_method( api_pb2_v1.GROUPS_UPDATE_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.GroupsUpdateResponse, path='groups/{groupName}', http_method='POST', name='groups.update') def groups_update(self, request): """Update a group's settings and users.""" mar = self.mar_factory(request) group_id = mar.viewed_user_auth.user_id member_ids_dict, owner_ids_dict = self._services.usergroup.LookupMembers( mar.cnxn, [group_id]) owner_ids = owner_ids_dict.get(group_id, []) member_ids = member_ids_dict.get(group_id, []) if not permissions.CanEditGroup( mar.perms, mar.auth.effective_ids, owner_ids): raise permissions.PermissionException( 'The user is not allowed to edit this group.') group_settings = self._services.usergroup.GetGroupSettings( mar.cnxn, group_id) if (request.who_can_view_members or request.ext_group_type or request.last_sync_time or request.friend_projects): group_settings.who_can_view_members = ( request.who_can_view_members or group_settings.who_can_view_members) group_settings.ext_group_type = ( request.ext_group_type or group_settings.ext_group_type) group_settings.last_sync_time = ( request.last_sync_time or group_settings.last_sync_time) if framework_constants.NO_VALUES in request.friend_projects: group_settings.friend_projects = [] else: id_dict = self._services.project.LookupProjectIDs( mar.cnxn, request.friend_projects) group_settings.friend_projects = ( id_dict.values() or group_settings.friend_projects) self._services.usergroup.UpdateSettings( mar.cnxn, group_id, group_settings) if request.groupOwners or request.groupMembers: self._services.usergroup.RemoveMembers( mar.cnxn, group_id, owner_ids + member_ids) owners_dict = self._services.user.LookupUserIDs( mar.cnxn, request.groupOwners, autocreate=True) self._services.usergroup.UpdateMembers( mar.cnxn, group_id, owners_dict.values(), 'owner') members_dict = self._services.user.LookupUserIDs( mar.cnxn, request.groupMembers, autocreate=True) self._services.usergroup.UpdateMembers( mar.cnxn, group_id, members_dict.values(), 'member') return api_pb2_v1.GroupsUpdateResponse() @monorail_api_method( api_pb2_v1.COMPONENTS_LIST_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.ComponentsListResponse, path='projects/{projectId}/components', http_method='GET', name='components.list') def components_list(self, request): """List all components of a given project.""" mar = self.mar_factory(request) config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id) components = [api_pb2_v1_helpers.convert_component_def( cd, mar, self._services) for cd in config.component_defs] return api_pb2_v1.ComponentsListResponse( components=components) @monorail_api_method( api_pb2_v1.COMPONENTS_CREATE_REQUEST_RESOURCE_CONTAINER, api_pb2_v1.Component, path='projects/{projectId}/components', http_method='POST', name='components.create') def components_create(self, request): """Create a component.""" mar = self.mar_factory(request) if not mar.perms.CanUsePerm( permissions.EDIT_PROJECT, mar.auth.effective_ids, mar.project, []): raise permissions.PermissionException( 'User is not allowed to create components for this project') config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id) leaf_name = request.componentName if not tracker_constants.COMPONENT_NAME_RE.match(leaf_name): raise config_svc.InvalidComponentNameException( 'The component name %s is invalid.' % leaf_name) parent_path = request.parentPath if parent_path: parent_def = tracker_bizobj.FindComponentDef(parent_path, config) if not parent_def: raise config_svc.NoSuchComponentException( 'Parent component %s does not exist.' % parent_path) if not permissions.CanEditComponentDef( mar.auth.effective_ids, mar.perms, mar.project, parent_def, config): raise permissions.PermissionException( 'User is not allowed to add a subcomponent to component %s' % parent_path) path = '%s>%s' % (parent_path, leaf_name) else: path = leaf_name if tracker_bizobj.FindComponentDef(path, config): raise config_svc.InvalidComponentNameException( 'The name %s is already in use.' % path) created = int(time.time()) user_emails = set() user_emails.update([mar.auth.email] + request.admin + request.cc) user_ids_dict = self._services.user.LookupUserIDs( mar.cnxn, list(user_emails), autocreate=False) admin_ids = [user_ids_dict[uname] for uname in request.admin] cc_ids = [user_ids_dict[uname] for uname in request.cc] label_ids = [] # TODO(jrobbins): allow API clients to specify this too. component_id = self._services.config.CreateComponentDef( mar.cnxn, mar.project_id, path, request.description, request.deprecated, admin_ids, cc_ids, created, user_ids_dict[mar.auth.email], label_ids) return api_pb2_v1.Component( componentId=component_id, projectName=request.projectId, componentPath=path, description=request.description, admin=request.admin, cc=request.cc, deprecated=request.deprecated, created=datetime.datetime.fromtimestamp(created), creator=mar.auth.email) @monorail_api_method( api_pb2_v1.COMPONENTS_DELETE_REQUEST_RESOURCE_CONTAINER, message_types.VoidMessage, path='projects/{projectId}/components/{componentPath}', http_method='DELETE', name='components.delete') def components_delete(self, request): """Delete a component.""" mar = self.mar_factory(request) config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id) component_path = request.componentPath component_def = tracker_bizobj.FindComponentDef( component_path, config) if not component_def: raise config_svc.NoSuchComponentException( 'The component %s does not exist.' % component_path) if not permissions.CanViewComponentDef( mar.auth.effective_ids, mar.perms, mar.project, component_def): raise permissions.PermissionException( 'User is not allowed to view this component %s' % component_path) if not permissions.CanEditComponentDef( mar.auth.effective_ids, mar.perms, mar.project, component_def, config): raise permissions.PermissionException( 'User is not allowed to delete this component %s' % component_path) allow_delete = not tracker_bizobj.FindDescendantComponents( config, component_def) if not allow_delete: raise permissions.PermissionException( 'User tried to delete component that had subcomponents') self._services.issue.DeleteComponentReferences( mar.cnxn, component_def.component_id) self._services.config.DeleteComponentDef( mar.cnxn, mar.project_id, component_def.component_id) return message_types.VoidMessage() @monorail_api_method( api_pb2_v1.COMPONENTS_UPDATE_REQUEST_RESOURCE_CONTAINER, message_types.VoidMessage, path='projects/{projectId}/components/{componentPath}', http_method='POST', name='components.update') def components_update(self, request): """Update a component.""" mar = self.mar_factory(request) config = self._services.config.GetProjectConfig(mar.cnxn, mar.project_id) component_path = request.componentPath component_def = tracker_bizobj.FindComponentDef( component_path, config) if not component_def: raise config_svc.NoSuchComponentException( 'The component %s does not exist.' % component_path) if not permissions.CanViewComponentDef( mar.auth.effective_ids, mar.perms, mar.project, component_def): raise permissions.PermissionException( 'User is not allowed to view this component %s' % component_path) if not permissions.CanEditComponentDef( mar.auth.effective_ids, mar.perms, mar.project, component_def, config): raise permissions.PermissionException( 'User is not allowed to edit this component %s' % component_path) original_path = component_def.path new_path = component_def.path new_docstring = component_def.docstring new_deprecated = component_def.deprecated new_admin_ids = component_def.admin_ids new_cc_ids = component_def.cc_ids update_filterrule = False for update in request.updates: if update.field == api_pb2_v1.ComponentUpdateFieldID.LEAF_NAME: leaf_name = update.leafName if not tracker_constants.COMPONENT_NAME_RE.match(leaf_name): raise config_svc.InvalidComponentNameException( 'The component name %s is invalid.' % leaf_name) if '>' in original_path: parent_path = original_path[:original_path.rindex('>')] new_path = '%s>%s' % (parent_path, leaf_name) else: new_path = leaf_name conflict = tracker_bizobj.FindComponentDef(new_path, config) if conflict and conflict.component_id != component_def.component_id: raise config_svc.InvalidComponentNameException( 'The name %s is already in use.' % new_path) update_filterrule = True elif update.field == api_pb2_v1.ComponentUpdateFieldID.DESCRIPTION: new_docstring = update.description elif update.field == api_pb2_v1.ComponentUpdateFieldID.ADMIN: user_ids_dict = self._services.user.LookupUserIDs( mar.cnxn, list(update.admin), autocreate=True) new_admin_ids = [user_ids_dict[email] for email in update.admin] elif update.field == api_pb2_v1.ComponentUpdateFieldID.CC: user_ids_dict = self._services.user.LookupUserIDs( mar.cnxn, list(update.cc), autocreate=True) new_cc_ids = [user_ids_dict[email] for email in update.cc] update_filterrule = True elif update.field == api_pb2_v1.ComponentUpdateFieldID.DEPRECATED: new_deprecated = update.deprecated else: logging.error('Unknown component field %r', update.field) new_modified = int(time.time()) new_modifier_id = self._services.user.LookupUserID( mar.cnxn, mar.auth.email, autocreate=False) logging.info( 'Updating component id %d: path-%s, docstring-%s, deprecated-%s,' ' admin_ids-%s, cc_ids-%s modified by %s', component_def.component_id, new_path, new_docstring, new_deprecated, new_admin_ids, new_cc_ids, new_modifier_id) self._services.config.UpdateComponentDef( mar.cnxn, mar.project_id, component_def.component_id, path=new_path, docstring=new_docstring, deprecated=new_deprecated, admin_ids=new_admin_ids, cc_ids=new_cc_ids, modified=new_modified, modifier_id=new_modifier_id) # TODO(sheyang): reuse the code in componentdetails if original_path != new_path: # If the name changed then update all of its subcomponents as well. subcomponent_ids = tracker_bizobj.FindMatchingComponentIDs( original_path, config, exact=False) for subcomponent_id in subcomponent_ids: if subcomponent_id == component_def.component_id: continue subcomponent_def = tracker_bizobj.FindComponentDefByID( subcomponent_id, config) subcomponent_new_path = subcomponent_def.path.replace( original_path, new_path, 1) self._services.config.UpdateComponentDef( mar.cnxn, mar.project_id, subcomponent_def.component_id, path=subcomponent_new_path) if update_filterrule: filterrules_helpers.RecomputeAllDerivedFields( mar.cnxn, self._services, mar.project, config) return message_types.VoidMessage()
from twisted.cred import credentials import buildslave from buildslave.pbutil import ReconnectingPBClientFactory from buildslave.commands import registry, base from buildslave import monkeypatches from infra_libs import ts_mon connected_metric = ts_mon.BooleanMetric( 'buildbot/slave/connected', 'Whether the slave is currently connected to its master.', None) connection_failures_metric = ts_mon.CounterMetric( 'buildbot/slave/connection_failures', 'Count of failures connecting to the buildbot master.', [ts_mon.StringField('reason')]) running_metric = ts_mon.BooleanMetric( 'buildbot/slave/is_building', 'Whether a build step is currently in progress.', [ts_mon.StringField('builder')]) steps_metric = ts_mon.CounterMetric( 'buildbot/slave/steps', 'Count of build steps run by each builder on this slave.', [ts_mon.StringField('builder'), ts_mon.BooleanField('success')]) class UnknownCommand(pb.Error): pass
def PerformStage(self): """Perform the actual work for this stage. This includes final metadata archival, and update CIDB with our final status as well as producting a logged build result summary. """ build_identifier, _ = self._run.GetCIDBHandle() build_id = build_identifier.cidb_id buildbucket_id = build_identifier.buildbucket_id if results_lib.Results.BuildSucceededSoFar(self.buildstore, buildbucket_id, self.name): final_status = constants.BUILDER_STATUS_PASSED else: final_status = constants.BUILDER_STATUS_FAILED if not hasattr(self._run.attrs, 'release_tag'): # If, for some reason, sync stage was not completed and # release_tag was not set. Set it to None here because # ArchiveResults() depends the existence of this attr. self._run.attrs.release_tag = None # Set up our report metadata. self._run.attrs.metadata.UpdateWithDict( self.GetReportMetadata( final_status=final_status, completion_instance=self._completion_instance)) src_root = self._build_root # Workspace builders use a different buildroot for overlays. if self._run.config.workspace_branch and self._run.options.workspace: src_root = self._run.options.workspace # Add tags for the arches and statuses of the build. # arches requires crossdev which isn't available at the early part of the # build. arches = [] for board in self._run.config['boards']: toolchains = toolchain.GetToolchainsForBoard(board, buildroot=src_root) default = list( toolchain.FilterToolchains(toolchains, 'default', True)) if default: try: arches.append(toolchain.GetArchForTarget(default[0])) except cros_build_lib.RunCommandError as e: logging.warning( 'Unable to retrieve arch for board %s default toolchain %s: %s', board, default, e) tags = { 'arches': arches, 'status': final_status, } results = self._run.attrs.metadata.GetValue('results') for stage in results: tags['stage_status:%s' % stage['name']] = stage['status'] tags['stage_summary:%s' % stage['name']] = stage['summary'] self._run.attrs.metadata.UpdateKeyDictWithDict(constants.METADATA_TAGS, tags) # Some operations can only be performed if a valid version is available. try: self._run.GetVersionInfo() self.ArchiveResults(final_status) metadata_url = os.path.join(self.upload_url, constants.METADATA_JSON) except cbuildbot_run.VersionNotSetError: logging.error('A valid version was never set for this run. ' 'Can not archive results.') metadata_url = '' results_lib.Results.Report(sys.stdout, current_version=(self._run.attrs.release_tag or '')) # Upload goma log if used for BuildPackage and TestSimpleChrome. _UploadAndLinkGomaLogIfNecessary( 'BuildPackages', self._run.config.name, self._run.options.goma_dir, self._run.options.goma_client_json, self._run.attrs.metadata.GetValueWithDefault('goma_tmp_dir')) _UploadAndLinkGomaLogIfNecessary( 'TestSimpleChromeWorkflow', self._run.config.name, self._run.options.goma_dir, self._run.options.goma_client_json, self._run.attrs.metadata.GetValueWithDefault( 'goma_tmp_dir_for_simple_chrome')) if self.buildstore.AreClientsReady(): status_for_db = final_status # TODO(pprabhu): After BuildData and CBuildbotMetdata are merged, remove # this extra temporary object creation. # XXX:HACK We're creating a BuildData with an empty URL. Don't try to # MarkGathered this object. build_data = metadata_lib.BuildData( '', self._run.attrs.metadata.GetDict()) # TODO(akeshet): Find a clearer way to get the "primary upload url" for # the metadata.json file. One alternative is _GetUploadUrls(...)[0]. # Today it seems that element 0 of its return list is the primary upload # url, but there is no guarantee or unit test coverage of that. self.buildstore.FinishBuild(build_id, status=status_for_db, summary=build_data.failure_message, metadata_url=metadata_url) duration = self._GetBuildDuration() mon_fields = { 'status': status_for_db, 'build_config': self._run.config.name, 'important': self._run.config.important } metrics.Counter( constants.MON_BUILD_COMP_COUNT).increment(fields=mon_fields) metrics.CumulativeSecondsDistribution( constants.MON_BUILD_DURATION).add(duration, fields=mon_fields) if self._run.options.sanity_check_build: metrics.Counter( constants.MON_BUILD_SANITY_COMP_COUNT).increment( fields=mon_fields) metrics.Gauge( constants.MON_BUILD_SANITY_ID, description= 'The build number of the latest sanity build. Used ' 'for recovering the link to the latest failing build ' 'in the alert when a sanity build fails.', field_spec=[ ts_mon.StringField('status'), ts_mon.StringField('build_config'), ts_mon.StringField('builder_name'), ts_mon.BooleanField('important') ]).set(self._run.buildnumber, fields=dict( mon_fields, builder_name=self._run.GetBuilderName())) if config_lib.IsMasterCQ(self._run.config): self_destructed = self._run.attrs.metadata.GetValueWithDefault( constants.SELF_DESTRUCTED_BUILD, False) mon_fields = { 'status': status_for_db, 'self_destructed': self_destructed } metrics.CumulativeSecondsDistribution( constants.MON_CQ_BUILD_DURATION).add(duration, fields=mon_fields) annotator_link = uri_lib.ConstructAnnotatorUri(build_id) logging.PrintBuildbotLink('Build annotator', annotator_link) # From this point forward, treat all exceptions as warnings. self._post_completion = True # Dump report about things we retry. retry_stats.ReportStats(sys.stdout)
# Each is identified by an int shard ID. # And there is one connection to the master DB identified by key MASTER_CNXN. MASTER_CNXN = 'master_cnxn' CONNECTION_COUNT = ts_mon.CounterMetric( 'monorail/sql/connection_count', 'Count of connections made to the SQL database.', [ts_mon.BooleanField('success')]) DB_CNXN_LATENCY = ts_mon.CumulativeDistributionMetric( 'monorail/sql/db_cnxn_latency', 'Time needed to establish a DB connection.', None) DB_QUERY_LATENCY = ts_mon.CumulativeDistributionMetric( 'monorail/sql/db_query_latency', 'Time needed to make a DB query.', [ts_mon.StringField('type')]) DB_COMMIT_LATENCY = ts_mon.CumulativeDistributionMetric( 'monorail/sql/db_commit_latency', 'Time needed to make a DB commit.', None) DB_ROLLBACK_LATENCY = ts_mon.CumulativeDistributionMetric( 'monorail/sql/db_rollback_latency', 'Time needed to make a DB rollback.', None) DB_RETRY_COUNT = ts_mon.CounterMetric('monorail/sql/db_retry_count', 'Count of queries retried.', None) DB_QUERY_COUNT = ts_mon.CounterMetric('monorail/sql/db_query_count', 'Count of queries sent to the DB.', [ts_mon.StringField('type')])
class LoadApiClientConfigs(webapp2.RequestHandler): config_loads = ts_mon.CounterMetric( 'monorail/client_config_svc/loads', 'Results of fetches from luci-config.', [ts_mon.BooleanField('success'), ts_mon.StringField('type')]) def get(self): authorization_token, _ = app_identity.get_access_token( framework_constants.OAUTH_SCOPE) response = urlfetch.fetch(LUCI_CONFIG_URL, method=urlfetch.GET, follow_redirects=False, headers={ 'Content-Type': 'application/json; charset=UTF-8', 'Authorization': 'Bearer ' + authorization_token }) if response.status_code != 200: logging.error('Invalid response from luci-config: %r', response) self.config_loads.increment({ 'success': False, 'type': 'luci-cfg-error' }) self.abort(500, 'Invalid response from luci-config') try: content_text = self._process_response(response) except Exception as e: self.abort(500, str(e)) logging.info('luci-config content decoded: %r.', content_text) configs = ClientConfig(configs=content_text, key_name='api_client_configs') configs.put() self.config_loads.increment({'success': True, 'type': 'success'}) def _process_response(self, response): try: content = json.loads(response.content) except ValueError: logging.error('Response was not JSON: %r', response.content) self.config_loads.increment({ 'success': False, 'type': 'json-load-error' }) raise try: config_content = content['content'] except KeyError: logging.error('JSON contained no content: %r', content) self.config_loads.increment({ 'success': False, 'type': 'json-key-error' }) raise try: content_text = base64.b64decode(config_content) except TypeError: logging.error('Content was not b64: %r', config_content) self.config_loads.increment({ 'success': False, 'type': 'b64-decode-error' }) raise try: cfg = api_clients_config_pb2.ClientCfg() protobuf.text_format.Merge(content_text, cfg) except: logging.error('Content was not a valid ClientCfg proto: %r', content_text) self.config_loads.increment({ 'success': False, 'type': 'proto-load-error' }) raise return content_text
class EventMonUploader(webapp2.RequestHandler): num_test_results = ts_mon.CounterMetric( 'test_results/num_test_results', 'Number of reported test results', [ts_mon.StringField('result_type'), ts_mon.StringField('master'), ts_mon.StringField('builder'), ts_mon.StringField('test_type')]) def post(self): if not self.request.body: logging.error('Missing request payload') self.response.set_status(400) return try: payload = json.loads(self.request.body) except ValueError: logging.error('Failed to parse request payload as JSON') self.response.set_status(400) return # Retrieve test json from datastore based on task parameters. master = payload.get('master') builder = payload.get('builder') build_number = payload.get('build_number') test_type = payload.get('test_type') step_name = payload.get('step_name') if (not master or not builder or build_number is None or not test_type or not step_name): logging.error( 'Missing required parameters: (master=%s, builder=%s, ' 'build_number=%s, test_type=%s, step_name=%s)' % (master, builder, build_number, test_type, step_name)) self.response.set_status(400) return files = TestFile.get_files( master, builder, test_type, build_number, 'full_results.json', load_data=True, limit=1) if not files: logging.error('Failed to find full_results.json for (%s, %s, %s, %s)' % ( master, builder, build_number, test_type)) self.response.set_status(404) return file_json = JsonResults.load_json(files[0].data) # Create a proto event and send it to event_mon. event = event_mon.Event('POINT') test_results = event.proto.test_results test_results.master_name = master test_results.builder_name = builder test_results.build_number = int(build_number) test_results.test_type = test_type test_results.step_name = step_name if 'interrupted' in file_json: test_results.interrupted = file_json['interrupted'] if 'version' in file_json: test_results.version = file_json['version'] if 'seconds_since_epoch' in file_json: test_results.usec_since_epoch = long( float(file_json['seconds_since_epoch']) * 1000 * 1000) def convert_test_result_type(json_val): self.num_test_results.increment({ 'result_type': json_val, 'master': master, 'builder': builder, 'test_type': test_type}) try: return (event_mon.protos.chrome_infra_log_pb2.TestResultsEvent. TestResultType.Value(json_val.upper().replace('+', '_'))) except ValueError: return event_mon.protos.chrome_infra_log_pb2.TestResultsEvent.UNKNOWN tests = util.flatten_tests_trie( file_json.get('tests', {}), file_json.get('path_delimiter', '/')) for name, test in tests.iteritems(): test_result = test_results.tests.add() test_result.test_name = name test_result.actual.extend( convert_test_result_type(res) for res in test['actual']) test_result.expected.extend( convert_test_result_type(res) for res in test['expected']) event.send()
class ServiceThread(threading.Thread): """Thread that controls a single Service object. The methods on this object (start_service(), stop_service(), etc.) can be called from any thread and are asynchronous - they just instruct the thread to perform the given action on the Service. This thread also polls the service occasionally and restarts it if it crashed. """ failures = ts_mon.CounterMetric('service_manager/failures', 'Number of times each service unexpectedly exited and was restarted by ' 'service manager', [ts_mon.StringField('service')]) reconfigs = ts_mon.CounterMetric('service_manager/reconfigs', 'Number of times each service was restarted because its configuration ' 'changed', [ts_mon.StringField('service')]) upgrades = ts_mon.CounterMetric('service_manager/upgrades', 'Number of times each service was restarted because its version ' 'changed', [ts_mon.StringField('service')]) def __init__(self, poll_interval, state_directory, service_config, cloudtail, wait_condition=None): """ Args: poll_interval: How often (in seconds) to restart failed services. state_directory: A file will be created in this directory (with the same name as the service) when it is running containing its PID and starttime. service_config: A dictionary containing the service's config. See README for a description of the fields. cloudtail: An object that knows how to start cloudtail. """ super(ServiceThread, self).__init__() if wait_condition is None: # pragma: no cover wait_condition = threading.Condition() self._poll_interval = poll_interval self._state_directory = state_directory self._cloudtail = cloudtail self._service = service.Service(self._state_directory, service_config, self._cloudtail) self._condition = wait_condition # Protects _state. self._state = _State() # _condition must be held. self._state_changed = False self._started = False # Whether we started the service already. def _wait(self): with self._condition: if not self._state_changed: # pragma: no cover self._condition.wait(self._poll_interval) # Clone the state object so we can release the lock. ret = self._state.clone() self._state.new_config = None self._state_changed = False return ret @contextlib.contextmanager def _change_state(self): with self._condition: yield self._state_changed = True self._condition.notify() def run(self): while True: try: state = self._wait() if state.exit: return elif state.new_config is not None: # Stop the service if it's currently running. self._service.stop() # Recreate it with the new config and start it. self.reconfigs.increment(fields={'service': self._service.name}) self._service = service.Service(self._state_directory, state.new_config, self._cloudtail) self._service.start() self._started = True elif state.should_run == False: # Ensure the service is stopped. self._service.stop() self._started = False elif state.should_run == True: try: proc_state = self._service.get_running_process_state() except service.UnexpectedProcessStateError: self.failures.increment(fields={'service': self._service.name}) logging.exception('Unexpected error getting state for service %s', self._service.name) except service.ProcessNotRunning as ex: if self._started: # We started it last time but it's not running any more. self.failures.increment(fields={'service': self._service.name}) LOGGER.warning('Service %s failed (%r), restarting', self._service.name, ex) else: # We're about to start it for the first time. LOGGER.info('Starting service %s for the first time (%r)', self._service.name, ex) else: if self._service.has_version_changed(proc_state): self.upgrades.increment(fields={'service': self._service.name}) LOGGER.info('Service %s has a new package version, restarting', self._service.name) self._service.stop() elif self._service.has_cmd_changed(proc_state): self.reconfigs.increment(fields={'service': self._service.name}) LOGGER.info( 'Service %s has new command: was %s, restarting with %s', self._service.name, proc_state.cmd, self._service.cmd) self._service.stop() # Ensure the service is running. self._service.start() self._started = True except Exception: LOGGER.exception('Service thread failed for service %s', self._service.name) def start_service(self): with self._change_state(): self._state.should_run = True def stop_service(self): with self._change_state(): self._state.should_run = False def stop(self, join=True): with self._change_state(): self._state.exit = True if join: # pragma: no cover self.join() def restart_with_new_config(self, new_config): with self._change_state(): self._state.new_config = new_config self._state.should_run = True
class SpamService(object): """The persistence layer for spam reports.""" issue_actions = ts_mon.CounterMetric( 'monorail/spam_svc/issue', 'Count of things that happen to issues.', [ts_mon.StringField('type')]) comment_actions = ts_mon.CounterMetric( 'monorail/spam_svc/comment', 'Count of things that happen to comments.', [ts_mon.StringField('type')]) prediction_api_failures = ts_mon.CounterMetric( 'mononrail/spam_svc/prediction_api_failure', 'Failures calling the prediction API', None) def __init__(self): self.report_tbl = sql.SQLTableManager(SPAMREPORT_TABLE_NAME) self.verdict_tbl = sql.SQLTableManager(SPAMVERDICT_TABLE_NAME) self.issue_tbl = sql.SQLTableManager(ISSUE_TABLE) self.prediction_service = None try: credentials = GoogleCredentials.get_application_default() self.prediction_service = build('prediction', 'v1.6', http=httplib2.Http(), credentials=credentials) except (Oauth2ClientError, ApiClientError): logging.error("Error getting GoogleCredentials: %s" % sys.exc_info()[0]) def LookupIssueFlaggers(self, cnxn, issue_id): """Returns users who've reported the issue or its comments as spam. Returns a tuple. First element is a list of users who flagged the issue; second element is a dictionary of comment id to a list of users who flagged that comment. """ rows = self.report_tbl.Select(cnxn, cols=['user_id', 'comment_id'], issue_id=issue_id) issue_reporters = [] comment_reporters = collections.defaultdict(list) for row in rows: if row[1]: comment_reporters[row[1]].append(row[0]) else: issue_reporters.append(row[0]) return issue_reporters, comment_reporters def LookupIssueFlagCounts(self, cnxn, issue_ids): """Returns a map of issue_id to flag counts""" rows = self.report_tbl.Select(cnxn, cols=['issue_id', 'COUNT(*)'], issue_id=issue_ids, group_by=['issue_id']) counts = {} for row in rows: counts[long(row[0])] = row[1] return counts def LookupIssueVerdicts(self, cnxn, issue_ids): """Returns a map of issue_id to most recent spam verdicts""" rows = self.verdict_tbl.Select( cnxn, cols=['issue_id', 'reason', 'MAX(created)'], issue_id=issue_ids, group_by=['issue_id']) counts = {} for row in rows: counts[long(row[0])] = row[1] return counts def LookupIssueVerdictHistory(self, cnxn, issue_ids): """Returns a map of issue_id to most recent spam verdicts""" rows = self.verdict_tbl.Select(cnxn, cols=[ 'issue_id', 'reason', 'created', 'is_spam', 'classifier_confidence', 'user_id', 'overruled' ], issue_id=issue_ids, order_by=[('issue_id', []), ('created', [])]) # TODO: group by issue_id, make class instead of dict for verdict. verdicts = [] for row in rows: verdicts.append({ 'issue_id': row[0], 'reason': row[1], 'created': row[2], 'is_spam': row[3], 'classifier_confidence': row[4], 'user_id': row[5], 'overruled': row[6], }) return verdicts def LookupCommentVerdictHistory(self, cnxn, comment_ids): """Returns a map of issue_id to most recent spam verdicts""" rows = self.verdict_tbl.Select(cnxn, cols=[ 'comment_id', 'reason', 'created', 'is_spam', 'classifier_confidence', 'user_id', 'overruled' ], comment_id=comment_ids, order_by=[('comment_id', []), ('created', [])]) # TODO: group by comment_id, make class instead of dict for verdict. verdicts = [] for row in rows: verdicts.append({ 'comment_id': row[0], 'reason': row[1], 'created': row[2], 'is_spam': row[3], 'classifier_confidence': row[4], 'user_id': row[5], 'overruled': row[6], }) return verdicts def FlagIssues(self, cnxn, issue_service, issues, reporting_user_id, flagged_spam): """Creates or deletes a spam report on an issue.""" verdict_updates = [] if flagged_spam: rows = [(issue.issue_id, issue.reporter_id, reporting_user_id) for issue in issues] self.report_tbl.InsertRows(cnxn, SPAMREPORT_ISSUE_COLS, rows, ignore=True) else: issue_ids = [issue.issue_id for issue in issues] self.report_tbl.Delete(cnxn, issue_id=issue_ids, user_id=reporting_user_id, comment_id=None) project_id = issues[0].project_id # Now record new verdicts and update issue.is_spam, if they've changed. ids = [issue.issue_id for issue in issues] counts = self.LookupIssueFlagCounts(cnxn, ids) previous_verdicts = self.LookupIssueVerdicts(cnxn, ids) for issue_id in counts: # If the flag counts changed enough to toggle the is_spam bit, need to # record a new verdict and update the Issue. if ((flagged_spam and counts[issue_id] >= settings.spam_flag_thresh or not flagged_spam and counts[issue_id] < settings.spam_flag_thresh) and (previous_verdicts[issue_id] != REASON_MANUAL if issue_id in previous_verdicts else True)): verdict_updates.append(issue_id) if len(verdict_updates) == 0: return # Some of the issues may have exceed the flag threshold, so issue verdicts # and mark as spam in those cases. rows = [(issue_id, flagged_spam, REASON_THRESHOLD, project_id) for issue_id in verdict_updates] self.verdict_tbl.InsertRows(cnxn, THRESHVERDICT_ISSUE_COLS, rows, ignore=True) update_issues = [] for issue in issues: if issue.issue_id in verdict_updates: issue.is_spam = flagged_spam update_issues.append(issue) if flagged_spam: self.issue_actions.increment_by(len(update_issues), {'type': 'flag'}) issue_service.UpdateIssues(cnxn, update_issues, update_cols=['is_spam']) def FlagComment(self, cnxn, issue_id, comment_id, reported_user_id, reporting_user_id, flagged_spam): """Creates or deletes a spam report on a comment.""" # TODO(seanmccullough): Bulk comment flagging? There's no UI for that. if flagged_spam: self.report_tbl.InsertRow(cnxn, ignore=True, issue_id=issue_id, comment_id=comment_id, reported_user_id=reported_user_id, user_id=reporting_user_id) self.comment_actions.increment({'type': 'flag'}) else: self.report_tbl.Delete(cnxn, issue_id=issue_id, comment_id=comment_id, user_id=reporting_user_id) def RecordClassifierIssueVerdict(self, cnxn, issue, is_spam, confidence, fail_open): reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER self.verdict_tbl.InsertRow(cnxn, issue_id=issue.issue_id, is_spam=is_spam, reason=reason, classifier_confidence=confidence, project_id=issue.project_id) if is_spam: self.issue_actions.increment({'type': 'classifier'}) # This is called at issue creation time, so there's nothing else to do here. def RecordManualIssueVerdicts(self, cnxn, issue_service, issues, user_id, is_spam): rows = [(user_id, issue.issue_id, is_spam, REASON_MANUAL, issue.project_id) for issue in issues] issue_ids = [issue.issue_id for issue in issues] # Overrule all previous verdicts. self.verdict_tbl.Update( cnxn, {'overruled': True}, [('issue_id IN (%s)' % sql.PlaceHolders(issue_ids), issue_ids)], commit=False) self.verdict_tbl.InsertRows(cnxn, MANUALVERDICT_ISSUE_COLS, rows, ignore=True) for issue in issues: issue.is_spam = is_spam if is_spam: self.issue_actions.increment_by(len(issues), {'type': 'manual'}) else: issue_service.AllocateNewLocalIDs(cnxn, issues) # This will commit the transaction. issue_service.UpdateIssues(cnxn, issues, update_cols=['is_spam']) def RecordManualCommentVerdict(self, cnxn, issue_service, user_service, comment_id, sequence_num, user_id, is_spam): # TODO(seanmccullough): Bulk comment verdicts? There's no UI for that. self.verdict_tbl.InsertRow(cnxn, ignore=True, user_id=user_id, comment_id=comment_id, is_spam=is_spam, reason=REASON_MANUAL) comment = issue_service.GetComment(cnxn, comment_id) comment.is_spam = is_spam issue = issue_service.GetIssue(cnxn, comment.issue_id) issue_service.SoftDeleteComment(cnxn, comment.project_id, issue.local_id, sequence_num, user_id, user_service, is_spam, True, is_spam) if is_spam: self.comment_actions.increment({'type': 'manual'}) def RecordClassifierCommentVerdict(self, cnxn, comment, is_spam, confidence, fail_open): reason = REASON_FAIL_OPEN if fail_open else REASON_CLASSIFIER self.verdict_tbl.InsertRow(cnxn, comment_id=comment.id, is_spam=is_spam, reason=reason, classifier_confidence=confidence, project_id=comment.project_id) if is_spam: self.comment_actions.increment({'type': 'classifier'}) def _predict(self, body): return self.prediction_service.trainedmodels().predict( project=settings.classifier_project_id, id=settings.classifier_model_id, body=body).execute() def _IsExempt(self, author, is_project_member): """Return True if the user is exempt from spam checking.""" if author.email is not None and author.email.endswith( settings.spam_whitelisted_suffixes): logging.info('%s whitelisted from spam filtering', author.email) return True if author.ignore_action_limits: logging.info('%s trusted not to spam', author.email) return True if is_project_member: logging.info('%s is a project member, assuming ham', author.email) return True return False def ClassifyIssue(self, issue, firstComment, reporter, is_project_member): """Classify an issue as either spam or ham. Args: issue: the Issue. firstComment: the first Comment on issue. reporter: User PB for the Issue reporter. is_project_member: True if reporter is a member of issue's project. Returns a JSON dict of classifier prediction results from the Cloud Prediction API. """ # Fail-safe: not spam. result = { 'outputLabel': 'ham', 'outputMulti': [{ 'label': 'ham', 'score': '1.0' }], 'failed_open': False } if self._IsExempt(reporter, is_project_member): return result if not self.prediction_service: logging.error("prediction_service not initialized.") return result features = spam_helpers.GenerateFeatures(issue.summary, firstComment.content, settings.spam_feature_hashes) remaining_retries = 3 while remaining_retries > 0: try: result = self._predict({'input': { 'csvInstance': features, }}) result['failed_open'] = False return result except Exception as ex: remaining_retries = remaining_retries - 1 self.prediction_api_failures.increment() logging.error('Error calling prediction API: %s' % ex) result['failed_open'] = True return result def ClassifyComment(self, comment_content, commenter, is_project_member=True): """Classify a comment as either spam or ham. Args: comment: the comment text. commenter: User PB for the user who authored the comment. Returns a JSON dict of classifier prediction results from the Cloud Prediction API. """ # Fail-safe: not spam. result = { 'outputLabel': 'ham', 'outputMulti': [{ 'label': 'ham', 'score': '1.0' }], 'failed_open': False } if self._IsExempt(commenter, is_project_member): return result if not self.prediction_service: logging.error("prediction_service not initialized.") self.prediction_api_failures.increment() result['failed_open'] = True return result features = spam_helpers.GenerateFeatures('', comment_content, settings.spam_feature_hashes) remaining_retries = 3 while remaining_retries > 0: try: result = self._predict({'input': { 'csvInstance': features, }}) result['failed_open'] = False return result except Exception as ex: remaining_retries = remaining_retries - 1 self.prediction_api_failures.increment() logging.error('Error calling prediction API: %s' % ex) result['failed_open'] = True return result def GetIssueClassifierQueue(self, cnxn, _issue_service, project_id, offset=0, limit=10): """Returns list of recent issues with spam verdicts, ranked in ascending order of confidence (so uncertain items are first). """ # TODO(seanmccullough): Optimize pagination. This query probably gets # slower as the number of SpamVerdicts grows, regardless of offset # and limit values used here. Using offset,limit in general may not # be the best way to do this. issue_results = self.verdict_tbl.Select( cnxn, cols=[ 'issue_id', 'is_spam', 'reason', 'classifier_confidence', 'created' ], where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('issue_id IS NOT NULL', []), ], order_by=[ ('classifier_confidence ASC', []), ('created ASC', []), ], group_by=['issue_id'], offset=offset, limit=limit, ) ret = [] for row in issue_results: ret.append( ModerationItem( issue_id=long(row[0]), is_spam=row[1] == 1, reason=row[2], classifier_confidence=row[3], verdict_time='%s' % row[4], )) count = self.verdict_tbl.SelectValue( cnxn, col='COUNT(*)', where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('issue_id IS NOT NULL', []), ]) return ret, count def GetIssueFlagQueue(self, cnxn, _issue_service, project_id, offset=0, limit=10): """Returns list of recent issues that have been flagged by users""" issue_flags = self.report_tbl.Select( cnxn, cols=[ "Issue.project_id", "Report.issue_id", "count(*) as count", "max(Report.created) as latest", "count(distinct Report.user_id) as users" ], left_joins=["Issue ON Issue.id = Report.issue_id"], where=[('Report.issue_id IS NOT NULL', []), ("Issue.project_id == %v", [project_id])], order_by=[('count DESC', [])], group_by=['Report.issue_id'], offset=offset, limit=limit) ret = [] for row in issue_flags: ret.append( ModerationItem( project_id=row[0], issue_id=row[1], count=row[2], latest_report=row[3], num_users=row[4], )) count = self.verdict_tbl.SelectValue( cnxn, col='COUNT(DISTINCT Report.issue_id)', where=[('Issue.project_id = %s', [project_id])], left_joins=["Issue ON Issue.id = SpamReport.issue_id"]) return ret, count def GetCommentClassifierQueue(self, cnxn, _issue_service, project_id, offset=0, limit=10): """Returns list of recent comments with spam verdicts, ranked in ascending order of confidence (so uncertain items are first). """ # TODO(seanmccullough): Optimize pagination. This query probably gets # slower as the number of SpamVerdicts grows, regardless of offset # and limit values used here. Using offset,limit in general may not # be the best way to do this. comment_results = self.verdict_tbl.Select( cnxn, cols=[ 'issue_id', 'is_spam', 'reason', 'classifier_confidence', 'created' ], where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('comment_id IS NOT NULL', []), ], order_by=[ ('classifier_confidence ASC', []), ('created ASC', []), ], group_by=['comment_id'], offset=offset, limit=limit, ) ret = [] for row in comment_results: ret.append( ModerationItem( comment_id=long(row[0]), is_spam=row[1] == 1, reason=row[2], classifier_confidence=row[3], verdict_time='%s' % row[4], )) count = self.verdict_tbl.SelectValue( cnxn, col='COUNT(*)', where=[ ('project_id = %s', [project_id]), ('classifier_confidence <= %s', [settings.classifier_moderation_thresh]), ('overruled = %s', [False]), ('comment_id IS NOT NULL', []), ]) return ret, count def GetTrainingIssues(self, cnxn, issue_service, since, offset=0, limit=100): """Returns list of recent issues with human-labeled spam/ham verdicts. """ # get all of the manual verdicts in the past day. results = self.verdict_tbl.Select( cnxn, cols=['issue_id'], where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('issue_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ], offset=offset, limit=limit, ) issue_ids = [long(row[0]) for row in results if row[0]] issues = issue_service.GetIssues(cnxn, issue_ids) comments = issue_service.GetCommentsForIssues(cnxn, issue_ids) first_comments = {} for issue in issues: first_comments[issue.issue_id] = ( comments[issue.issue_id][0].content if issue.issue_id in comments else "[Empty]") count = self.verdict_tbl.SelectValue(cnxn, col='COUNT(*)', where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('issue_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ]) return issues, first_comments, count def GetTrainingComments(self, cnxn, issue_service, since, offset=0, limit=100): """Returns list of recent comments with human-labeled spam/ham verdicts. """ # get all of the manual verdicts in the past day. results = self.verdict_tbl.Select( cnxn, cols=['comment_id'], where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('comment_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ], offset=offset, limit=limit, ) comment_ids = [long(row[0]) for row in results if row[0]] # Don't care about sequence numbers in this context yet. comments = issue_service.GetCommentsByID(cnxn, comment_ids, defaultdict(int)) count = self.verdict_tbl.SelectValue(cnxn, col='COUNT(*)', where=[ ('overruled = %s', [False]), ('reason = %s', ['manual']), ('comment_id IS NOT NULL', []), ('created > %s', [since.isoformat()]), ]) return comments, count
import re import time from logging.handlers import TimedRotatingFileHandler import buildbot.status.results from buildbot.status.base import StatusReceiverMultiService from twisted.python import log as twisted_log from common import chromium_utils from infra_libs import ts_mon step_field_spec = [ ts_mon.StringField('builder'), ts_mon.StringField('master'), ts_mon.StringField('project_id'), ts_mon.StringField('result'), ts_mon.StringField('slave'), ts_mon.StringField('step_name'), ts_mon.StringField('subproject_tag'), ] step_durations = ts_mon.CumulativeDistributionMetric( 'buildbot/master/builders/steps/durations', 'Time (in seconds) from step start to step end', step_field_spec, units=ts_mon.MetricsDataUnits.SECONDS, # Use fixed-width bucketer up to 2.7 hours with 10-second precision. bucketer=ts_mon.FixedWidthBucketer(10, 1000))