class SectionTree(ndb.Model):
tree = ndb.PickleProperty('t')
# tree = [
# {
# 'key_us' : 'urlsafe_key',
# 'name' : 'What we do',
# 'pages_names': OrderedDict({
# 'page_key_us_1': {
# 'en' : 'How it works?',
# 'it' : 'Come funziona?'
# },
# 'page_key_us_2': {
# 'en': 'How it works2?',
# 'it': 'Come funziona2?'
# },
# }),
# 'subs': [
# {
# 'key_us' : 'urlsafe_key',
# 'name' : 'Specialized in',
# 'pages_names': OrderedDict({}),
# 'subs': {...}
# },
# {...}
# },
# { ... }
# ]
ins_timestamp = ndb.DateTimeProperty('i_ts',
auto_now_add=True,
indexed=False)
upd_timestamp = ndb.DateTimeProperty('u_ts', auto_now=True, indexed=False)
def __repr__(self):
return "<SectionTree %s>" % self.key
def get_id(self):
return self.key.id()
@classmethod
def get_by_key(cls, k):
return ndb.Key(urlsafe=k).get()
class CustomEventCondition(ndb.Model):
"""Datastore model representing a condition from which to build an ndb.Query.
Attributes:
name: str, names a property on the NDB model.
opsymbol: str, specifies a comparison operator.
value: pickle, contains the actual value, which could be anything.
"""
name = ndb.StringProperty(required=True)
opsymbol = ndb.StringProperty(required=True)
value = ndb.PickleProperty(required=True)
def get_filter(self):
return ndb.query.FilterNode(self.name, self.opsymbol,
_apply_timedelta(self.value))
def match(self, entity):
"""Determines whether an entity matches the condition.
This is for manually running a condition on an entity outside an NDB query.
Args:
entity: a Datastore entity on which to test a match of this condition.
Returns:
True if the entity matches, else False.
"""
entity_value = getattr(entity, self.name)
condition_value = _apply_timedelta(self.value)
if self.opsymbol in ['<', '<=', '>', '>='] and entity_value is None:
# Unset properties are set to None, and so are treated as 0 in
# comparisons; we prefer to filter.
return False
elif ((self.opsymbol == '<' and entity_value < condition_value)
or (self.opsymbol == '<=' and entity_value <= condition_value)
or (self.opsymbol == '==' and entity_value == condition_value)
or (self.opsymbol == '!=' and entity_value != condition_value)
or (self.opsymbol == '>' and entity_value > condition_value)
or (self.opsymbol == '>=' and entity_value >= condition_value)):
return True
else:
return False
class UserProfile(ndb.Expando):
"""
``ndb.Expando`` is used to store the user_info object as well as
any additional information specific to a strategy.
"""
_default_indexed = False
user_info = ndb.JsonProperty(indexed=False, compressed=True)
credentials = ndb.PickleProperty(indexed=False)
@classmethod
def get_or_create(cls, auth_id, user_info, **kwargs):
"""
"""
profile = cls.get_by_id(auth_id)
if profile is None:
profile = cls(id=auth_id)
profile.user_info = user_info
profile.populate(**kwargs)
profile.put()
return profile
class MultiplayerOptions(ndb.Model):
ndb_mode = msgprop.EnumProperty(
NDB_MultiGameType, default=NDB_MultiGameType.SIMUSOLO)
ndb_shared = msgprop.EnumProperty(NDB_ShareType, repeated=True)
def get_mode(self): return MultiplayerGameType.from_ndb(self.ndb_mode)
def set_mode(self, mode): self.ndb_mode = mode.to_ndb()
def get_shared(self): return [ShareType.from_ndb(ndb_st)
for ndb_st in self.ndb_shared]
def set_shared(self, shared): self.ndb_shared = [s.to_ndb() for s in shared]
mode = property(get_mode, set_mode)
shared = property(get_shared, set_shared)
enabled = ndb.BooleanProperty(default=False)
cloned = ndb.BooleanProperty(default=True)
hints = ndb.BooleanProperty(default=True)
teams = ndb.PickleProperty(default={})
@staticmethod
def from_url(qparams):
opts = MultiplayerOptions()
opts.enabled = int(qparams.get("players", 1)) > 1
if opts.enabled:
opts.mode = MultiplayerGameType(qparams.get("sync_mode", "None"))
opts.cloned = qparams.get("sync_gen") != "disjoint"
opts.hints = bool(opts.cloned and qparams.get("sync_hints"))
opts.shared = enums_from_strlist(ShareType, qparams.getall("sync_shared"))
teamsRaw = qparams.get("teams")
if teamsRaw and opts.mode == MultiplayerGameType.SHARED and opts.cloned:
cnt = 1
teams = {}
for teamRaw in teamsRaw.split("|"):
teams[cnt] = [int(p) for p in teamRaw.split(",")]
cnt += 1
opts.teams = teams
return opts
class Talk(ndb.Model):
talk_title = ndb.StringProperty()
details = ndb.PickleProperty()
created = ndb.DateTimeProperty(auto_now_add=True)
directory_listing = ndb.StringProperty()
def __init__(self, *args, **kwargs):
super(Talk, self).__init__(*args, **kwargs)
self.talk_title = ""
self.directory_listing = "Listed"
self.details = {}
def field(self, f):
if (self.details.has_key(f)):
return self.details[f]
return ""
def field_ascii(self, f):
return self.field(f).encode('ascii', 'ignore')
def set_field(self, field, value):
self.details[field] = value
@property
def title(self):
return self.talk_title
@title.setter
def title(self, t):
self.talk_title = t
def is_listed(self):
return "Listed" == self.directory_listing
def hide_listing(self):
self.directory_listing = "Not listed"
def show_listing(self):
self.directory_listing = "Listed"
class ProcessedNotifications(ndb.Model):
"""Track if we processed the object notifications for a project today."""
date_project_dict = ndb.PickleProperty()
@classmethod
def getInstance(cls):
"""Returns the single instance of this entity."""
instance_key = ndb.Key(ProcessedNotifications,
'ProcessedNotifications')
instance = instance_key.get()
if instance is None:
instance = ProcessedNotifications(key=instance_key)
instance.put()
return instance
@classmethod
@ndb.transactional
def processForToday(cls, project):
"""Mark a project as having been processed (alerts/emails sent) today.
Args:
project: Name of project to process. Returns: True if we haven't and
should process. This fuction modifies the map and assumes an email
will be sent.
Returns:
True if the project was not processed today, False otherwise.
"""
processed = ProcessedNotifications.getInstance()
if processed.date_project_dict is None:
processed.date_project_dict = {}
today = date.today()
if today not in processed.date_project_dict:
processed.date_project_dict.clear()
processed.date_project_dict[today] = []
if project in processed.date_project_dict[today]:
return False
processed.date_project_dict[today].append(project)
processed.put()
return True
class KVS(ndb.Model):
key_str = ndb.StringProperty()
value = ndb.PickleProperty()
value_str = ndb.StringProperty()
@classmethod
def get(cls, key_str):
key = str(ndb.Key("KVS", key_str).id())
res = KVS.get_by_id(key, use_cache=False, use_memcache=False)
if res is not None:
return res.value
@classmethod
def put(cls, key_str, value):
key = str(ndb.Key("KVS", key_str).id())
kvs_pair = KVS.get_or_insert(key,
key_str=key_str,
value=value,
value_str=str(value))
kvs_pair.value = value
kvs_pair.value_str = str(value)
ndb.Model.put(kvs_pair)
class Game(ndb.Model):
"""A Kind for Game, instantiate with Player as parent"""
name = ndb.StringProperty(required=True)
seatsAvailable = ndb.IntegerProperty(default=2)
playerOne = ndb.StringProperty()
playerTwo = ndb.StringProperty()
board = ndb.PickleProperty()
gameCurrentMove = ndb.IntegerProperty(default=0)
nextPlayer = ndb.StringProperty()
gameOver = ndb.BooleanProperty(default=False)
gameWinner = ndb.StringProperty()
@property
def _copyGameToForm(self):
"""Copy relevant fields from Game to GameForm."""
gf = GameForm()
for field in gf.all_fields():
if hasattr(self, field.name):
setattr(gf, field.name, getattr(self, field.name))
elif field.name == "websafeKey":
setattr(gf, field.name, self.key.urlsafe())
elif field.name == "gameBoard":
setattr(gf, field.name, ' '.join(getattr(self, 'board')))
gf.check_initialized()
return gf
@property
def _isWon(self):
"""when the tic-tac-toe game comes to a winning connection"""
return (self.board[0] == self.board[4] == self.board[8] != ''
or self.board[2] == self.board[4] == self.board[6] != ''
or self.board[0] == self.board[1] == self.board[2] != ''
or self.board[3] == self.board[4] == self.board[5] != ''
or self.board[6] == self.board[7] == self.board[8] != ''
or self.board[0] == self.board[3] == self.board[6] != ''
or self.board[1] == self.board[4] == self.board[7] != ''
or self.board[2] == self.board[5] == self.board[8] != '')
class BasicModel(ndb.Model):
"""Common data/operations for all storage models
"""
int_field = ndb.IntegerProperty()
float_field = ndb.FloatProperty()
boolean_field = ndb.BooleanProperty()
string_field = ndb.StringProperty()
text_field = ndb.TextProperty()
blob_field = ndb.BlobProperty()
datetime_field = ndb.DateTimeProperty()
date_field = ndb.DateProperty()
time_field = ndb.TimeProperty()
geopt_field = ndb.GeoPtProperty()
key_field = ndb.KeyProperty()
blobkey_field = ndb.BlobKeyProperty()
user_field = ndb.UserProperty()
single_structured_field = ndb.StructuredProperty(ChildModel)
repeated_structured_field = ndb.StructuredProperty(ChildModel,
repeated=True)
json_field = ndb.JsonProperty()
pickle_field = ndb.PickleProperty()
computed_field = ndb.ComputedProperty(
lambda self: self.string_field.upper())
class Job(ndb.Model):
"""A Pinpoint job."""
state = ndb.PickleProperty(required=True, compressed=True)
#####
# Job arguments passed in through the API.
#####
# Request parameters.
arguments = ndb.JsonProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
comparison_mode = ndb.StringProperty()
# The Gerrit server url and change id of the code review to update upon
# completion.
gerrit_server = ndb.StringProperty()
gerrit_change_id = ndb.StringProperty()
# User-provided name of the job.
name = ndb.StringProperty()
tags = ndb.JsonProperty()
# Email of the job creator.
user = ndb.StringProperty()
#####
# Job state generated by running the job.
#####
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
# This differs from "created" since there may be a lag between the time it
# was queued and when the scheduler actually starts the job.
started_time = ndb.DateTimeProperty(required=False)
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
started = ndb.BooleanProperty(default=True)
completed = ndb.ComputedProperty(
lambda self: self.started and not self.task)
failed = ndb.ComputedProperty(
lambda self: bool(self.exception_details_dict))
running = ndb.ComputedProperty(
lambda self: self.started and not self.cancelled and self.task and len(
self.task) > 0)
cancelled = ndb.BooleanProperty(default=False)
cancel_reason = ndb.TextProperty()
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
exception_details = ndb.JsonProperty()
difference_count = ndb.IntegerProperty()
retry_count = ndb.IntegerProperty(default=0)
# We expose the configuration as a first-class property of the Job.
configuration = ndb.ComputedProperty(
lambda self: self.arguments.get('configuration'))
# TODO(simonhatch): After migrating all Pinpoint entities, this can be
# removed.
# crbug.com/971370
@classmethod
def _post_get_hook(cls, key, future): # pylint: disable=unused-argument
e = future.get_result()
if not e:
return
if not getattr(e, 'exception_details'):
e.exception_details = e.exception_details_dict
# TODO(simonhatch): After migrating all Pinpoint entities, this can be
# removed.
# crbug.com/971370
@property
def exception_details_dict(self):
if hasattr(self, 'exception_details'):
if self.exception_details:
return self.exception_details
if hasattr(self, 'exception'):
exc = self.exception
if exc:
return {'message': exc.splitlines()[-1], 'traceback': exc}
return None
@classmethod
def New(cls,
quests,
changes,
arguments=None,
bug_id=None,
comparison_mode=None,
comparison_magnitude=None,
gerrit_server=None,
gerrit_change_id=None,
name=None,
pin=None,
tags=None,
user=None):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: Either 'functional' or 'performance', which the Job uses
to figure out whether to perform a functional or performance bisect.
If None, the Job will not automatically add any Attempts or Changes.
comparison_magnitude: The estimated size of the regression or improvement
to look for. Smaller magnitudes require more repeats.
gerrit_server: Server of the Gerrit code review to update with job
results.
gerrit_change_id: Change id of the Gerrit code review to update with job
results.
name: The user-provided name of the Job.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
Returns:
A Job object.
"""
state = job_state.JobState(quests,
comparison_mode=comparison_mode,
comparison_magnitude=comparison_magnitude,
pin=pin)
job = cls(state=state,
arguments=arguments or {},
bug_id=bug_id,
comparison_mode=comparison_mode,
gerrit_server=gerrit_server,
gerrit_change_id=gerrit_change_id,
name=name,
tags=tags,
user=user,
started=False,
cancelled=False)
for c in changes:
job.AddChange(c)
job.put()
# At this point we already have an ID, so we should go through each of the
# quests associated with the state, and provide the Job ID through a common
# API.
job.state.PropagateJob(job)
job.put()
return job
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.failed:
return 'Failed'
if self.cancelled:
return 'Cancelled'
if self.completed:
return 'Completed'
if self.running:
return 'Running'
# By default, we assume that the Job is queued.
return 'Queued'
@property
def url(self):
host = os.environ['HTTP_HOST']
# TODO(crbug.com/939723): Remove this workaround when not needed.
if host == 'pinpoint.chromeperf.appspot.com':
host = 'pinpoint-dot-chromeperf.appspot.com'
return 'https://%s/job/%s' % (host, self.job_id)
@property
def results_url(self):
if not self.task:
url = results2.GetCachedResults2(self)
if url:
return url
# Point to the default status page if no results are available.
return '/results2/%s' % self.job_id
@property
def auto_name(self):
if self.name:
return self.name
if self.comparison_mode == job_state.FUNCTIONAL:
name = 'Functional bisect'
elif self.comparison_mode == job_state.PERFORMANCE:
name = 'Performance bisect'
else:
name = 'Try job'
if self.configuration:
name += ' on ' + self.configuration
if 'benchmark' in self.arguments:
name += '/' + self.arguments['benchmark']
return name
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
"""Starts the Job and updates it in the Datastore.
This method is designed to return fast, so that Job creation is responsive
to the user. It schedules the Job on the task queue without running
anything. It also posts a bug comment, and updates the Datastore.
"""
self._Schedule()
self.started = True
self.started_time = datetime.datetime.now()
self.put()
title = _ROUND_PUSHPIN + ' Pinpoint job started.'
comment = '\n'.join((title, self.url))
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
send_email=False,
_retry_options=RETRY_OPTIONS)
def _IsTryJob(self):
return not self.comparison_mode or self.comparison_mode == job_state.TRY
def _Complete(self):
logging.debug('Job [%s]: Completed', self.job_id)
if not self._IsTryJob():
self.difference_count = len(self.state.Differences())
try:
results2.ScheduleResults2Generation(self)
except taskqueue.Error as e:
logging.debug('Failed ScheduleResults2Generation: %s', str(e))
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
scheduler.Complete(self)
def _FormatAndPostBugCommentOnComplete(self):
if self._IsTryJob():
# There is no comparison metric.
title = "<b>%s Job complete. See results below.</b>" % _ROUND_PUSHPIN
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
'\n'.join((title, self.url)),
_retry_options=RETRY_OPTIONS)
return
# There is a comparison metric.
differences = self.state.Differences()
if not differences:
title = "<b>%s Couldn't reproduce a difference.</b>" % _ROUND_PUSHPIN
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
'\n'.join((title, self.url)),
_retry_options=RETRY_OPTIONS)
return
difference_details = []
authors_with_deltas = {}
commit_infos = []
for change_a, change_b in differences:
if change_b.patch:
commit_info = change_b.patch.AsDict()
else:
commit_info = change_b.last_commit.AsDict()
values_a = self.state.ResultValues(change_a)
values_b = self.state.ResultValues(change_b)
difference = _FormatDifferenceForBug(commit_info, values_a,
values_b, self.state.metric)
difference_details.append(difference)
commit_infos.append(commit_info)
if values_a and values_b:
authors_with_deltas[commit_info['author']] = job_state.Mean(
values_b) - job_state.Mean(values_a)
deferred.defer(_UpdatePostAndMergeDeferred,
difference_details,
commit_infos,
authors_with_deltas,
self.bug_id,
self.tags,
self.url,
_retry_options=RETRY_OPTIONS)
def _UpdateGerritIfNeeded(self):
if self.gerrit_server and self.gerrit_change_id:
deferred.defer(_UpdateGerritDeferred,
self.gerrit_server,
self.gerrit_change_id,
'%s Job complete.\n\nSee results at: %s' %
(_ROUND_PUSHPIN, self.url),
_retry_options=RETRY_OPTIONS)
def Fail(self, exception=None):
tb = traceback.format_exc() or ''
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
exc_message = ''
if exception:
exc_message = exception
elif exc_info[1]:
exc_message = sys.exc_info()[1].message
self.exception_details = {
'message': exc_message,
'traceback': tb,
}
self.task = None
comment = '\n'.join((title, self.url, '', exc_message))
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
_retry_options=RETRY_OPTIONS)
scheduler.Complete(self)
def _Schedule(self, countdown=_TASK_INTERVAL):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
name=task_name,
countdown=countdown)
except (apiproxy_errors.DeadlineExceededError,
taskqueue.TransientError):
raise errors.RecoverableError()
self.task = task.name
def _MaybeScheduleRetry(self):
if not hasattr(self, 'retry_count') or self.retry_count is None:
self.retry_count = 0
if self.retry_count >= _MAX_RECOVERABLE_RETRIES:
return False
self.retry_count += 1
# Back off exponentially
self._Schedule(countdown=_TASK_INTERVAL * (2**self.retry_count))
return True
def Run(self):
"""Runs this Job.
Loops through all Attempts and checks the status of each one, kicking off
tasks as needed. Does not block to wait for all tasks to finish. Also
compares adjacent Changes' results and adds any additional Attempts or
Changes as needed. If there are any incomplete tasks, schedules another
Run() call on the task queue.
"""
self.exception_details = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if not self._IsTryJob():
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
self.retry_count = 0
except errors.RecoverableError:
try:
if not self._MaybeScheduleRetry():
self.Fail(errors.RETRY_LIMIT)
except errors.RecoverableError:
self.Fail(errors.RETRY_FAILED)
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
if self.completed:
timing_record.RecordJobTiming(self)
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def AsDict(self, options=None):
d = {
'job_id': self.job_id,
'configuration': self.configuration,
'results_url': self.results_url,
'arguments': self.arguments,
'bug_id': self.bug_id,
'comparison_mode': self.comparison_mode,
'name': self.auto_name,
'user': self.user,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'difference_count': self.difference_count,
'exception': self.exception_details_dict,
'status': self.status,
'cancel_reason': self.cancel_reason,
}
if not options:
return d
if OPTION_STATE in options:
d.update(self.state.AsDict())
if OPTION_ESTIMATE in options and not self.started:
d.update(self._GetRunTimeEstimate())
if OPTION_TAGS in options:
d['tags'] = {'tags': self.tags}
return d
def _GetRunTimeEstimate(self):
result = timing_record.GetSimilarHistoricalTimings(self)
if not result:
return {}
timings = [t.total_seconds() for t in result.timings]
return {
'estimate': {
'timings': timings,
'tags': result.tags
},
'queue_stats': scheduler.QueueStats(self.configuration)
}
def Cancel(self, user, reason):
# We cannot cancel an already cancelled job.
if self.cancelled:
logging.warning(
'Attempted to cancel a cancelled job "%s"; user = %s, reason = %s',
self.job_id, user, reason)
raise errors.CancelError('Job already cancelled.')
if not scheduler.Cancel(self):
raise errors.CancelError('Scheduler failed to cancel job.')
self.cancelled = True
self.cancel_reason = '{}: {}'.format(user, reason)
# Remove any "task" identifiers.
self.task = None
self.put()
title = _ROUND_PUSHPIN + ' Pinpoint job cancelled.'
comment = u'{}\n{}\n\nCancelled by {}, reason given: {}'.format(
title, self.url, user, reason)
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
send_email=False,
_retry_options=RETRY_OPTIONS)
class Job(ndb.Model):
"""A Pinpoint job."""
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
updated = ndb.DateTimeProperty(required=True, auto_now=True)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
# Request parameters.
arguments = ndb.JsonProperty(required=True)
repeat_count = ndb.IntegerProperty(required=True)
# If True, the service should pick additional Changes to run (bisect).
# If False, only run the Changes explicitly added by the user.
auto_explore = ndb.BooleanProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
state = ndb.PickleProperty(required=True)
@classmethod
def New(cls,
arguments,
quests,
auto_explore,
repeat_count=_DEFAULT_REPEAT_COUNT,
bug_id=None):
repeat_count = repeat_count or _DEFAULT_REPEAT_COUNT
# Create job.
return cls(arguments=arguments,
auto_explore=auto_explore,
repeat_count=repeat_count,
bug_id=bug_id,
state=_JobState(quests, repeat_count))
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.task:
return 'Running'
if self.exception:
return 'Failed'
return 'Completed'
@property
def url(self):
return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
self.Schedule()
self._PostBugComment('started', send_email=False)
def Complete(self):
self._PostBugComment('completed', include_differences=True)
def Fail(self):
self.exception = traceback.format_exc()
self._PostBugComment('stopped with an error ' + _CRYING_CAT_FACE)
def Schedule(self):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
name=task_name,
countdown=_TASK_INTERVAL)
except apiproxy_errors.DeadlineExceededError:
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
name=task_name,
countdown=_TASK_INTERVAL)
self.task = task.name
def Run(self):
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.auto_explore:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self.Schedule()
else:
self.Complete()
except BaseException:
self.Fail()
raise
def AsDict(self, include_state=True):
d = {
'job_id': self.job_id,
'arguments': self.arguments,
'auto_explore': self.auto_explore,
'bug_id': self.bug_id,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'exception': self.exception,
'status': self.status,
}
if include_state:
d.update(self.state.AsDict())
return d
def _PostBugComment(self,
status,
include_differences=False,
send_email=True):
if not self.bug_id:
return
title = '%s Pinpoint job %s.' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
change_details = []
if include_differences:
# Include list of Changes.
differences = tuple(self.state.Differences())
if differences:
if len(differences) == 1:
change_details.append(
'<b>Found significant differences after 1 commit:</b>')
else:
change_details.append(
'<b>Found significant differences after each of %d commits:</b>'
% len(differences))
for _, change in differences:
change_details.append(_FormatChangeForBug(change))
else:
change_details.append(
"<b>Couldn't reproduce a difference.</b>")
comment = '\n\n'.join([header] + change_details)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id,
comment,
send_email=send_email)
class DemoUserInfo(ndb.Model):
asof = ndb.DateTimeProperty(auto_now_add=True)
user_id = ndb.StringProperty()
email = ndb.StringProperty()
nickname = ndb.StringProperty()
ds_key = ndb.StringProperty()
fetching = ndb.IntegerProperty(default = 0)
calculating = ndb.IntegerProperty(default = 0)
calc_done = ndb.BooleanProperty(default = False)
tweets = ndb.PickleProperty(default = [])
calc_stats = ndb.TextProperty()
def filename(self):
return 'user_id: {user_id}, email: {email}, nickname: {nickname}, asof: {asof}' \
.format(asof = self.asof.isoformat()[:19], user_id = self.user_id, email = self.email, nickname = self.nickname)
@ndb.transactional
def indicate_fetch_begun(self):
key = self.key
ent = key.get()
ent.fetching += 1
ent.put()
return ent
@ndb.transactional
def indicate_fetch_ended(self):
key = self.key
ent = key.get()
ent.fetching -= 1
ent.put()
return ent
@ndb.transactional
def indicate_calc_begun(self):
key = self.key
ent = key.get()
if 0 == ent.calculating:
ent.calc_stats = json.dumps(dict())
ent.calculating += 1
ent.calc_done = False
ent.put()
return ent
@ndb.transactional
def indicate_calc_ended(self, batch_stats):
key = self.key
ent = key.get()
ent.calculating -= 1
# logging.debug('%d', self.calculating)
if 0 == ent.calculating:
ent.calc_done = True
calc_stats = json.loads(ent.calc_stats)
for stat in batch_stats:
if stat not in calc_stats.keys():
calc_stats[stat] = 0.0
calc_stats[stat] += batch_stats[stat]
ent.calc_stats = json.dumps(calc_stats)
ent.put()
logging.info('<indicate_calc_ended %d %s %s/>', ent.calculating, ent.calc_done, ent.calc_stats)
return ent
@ndb.transactional
def extend_tweets(self, tweets):
key = self.key
ent = key.get()
ent.tweets.extend(tweets)
ent.put()
return ent
@ndb.transactional
def set_ds_key(self, ds_key):
key = self.key
ent = key.get()
if not ent.ds_key:
ent.ds_key = ds_key
ent.put()
else:
if ds_key != ent.ds_key:
logging.error('changing ds_key from %s to %s? Makes no sense!', ent.ds_key, ds_key)
return ent
@classmethod
def latest_for_user(cls, user):
if not user:
return None
dui = cls.query(cls.user_id == user.user_id()).order(-cls.asof).get()
return dui
def purge(self):
if self.ds_key:
matrix = Matrix.find(ds_key = self.ds_key)
if matrix:
matrix.purge()
class Job(ndb.Model):
"""A Pinpoint job."""
state = ndb.PickleProperty(required=True, compressed=True)
#####
# Job arguments passed in through the API.
#####
# Request parameters.
arguments = ndb.JsonProperty(required=True)
bug_id = ndb.IntegerProperty()
comparison_mode = ndb.StringProperty()
# The Gerrit server url and change id of the code review to update upon
# completion.
gerrit_server = ndb.StringProperty()
gerrit_change_id = ndb.StringProperty()
# User-provided name of the job.
name = ndb.StringProperty()
tags = ndb.JsonProperty()
# Email of the job creator.
user = ndb.StringProperty()
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
# This differs from "created" since there may be a lag between the time it
# was queued and when the scheduler actually starts the job.
started_time = ndb.DateTimeProperty(required=False)
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
started = ndb.BooleanProperty(default=True)
completed = ndb.ComputedProperty(lambda self: self.done or (
not self.use_execution_engine and self.started and not self.task))
failed = ndb.ComputedProperty(
lambda self: bool(self.exception_details_dict))
running = ndb.ComputedProperty(IsRunning)
cancelled = ndb.BooleanProperty(default=False)
cancel_reason = ndb.TextProperty()
# Because of legacy reasons, `completed` is not exactly representative of
# whether a job is "done" from the execution engine's point of view. We're
# introducing this top-level property as a definitive flag, transactionally
# enforced, to signal whether we are done executing a job.
done = ndb.BooleanProperty(default=False)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
exception_details = ndb.JsonProperty()
difference_count = ndb.IntegerProperty()
retry_count = ndb.IntegerProperty(default=0)
# We expose the configuration as a first-class property of the Job.
configuration = ndb.ComputedProperty(
lambda self: self.arguments.get('configuration'))
# Pull out the benchmark, chart, and statistic as a structured property at the
# top-level, so that we can analyse these in a structured manner.
benchmark_arguments = ndb.StructuredProperty(BenchmarkArguments)
# Indicate whether we should evaluate this job's tasks through the execution
# engine.
use_execution_engine = ndb.BooleanProperty(default=False, indexed=False)
# TODO(simonhatch): After migrating all Pinpoint entities, this can be
# removed.
# crbug.com/971370
@classmethod
def _post_get_hook(cls, key, future): # pylint: disable=unused-argument
e = future.get_result()
if not e:
return
if not getattr(e, 'exception_details'):
e.exception_details = e.exception_details_dict
# TODO(simonhatch): After migrating all Pinpoint entities, this can be
# removed.
# crbug.com/971370
@property
def exception_details_dict(self):
if hasattr(self, 'exception_details'):
if self.exception_details:
return self.exception_details
if hasattr(self, 'exception'):
exc = self.exception
if exc:
return {'message': exc.splitlines()[-1], 'traceback': exc}
return None
@classmethod
def New(cls,
quests,
changes,
arguments=None,
bug_id=None,
comparison_mode=None,
comparison_magnitude=None,
gerrit_server=None,
gerrit_change_id=None,
name=None,
pin=None,
tags=None,
user=None,
use_execution_engine=False):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: Either 'functional' or 'performance', which the Job uses
to figure out whether to perform a functional or performance bisect. If
None, the Job will not automatically add any Attempts or Changes.
comparison_magnitude: The estimated size of the regression or improvement
to look for. Smaller magnitudes require more repeats.
gerrit_server: Server of the Gerrit code review to update with job
results.
gerrit_change_id: Change id of the Gerrit code review to update with job
results.
name: The user-provided name of the Job.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
use_execution_engine: A bool indicating whether to use the experimental
execution engine. Currently defaulted to False, but will be switched to
True and eventually removed as an option later.
Returns:
A Job object.
"""
state = job_state.JobState(quests,
comparison_mode=comparison_mode,
comparison_magnitude=comparison_magnitude,
pin=pin)
args = arguments or {}
job = cls(state=state,
arguments=args,
bug_id=bug_id,
comparison_mode=comparison_mode,
gerrit_server=gerrit_server,
gerrit_change_id=gerrit_change_id,
name=name,
tags=tags,
user=user,
started=False,
cancelled=False,
use_execution_engine=use_execution_engine)
# Pull out the benchmark arguments to the top-level.
job.benchmark_arguments = BenchmarkArguments.FromArgs(args)
if use_execution_engine:
# Short-circuit the process because we don't need any further processing
# here when we're using the execution engine.
job.put()
return job
for c in changes:
job.AddChange(c)
job.put()
# At this point we already have an ID, so we should go through each of the
# quests associated with the state, and provide the Job ID through a common
# API.
job.state.PropagateJob(job)
job.put()
return job
def PostCreationUpdate(self):
title = _ROUND_PUSHPIN + ' Pinpoint job created and queued.'
pending = 0
if self.configuration:
try:
pending = scheduler.QueueStats(self.configuration).get(
'queued_jobs', 0)
except (scheduler.QueueNotFound, ndb.BadRequestError) as e:
logging.warning(
'Error encountered fetching queue named "%s": %s ',
self.configuration, e)
comment = CREATED_COMMENT_FORMAT.format(
title=title,
url=self.url,
configuration=self.configuration
if self.configuration else '(None)',
pending=pending)
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
send_email=True,
_retry_options=RETRY_OPTIONS)
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.failed:
return 'Failed'
if self.cancelled:
return 'Cancelled'
if self.completed:
return 'Completed'
if self.running:
return 'Running'
# By default, we assume that the Job is queued.
return 'Queued'
@property
def url(self):
host = os.environ['HTTP_HOST']
# TODO(crbug.com/939723): Remove this workaround when not needed.
if host == 'pinpoint.chromeperf.appspot.com':
host = 'pinpoint-dot-chromeperf.appspot.com'
return 'https://%s/job/%s' % (host, self.job_id)
@property
def results_url(self):
if not self.task:
url = results2.GetCachedResults2(self)
if url:
return url
# Point to the default status page if no results are available.
return '/results2/%s' % self.job_id
@property
def auto_name(self):
if self.name:
return self.name
if self.comparison_mode == job_state.FUNCTIONAL:
name = 'Functional bisect'
elif self.comparison_mode == job_state.PERFORMANCE:
name = 'Performance bisect'
else:
name = 'Try job'
if self.configuration:
name += ' on ' + self.configuration
if 'benchmark' in self.arguments:
name += '/' + self.arguments['benchmark']
return name
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
"""Starts the Job and updates it in the Datastore.
This method is designed to return fast, so that Job creation is responsive
to the user. It schedules the Job on the task queue without running
anything. It also posts a bug comment, and updates the Datastore.
"""
if self.use_execution_engine:
# Treat this as if it's a poll, and run the handler here.
try:
task_module.Evaluate(
self,
event_module.Event(type='initiate',
target_task=None,
payload={}),
task_evaluator.ExecutionEngine(self)),
except task_module.Error as error:
logging.error('Failed: %s', error)
self.Fail()
self.put()
return
else:
self._Schedule()
self.started = True
self.started_time = datetime.datetime.now()
self.put()
title = _ROUND_PUSHPIN + ' Pinpoint job started.'
comment = '\n'.join((title, self.url))
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
send_email=True,
_retry_options=RETRY_OPTIONS)
def _IsTryJob(self):
return not self.comparison_mode or self.comparison_mode == job_state.TRY
def _Complete(self):
logging.debug('Job [%s]: Completed', self.job_id)
if self.use_execution_engine:
scheduler.Complete(self)
# Only proceed if this thread/process is the one that successfully marked
# a job done.
if not MarkDone(self.job_id):
return
logging.debug('Job [%s]: Marked done', self.job_id)
if not self._IsTryJob() and not self.use_execution_engine:
self.difference_count = len(self.state.Differences())
try:
# TODO(dberris): Migrate results2 generation to tasks and evaluators.
results2.ScheduleResults2Generation(self)
except taskqueue.Error as e:
logging.debug('Failed ScheduleResults2Generation: %s', str(e))
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
scheduler.Complete(self)
def _FormatAndPostBugCommentOnComplete(self):
logging.debug('Processing outputs.')
if self._IsTryJob():
# There is no comparison metric.
title = '<b>%s Job complete. See results below.</b>' % _ROUND_PUSHPIN
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
'\n'.join((title, self.url)),
labels=['Pinpoint-Tryjob-Completed'],
_retry_options=RETRY_OPTIONS)
return
# There is a comparison metric.
differences = []
result_values = {}
if not self.use_execution_engine:
differences = self.state.Differences()
for change_a, change_b in differences:
result_values.setdefault(change_a,
self.state.ResultValues(change_a))
result_values.setdefault(change_b,
self.state.ResultValues(change_b))
else:
logging.debug('Execution Engine: Finding culprits.')
context = task_module.Evaluate(
self, event_module.SelectEvent(),
evaluators.Selector(
event_type='select',
include_keys={'culprits', 'change', 'result_values'}))
differences = [
(change_module.ReconstituteChange(change_a),
change_module.ReconstituteChange(change_b))
for change_a, change_b in context.get('performance_bisection',
{}).get('culprits', [])
]
result_values = {
change_module.ReconstituteChange(v.get('change')):
v.get('result_values')
for v in context.values()
if 'change' in v and 'result_values' in v
}
if not differences:
title = "<b>%s Couldn't reproduce a difference.</b>" % _ROUND_PUSHPIN
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
'\n'.join((title, self.url)),
labels=['Pinpoint-No-Repro'],
_retry_options=RETRY_OPTIONS)
return
# Collect the result values for each of the differences
difference_details = []
commit_infos = []
commits_with_deltas = {}
for change_a, change_b in differences:
if change_b.patch:
commit = change_b.patch
else:
commit = change_b.last_commit
commit_info = commit.AsDict()
values_a = result_values[change_a]
values_b = result_values[change_b]
difference = _FormatDifferenceForBug(commit_info, values_a,
values_b, self.state.metric)
difference_details.append(difference)
commit_infos.append(commit_info)
if values_a and values_b:
mean_delta = job_state.Mean(values_b) - job_state.Mean(
values_a)
commits_with_deltas[commit.id_string] = (mean_delta,
commit_info)
deferred.defer(_UpdatePostAndMergeDeferred,
difference_details,
commit_infos,
list(commits_with_deltas.values()),
self.bug_id,
self.tags,
self.url,
_retry_options=RETRY_OPTIONS)
def _UpdateGerritIfNeeded(self):
if self.gerrit_server and self.gerrit_change_id:
deferred.defer(_UpdateGerritDeferred,
self.gerrit_server,
self.gerrit_change_id,
'%s Job complete.\n\nSee results at: %s' %
(_ROUND_PUSHPIN, self.url),
_retry_options=RETRY_OPTIONS)
def Fail(self, exception=None):
tb = traceback.format_exc() or ''
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
if exception is None:
if exc_info[1] is None:
# We've been called without a exception in sys.exc_info or in our args.
# This should not happen.
exception = errors.JobError('Unknown job error')
exception.category = 'pinpoint'
else:
exception = exc_info[1]
exc_message = exception.message
category = None
if isinstance(exception, errors.JobError):
category = exception.category
self.exception_details = {
'message': exc_message,
'traceback': tb,
'category': category,
}
self.task = None
comment = '\n'.join((title, self.url, '', exc_message))
# Short-circuit jobs failure updates when we are not the first one to mark a
# job done.
if self.use_execution_engine and not MarkDone(self.job_id):
return
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
labels=['Pinpoint-Job-Failed'],
send_email=True,
_retry_options=RETRY_OPTIONS)
scheduler.Complete(self)
def _Schedule(self, countdown=_TASK_INTERVAL):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
name=task_name,
countdown=countdown)
except (apiproxy_errors.DeadlineExceededError,
taskqueue.TransientError) as exc:
raise errors.RecoverableError(exc)
self.task = task.name
def _MaybeScheduleRetry(self):
if not hasattr(self, 'retry_count') or self.retry_count is None:
self.retry_count = 0
if self.retry_count >= _MAX_RECOVERABLE_RETRIES:
return False
self.retry_count += 1
# Back off exponentially
self._Schedule(countdown=_TASK_INTERVAL * (2**self.retry_count))
return True
def Run(self):
"""Runs this Job.
Loops through all Attempts and checks the status of each one, kicking off
tasks as needed. Does not block to wait for all tasks to finish. Also
compares adjacent Changes' results and adds any additional Attempts or
Changes as needed. If there are any incomplete tasks, schedules another
Run() call on the task queue.
"""
self.exception_details = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.use_execution_engine:
# Treat this as if it's a poll, and run the handler here.
context = task_module.Evaluate(
self,
event_module.Event(type='initiate',
target_task=None,
payload={}),
task_evaluator.ExecutionEngine(self))
result_status = context.get('performance_bisection',
{}).get('status')
if result_status not in {'failed', 'completed'}:
return
if result_status == 'failed':
execution_errors = context['find_culprit'].get(
'errors', [])
if execution_errors:
self.exception_details = execution_errors[0]
self._Complete()
return
if not self._IsTryJob():
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
self.retry_count = 0
except errors.RecoverableError as e:
try:
if not self._MaybeScheduleRetry():
self.Fail(errors.JobRetryLimitExceededError(wrapped_exc=e))
except errors.RecoverableError as e:
self.Fail(errors.JobRetryFailed(wrapped_exc=e))
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
if self.completed:
timing_record.RecordJobTiming(self)
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def AsDict(self, options=None):
d = {
'job_id': self.job_id,
'configuration': self.configuration,
'results_url': self.results_url,
'arguments': self.arguments,
'bug_id': self.bug_id,
'comparison_mode': self.comparison_mode,
'name': self.auto_name,
'user': self.user,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'difference_count': self.difference_count,
'exception': self.exception_details_dict,
'status': self.status,
'cancel_reason': self.cancel_reason,
}
if not options:
return d
if OPTION_STATE in options:
if self.use_execution_engine:
d.update(
task_module.Evaluate(
self,
event_module.Event(
type='serialize', target_task=None, payload={}),
job_serializer.Serializer()) or {})
else:
d.update(self.state.AsDict())
if OPTION_ESTIMATE in options and not self.started:
d.update(self._GetRunTimeEstimate())
if OPTION_TAGS in options:
d['tags'] = {'tags': self.tags}
return d
def _GetRunTimeEstimate(self):
result = timing_record.GetSimilarHistoricalTimings(self)
if not result:
return {}
timings = [t.total_seconds() for t in result.timings]
return {
'estimate': {
'timings': timings,
'tags': result.tags
},
'queue_stats': scheduler.QueueStats(self.configuration)
}
def Cancel(self, user, reason):
# We cannot cancel an already cancelled job.
if self.cancelled:
logging.warning(
'Attempted to cancel a cancelled job "%s"; user = %s, reason = %s',
self.job_id, user, reason)
raise errors.CancelError('Job already cancelled.')
if not scheduler.Cancel(self):
raise errors.CancelError('Scheduler failed to cancel job.')
self.cancelled = True
self.cancel_reason = '{}: {}'.format(user, reason)
# Remove any "task" identifiers.
self.task = None
self.put()
title = _ROUND_PUSHPIN + ' Pinpoint job cancelled.'
comment = u'{}\n{}\n\nCancelled by {}, reason given: {}'.format(
title, self.url, user, reason)
deferred.defer(_PostBugCommentDeferred,
self.bug_id,
comment,
send_email=True,
labels=['Pinpoint-Job-Cancelled'],
_retry_options=RETRY_OPTIONS)
class Job(ndb.Model):
"""A Pinpoint job."""
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
updated = ndb.DateTimeProperty(required=True, auto_now=True)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
# Request parameters.
arguments = ndb.JsonProperty(required=True)
repeat_count = ndb.IntegerProperty(required=True)
# If True, the service should pick additional Changes to run (bisect).
# If False, only run the Changes explicitly added by the user.
auto_explore = ndb.BooleanProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
state = ndb.PickleProperty(required=True)
@classmethod
def New(cls,
arguments,
quests,
auto_explore,
repeat_count=_DEFAULT_REPEAT_COUNT,
bug_id=None):
repeat_count = repeat_count or _DEFAULT_REPEAT_COUNT
# Create job.
return cls(arguments=arguments,
auto_explore=auto_explore,
repeat_count=repeat_count,
bug_id=bug_id,
state=_JobState(quests, repeat_count))
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.task:
return 'Running'
if self.exception:
return 'Failed'
return 'Completed'
@property
def url(self):
return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
self.Schedule()
self._PostBugComment('started')
def Complete(self):
self._PostBugComment('completed')
def Fail(self):
self.exception = traceback.format_exc()
self._PostBugComment('stopped with an error')
def Schedule(self):
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
countdown=_TASK_INTERVAL)
self.task = task.name
def Run(self):
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.auto_explore:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self.Schedule()
else:
self.Complete()
except BaseException:
self.Fail()
raise
def AsDict(self, include_state=True):
d = {
'job_id': self.job_id,
'arguments': self.arguments,
'auto_explore': self.auto_explore,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'exception': self.exception,
'status': self.status,
}
if include_state:
d.update(self.state.AsDict())
return d
def _PostBugComment(self, status):
if not self.bug_id:
return
title = '%s Pinpoint job %s.' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
# Include list of Changes.
change_details = []
for _, change in self.state.Differences():
# TODO: Store the commit info in the Commit.
commit = change.last_commit
commit_info = gitiles_service.CommitInfo(commit.repository_url,
commit.git_hash)
subject = commit_info['message'].split('\n', 1)[0]
author = commit_info['author']['email']
time = commit_info['committer']['time']
byline = 'By %s %s %s' % (author, _MIDDLE_DOT, time)
git_link = commit.repository + '@' + commit.git_hash
change_details.append('\n'.join((subject, byline, git_link)))
comment = '\n\n'.join([header] + change_details)
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)
class DatastoreCacheModel(ndb.Model):
data = ndb.PickleProperty(indexed=False, compressed=True)
expires = ndb.DateTimeProperty(indexed=False)
class OrderChangeLogEntry(ndb.Model):
what = ndb.StringProperty(indexed=False)
old = ndb.PickleProperty()
new = ndb.PickleProperty()
class Job(ndb.Model):
"""A Pinpoint job."""
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
updated = ndb.DateTimeProperty(required=True, auto_now=True)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.StringProperty()
# Request parameters.
configuration = ndb.StringProperty(required=True)
test_suite = ndb.StringProperty()
test = ndb.StringProperty()
metric = ndb.StringProperty()
# If True, the service should pick additional Changes to run (bisect).
# If False, only run the Changes explicitly added by the user.
auto_explore = ndb.BooleanProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
state = ndb.PickleProperty(required=True)
@classmethod
def New(cls, configuration, test_suite, test, metric, auto_explore, bug_id):
# Get list of quests.
quests = [quest_module.FindIsolate(configuration)]
if test_suite:
quests.append(quest_module.RunTest(configuration, test_suite, test))
if metric:
quests.append(quest_module.ReadValue(metric, test))
# Create job.
return cls(
configuration=configuration,
test_suite=test_suite,
test=test,
metric=metric,
auto_explore=auto_explore,
bug_id=bug_id,
state=_JobState(quests, _DEFAULT_MAX_ATTEMPTS))
@property
def job_id(self):
return self.key.urlsafe()
@property
def status(self):
if self.task:
return 'Running'
if self.exception:
return 'Failed'
return 'Completed'
@property
def url(self):
return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
self.Schedule()
comment = 'Pinpoint job started.\n' + self.url
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)
def Complete(self):
comment = 'Pinpoint job complete!\n' + self.url
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, comment, send_email=False)
def Schedule(self):
task = taskqueue.add(queue_name='job-queue', url='/api/run/' + self.job_id,
countdown=_TASK_INTERVAL)
self.task = task.name
def Run(self):
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.auto_explore:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self.Schedule()
else:
self.Complete()
except BaseException as e:
self.exception = str(e)
raise
def AsDict(self):
return {
'job_id': self.job_id,
'configuration': self.configuration,
'test_suite': self.test_suite,
'test': self.test,
'metric': self.metric,
'auto_explore': self.auto_explore,
'created': self.created.strftime('%Y-%m-%d %H:%M:%S %Z'),
'updated': self.updated.strftime('%Y-%m-%d %H:%M:%S %Z'),
'status': self.status,
'state': self.state.AsDict(),
}
class History(ndb.Model):
move_date = ndb.DateTimeProperty(required=True, auto_now_add=True)
user = ndb.KeyProperty(required=True, kind='User')
column = ndb.IntegerProperty(required=True)
board_state_after_move = ndb.PickleProperty(required=True)
class Person(ndb.Model):
chat_id = ndb.IntegerProperty()
name = ndb.StringProperty()
last_name = ndb.StringProperty()
username = ndb.StringProperty()
state = ndb.IntegerProperty(default=-1, indexed=True)
location = ndb.GeoPtProperty()
last_state = ndb.IntegerProperty()
last_mod = ndb.DateTimeProperty(auto_now=True)
search_count = ndb.IntegerProperty(default=0)
vote_reminder_enabled = ndb.BooleanProperty(default=True)
search_radius = ndb.IntegerProperty(default=1) #km
search_type = ndb.StringProperty(
default=parameters.SEARCH_TYPE_DRINKING_WATER)
search_locations = ndb.PickleProperty()
enabled = ndb.BooleanProperty(default=True)
def getFirstName(self, escapeMarkdown=True):
if escapeMarkdown:
return utility.escapeMarkdown(self.name.encode('utf-8'))
return self.name.encode('utf-8')
def getLastName(self, escapeMarkdown=True):
if escapeMarkdown:
return utility.escapeMarkdown(self.last_name.encode('utf-8'))
return self.last_name.encode('utf-8')
def getUsername(self):
return self.username.encode('utf-8') if self.username else None
def getUserInfoString(self, escapeMarkdown=True):
info = self.getFirstName(escapeMarkdown)
if self.last_name:
info += ' ' + self.getLastName(escapeMarkdown)
if self.username:
info += ' @' + self.getUsername()
info += ' ({})'.format(self.chat_id)
return info
def setEnabled(self, enabled, put=False):
self.enabled = enabled
if put:
self.put()
def updateUsername(self, username, put=False):
if (self.username != username):
self.username = username
if put:
self.put()
def setState(self, newstate, put=True):
self.last_state = self.state
self.state = newstate
if put:
self.put()
def isAdministrator(self):
result = self.chat_id in key.AMMINISTRATORI_ID
#logging.debug("Amministratore: " + str(result))
return result
def setLocation(self, latitude, longitude, put):
self.location = ndb.GeoPt(latitude, longitude)
if put:
self.put()
def getSearchType(self):
return self.search_type.encode('utf-8')
def getNextSearchRadius(self):
nextRadiusIndex = parameters.POSSIBLE_RADII.index(
self.search_radius) + 1
if nextRadiusIndex < len(parameters.POSSIBLE_RADII):
return parameters.POSSIBLE_RADII[nextRadiusIndex]
return None
def isAdmin(self):
return self.chat_id in key.AMMINISTRATORI_ID
def increaseSearchCount(self, put=False):
self.search_count += 1
if put:
self.put()
def isTimeToRemindToVote(self):
return self.vote_reminder_enabled \
and \
self.search_count % parameters.REMIND_VOTE_BOT_EVERY_N_SEARCH == 0
class Game(ndb.Model):
"""Game class"""
player1 = ndb.KeyProperty(required=True, kind='User')
player2 = ndb.KeyProperty(required=True, kind='User')
player1Colour = ndb.StringProperty(required=True)
player2Colour = ndb.StringProperty(required=True)
whose_turn = ndb.KeyProperty(required=True, kind='User')
created = ndb.DateTimeProperty(auto_now_add=True)
# Board is 7x6 (width x height), so 42 holes/spaces
board = ndb.PickleProperty(required=True)
holes_remaining = ndb.IntegerProperty(required=True, default=42)
game_over = ndb.BooleanProperty(required=True, default=False)
history = ndb.StructuredProperty(History, repeated=True)
@classmethod
def new_game(cls, user1, user2):
"""Creates and returns a new game"""
# Check if users exist
player1 = User.query(User.name == user1).get()
if not player1:
raise ValueError('Player 1 does not exist')
player2 = User.query(User.name == user2).get()
if not player2:
raise ValueError('Player 2 does not exist')
# Randomly choose which player plays first and set token colours
if random.choice(range(1, 3)) == 1:
whose_turn = player1.key
player1_colour = 'R'
player2_colour = "Y"
else:
whose_turn = player2.key
player1_colour = 'Y'
player2_colour = "R"
# Create board, game and put in datastore
game = Game(player1=player1.key,
player2=player2.key,
player1Colour=player1_colour,
player2Colour=player2_colour,
whose_turn=whose_turn,
board=Board())
game.put()
return game
def switch_turn(self):
"""Swaps who's turn it is in the game"""
if self.whose_turn == self.player1:
self.whose_turn = self.player2
else:
self.whose_turn = self.player1
def is_won(self):
"""Checks if the game has been won"""
return self.board.is_won()
def end_game(self, won):
self.game_over = True
if won:
# get the loser (Winner is in whose_turn)
if self.player1 == self.whose_turn:
loser = self.player2
else:
loser = self.player1
# Add the game to the score 'board'
score = Score(winning_user=self.whose_turn,
losing_user=loser,
date=date.today(),
holes_remaining=self.holes_remaining)
score.put()
# Update winning user rank
user_rank = UserRank.query(
UserRank.user_name == self.whose_turn).get()
if not user_rank:
user_rank = UserRank()
user_rank.user_name = self.whose_turn
user_rank.games_played = 1
user_rank.games_won = 1
else:
user_rank.games_played += 1
user_rank.games_won += 1
user_rank.win_ratio = user_rank.games_won / (
user_rank.games_played + 0.0)
user_rank.put()
# Update losing user rank
user_rank = UserRank.query(UserRank.user_name == loser).get()
if not user_rank:
user_rank = UserRank()
user_rank.user_name = loser
user_rank.games_played = 1
user_rank.games_won = 0
else:
user_rank.games_played += 1
user_rank.win_ratio = user_rank.games_won / (
user_rank.games_played + 0.0)
user_rank.put()
def to_form(self, message):
"""Returns a GameForm representation of the Game"""
form = GameForm()
form.urlsafe_key = self.key.urlsafe()
form.player1 = self.player1.get().name
form.player2 = self.player2.get().name
# TODO: return player colours
form.whose_turn = self.whose_turn.get().name
form.holes_remaining = self.holes_remaining
form.game_over = self.game_over
form.message = message
form.board = str(self.board.board)
return form
def history_to_form(self):
"""Returns a series of History Forms"""
forms = HistoryForms()
for history in self.history:
form = HistoryForm()
form.move_date = str(history.move_date)
form.user_name = history.user.get().name
form.column = history.column
print str(history.board_state_after_move)
form.board_state = str(history.board_state_after_move)
forms.items.append(form)
return forms
class Job(ndb.Model):
"""A Pinpoint job."""
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
# Request parameters.
arguments = ndb.JsonProperty(required=True)
# If True, the service should pick additional Changes to run (bisect).
# If False, only run the Changes explicitly added by the user.
auto_explore = ndb.BooleanProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
state = ndb.PickleProperty(required=True, compressed=True)
@classmethod
def New(cls, arguments, quests, auto_explore, bug_id=None):
# Create job.
return cls(
arguments=arguments,
auto_explore=auto_explore,
bug_id=bug_id,
state=_JobState(quests))
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.task:
return 'Running'
if self.exception:
return 'Failed'
return 'Completed'
@property
def url(self):
return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
self._Schedule()
title = _ROUND_PUSHPIN + ' Pinpoint job started.'
comment = '\n'.join((title, self.url))
self._PostBugComment(comment, send_email=False)
def _Complete(self):
# Format bug comment.
differences = tuple(self.state.Differences())
if not differences:
title = "<b>%s Couldn't reproduce a difference.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# Include list of Changes.
owner = None
cc_list = set()
commit_details = []
for _, change in differences:
commit_info = change.last_commit.Details()
author = commit_info['author']['email']
owner = author # TODO: Assign the largest difference, not the last one.
cc_list.add(author)
cc_list |= frozenset(re.findall('Reviewed-by: .+ <(.+)>',
commit_info['message']))
commit_details.append(_FormatCommitForBug(
change.last_commit, commit_info))
# Header.
if len(differences) == 1:
status = 'Found a significant difference after 1 commit.'
else:
status = ('Found significant differences after each of %d commits.' %
len(differences))
title = '<b>%s %s</b>' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
# Body.
body = '\n\n'.join(commit_details)
# Footer.
footer = ('Understanding performance regressions:\n'
' http://g.co/ChromePerformanceRegressions')
# Bring it all together.
comment = '\n\n'.join((header, body, footer))
self._PostBugComment(comment, status='Assigned',
cc_list=sorted(cc_list), owner=owner)
def _Fail(self):
self.exception = traceback.format_exc()
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
comment = '\n'.join((title, self.url))
self._PostBugComment(comment)
def _Schedule(self):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=_TASK_INTERVAL)
except apiproxy_errors.DeadlineExceededError:
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=_TASK_INTERVAL)
self.task = task.name
def Run(self):
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.auto_explore:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
except BaseException:
self._Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
def AsDict(self, include_state=True):
d = {
'job_id': self.job_id,
'arguments': self.arguments,
'auto_explore': self.auto_explore,
'bug_id': self.bug_id,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'exception': self.exception,
'status': self.status,
}
if include_state:
d.update(self.state.AsDict())
return d
def _PostBugComment(self, *args, **kwargs):
if not self.bug_id:
return
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, *args, **kwargs)
class ExampleStore(ndb.Model):
original_html = ndb.TextProperty('h', indexed=False)
microdata = ndb.TextProperty('m', indexed=False)
rdfa = ndb.TextProperty('r', indexed=False)
jsonld = ndb.TextProperty('j', indexed=False)
egmeta = ndb.PickleProperty('e', indexed=False)
keyvalue = ndb.StringProperty('o', indexed=True)
layer = ndb.StringProperty('l', indexed=False)
@staticmethod
def initialise():
EXAMPLESTORECACHE = []
import time
log.info("[%s]ExampleStore initialising Data Store" %
(getInstanceId(short=True)))
loops = 0
ret = 0
while loops < 10:
keys = ExampleStore.query().fetch(keys_only=True,
use_memcache=False,
use_cache=False)
count = len(keys)
if count == 0:
break
log.info("[%s]ExampleStore deleting %s keys" %
(getInstanceId(short=True), count))
ndb.delete_multi(keys, use_memcache=False, use_cache=False)
ret += count
loops += 1
time.sleep(0.01)
return {"ExampleStore": ret}
@staticmethod
def add(example):
e = ExampleStore(id=example.keyvalue,
original_html=example.original_html,
microdata=example.microdata,
rdfa=example.rdfa,
jsonld=example.jsonld,
egmeta=example.egmeta,
keyvalue=example.keyvalue,
layer=example.layer)
EXAMPLESTORECACHE.append(e)
@staticmethod
def store(examples):
for e in examples:
ExampleStore.add(e)
if len(EXAMPLESTORECACHE):
ndb.put_multi(EXAMPLESTORECACHE, use_cache=False)
def get(self, name):
if name == 'original_html':
return self.original_html
if name == 'microdata':
return self.microdata
if name == 'rdfa':
return self.rdfa
if name == 'jsonld':
return self.jsonld
return ""
@staticmethod
def getEgmeta(id):
em = ExampleStore.get_by_id(id)
ret = em.emeta
if ret:
return ret
return {}
class Session(ndb.Model):
session_id = ndb.StringProperty()
user_id = ndb.StringProperty()
updated = ndb.DateTimeProperty(auto_now=True)
data = ndb.PickleProperty(compressed=True, default={})
@staticmethod
def _generate_sid():
return security.generate_random_string(entropy=128)
@staticmethod
def _serializer():
engineauth_config = config.load_config()
return securecookie.SecureCookieSerializer(
engineauth_config['secret_key'])
def hash(self):
"""
Creates a unique hash from the session.
This will be used to check for session changes.
:return: A unique hash for the session
"""
dataStr = repr(self.data)
return "{}.{}.{}.{}.{}".format(self.session_id, self.user_id,
str(self.updated), hash(dataStr),
len(dataStr))
def serialize(self):
values = self.to_dict(include=['session_id', 'user_id'])
return self._serializer().serialize('_eauth', values)
@classmethod
def deserialize(cls, value):
return cls._serializer().deserialize('_eauth', value)
@classmethod
def get_by_value(cls, value):
v = cls.deserialize(value)
if v:
return cls.get_by_sid(v.get('session_id'))
return None
@classmethod
def get_by_sid(cls, sid):
return cls.get_by_id(sid)
@classmethod
def upgrade_to_user_session(cls, session_id, user_id):
old_session = cls.get_by_sid(session_id)
new_session = cls.create(user_id=user_id, data=old_session.data)
old_session.key.delete()
return new_session
@classmethod
def get_by_user_id(cls, user_id):
# TODO: make sure that the user doesn't have multiple sessions
user_id = str(user_id)
return cls.query(cls.user_id == user_id).get()
@classmethod
def create(cls, user_id=None, **kwargs):
if user_id is None:
session_id = cls._generate_sid()
else:
session_id = user_id = str(user_id)
session = cls(id=session_id,
session_id=session_id,
user_id=user_id,
**kwargs)
session.put()
return session
@classmethod
def remove_inactive(cls, days_ago=30, now=None):
import datetime
# for testing we want to be able to pass a value for now.
now = now or datetime.datetime.now()
dtd = now + datetime.timedelta(-days_ago)
for s in cls.query(cls.updated < dtd).fetch():
s.key.delete()
class HeaderEntity(ndb.Model):
content = ndb.PickleProperty()
class TryJob(internal_only_model.InternalOnlyModel):
"""Stores config and tracking info about a single try job."""
bot = ndb.StringProperty()
config = ndb.TextProperty()
bug_id = ndb.IntegerProperty()
email = ndb.StringProperty()
rietveld_issue_id = ndb.IntegerProperty()
rietveld_patchset_id = ndb.IntegerProperty()
master_name = ndb.StringProperty(default='ChromiumPerf', indexed=False)
buildbucket_job_id = ndb.StringProperty()
internal_only = ndb.BooleanProperty(default=False, indexed=True)
# Bisect run status (e.g., started, failed).
status = ndb.StringProperty(
default='pending',
choices=[
'pending', # Created, but job start has not been confirmed.
'started', # Job is confirmed started.
'failed', # Job terminated, red build.
'staled', # No updates from bots.
'completed', # Job terminated, green build.
'aborted', # Job terminated with abort (purple, early abort).
],
indexed=True)
# Number of times this job has been tried.
run_count = ndb.IntegerProperty(default=0)
# Last time this job was started.
last_ran_timestamp = ndb.DateTimeProperty()
job_type = ndb.StringProperty(default='bisect',
choices=['bisect', 'bisect-fyi', 'perf-try'])
# job_name attribute is used by try jobs of bisect FYI.
job_name = ndb.StringProperty(default=None)
# Results data coming from bisect bots.
results_data = ndb.JsonProperty(indexed=False)
log_record_id = ndb.StringProperty(indexed=False)
# Sets of emails of users who has confirmed this TryJob result is bad.
bad_result_emails = ndb.PickleProperty()
def SetStarted(self):
self.status = 'started'
self.run_count += 1
self.last_ran_timestamp = datetime.datetime.now()
self.put()
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'started')
def SetFailed(self):
self.status = 'failed'
self.put()
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'failed')
bisect_stats.UpdateBisectStats(self.bot, 'failed')
def SetStaled(self):
self.status = 'staled'
self.put()
# TODO(chrisphan): Add 'staled' state to bug_data and bisect_stats.
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'failed')
bisect_stats.UpdateBisectStats(self.bot, 'failed')
def SetCompleted(self):
self.status = 'completed'
self.put()
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'completed')
bisect_stats.UpdateBisectStats(self.bot, 'completed')
def GetConfigDict(self):
return json.loads(self.config.split('=', 1)[1])
def CheckFailureFromBuildBucket(self):
# Buildbucket job id is not always set.
if not self.buildbucket_job_id:
return
job_info = buildbucket_service.GetJobStatus(self.buildbucket_job_id)
data = job_info.get('build', {})
# Proceed if the job failed and the job status has
# not been updated
if data.get('result') != 'FAILURE' or self.status == 'failed':
return
data['result_details'] = json.loads(data['result_details_json'])
job_updates = {
'status': 'failed',
'failure_reason': data.get('failure_reason'),
'buildbot_log_url': data.get('url')
}
details = data.get('result_details')
if details:
properties = details.get('properties')
if properties:
job_updates['bisect_bot'] = properties.get('buildername')
job_updates['extra_result_code'] = properties.get(
'extra_result_code')
bisect_config = properties.get('bisect_config')
if bisect_config:
job_updates['try_job_id'] = bisect_config.get('try_job_id')
job_updates['bug_id'] = bisect_config.get('bug_id')
job_updates['command'] = bisect_config.get('command')
job_updates['test_type'] = bisect_config.get('test_type')
job_updates['metric'] = bisect_config.get('metric')
job_updates['good_revision'] = bisect_config.get(
'good_revision')
job_updates['bad_revision'] = bisect_config.get(
'bad_revision')
if not self.results_data:
self.results_data = {}
self.results_data.update(job_updates)
self.status = 'failed'
self.last_ran_timestamp = datetime.datetime.fromtimestamp(
float(data['updated_ts']) / 1000000)
self.put()
class Job(ndb.Model):
"""A Pinpoint job."""
state = ndb.PickleProperty(required=True, compressed=True)
#####
# Job arguments passed in through the API.
#####
# Request parameters.
arguments = ndb.JsonProperty(required=True)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
comparison_mode = ndb.StringProperty()
# The Gerrit server url and change id of the code review to update upon
# completion.
gerrit_server = ndb.StringProperty()
gerrit_change_id = ndb.StringProperty()
# User-provided name of the job.
name = ndb.StringProperty()
tags = ndb.JsonProperty()
# Email of the job creator.
user = ndb.StringProperty()
#####
# Job state generated by running the job.
#####
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
completed = ndb.ComputedProperty(lambda self: not self.task)
failed = ndb.ComputedProperty(lambda self: bool(self.exception))
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
difference_count = ndb.IntegerProperty()
retry_count = ndb.IntegerProperty(default=0)
@classmethod
def New(cls, quests, changes, arguments=None, bug_id=None,
comparison_mode=None, comparison_magnitude=None, gerrit_server=None,
gerrit_change_id=None, name=None, pin=None, tags=None, user=None):
"""Creates a new Job, adds Changes to it, and puts it in the Datstore.
Args:
quests: An iterable of Quests for the Job to run.
changes: An iterable of the initial Changes to run on.
arguments: A dict with the original arguments used to start the Job.
bug_id: A monorail issue id number to post Job updates to.
comparison_mode: Either 'functional' or 'performance', which the Job uses
to figure out whether to perform a functional or performance bisect.
If None, the Job will not automatically add any Attempts or Changes.
comparison_magnitude: The estimated size of the regression or improvement
to look for. Smaller magnitudes require more repeats.
gerrit_server: Server of the Gerrit code review to update with job
results.
gerrit_change_id: Change id of the Gerrit code review to update with job
results.
name: The user-provided name of the Job.
pin: A Change (Commits + Patch) to apply to every Change in this Job.
tags: A dict of key-value pairs used to filter the Jobs listings.
user: The email of the Job creator.
Returns:
A Job object.
"""
state = job_state.JobState(
quests, comparison_mode=comparison_mode,
comparison_magnitude=comparison_magnitude, pin=pin)
job = cls(state=state, arguments=arguments or {}, bug_id=bug_id,
comparison_mode=comparison_mode, gerrit_server=gerrit_server,
gerrit_change_id=gerrit_change_id,
name=name, tags=tags, user=user)
for c in changes:
job.AddChange(c)
job.put()
return job
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if not self.completed:
return 'Running'
if self.failed:
return 'Failed'
return 'Completed'
@property
def url(self):
host = os.environ['HTTP_HOST']
# TODO(crbug.com/939723): Remove this workaround when not needed.
if host == 'pinpoint.chromeperf.appspot.com':
host = 'pinpoint-dot-chromeperf.appspot.com'
return 'https://%s/job/%s' % (host, self.job_id)
@property
def results_url(self):
if not self.task:
url = results2.GetCachedResults2(self)
if url:
return url
# Point to the default status page if no results are available.
return '/results2/%s' % self.job_id
@property
def auto_name(self):
if self.name:
return self.name
if self.comparison_mode == job_state.FUNCTIONAL:
name = 'Functional bisect'
elif self.comparison_mode == job_state.PERFORMANCE:
name = 'Performance bisect'
else:
name = 'Try job'
if 'configuration' in self.arguments:
name += ' on ' + self.arguments['configuration']
if 'benchmark' in self.arguments:
name += '/' + self.arguments['benchmark']
return name
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
"""Starts the Job and updates it in the Datastore.
This method is designed to return fast, so that Job creation is responsive
to the user. It schedules the Job on the task queue without running
anything. It also posts a bug comment, and updates the Datastore.
"""
self._Schedule()
self.put()
title = _ROUND_PUSHPIN + ' Pinpoint job started.'
comment = '\n'.join((title, self.url))
self._PostBugComment(comment, send_email=False)
def _Complete(self):
if self.comparison_mode:
self.difference_count = len(self.state.Differences())
try:
results2.ScheduleResults2Generation(self)
except taskqueue.Error:
pass
self._FormatAndPostBugCommentOnComplete()
self._UpdateGerritIfNeeded()
def _FormatAndPostBugCommentOnComplete(self):
if not self.comparison_mode:
# There is no comparison metric.
title = "<b>%s Job complete. See results below.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# There is a comparison metric.
differences = self.state.Differences()
if not differences:
title = "<b>%s Couldn't reproduce a difference.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# Include list of Changes.
owner = None
sheriff = None
cc_list = set()
difference_details = []
for change_a, change_b in differences:
if change_b.patch:
commit_info = change_b.patch.AsDict()
else:
commit_info = change_b.last_commit.AsDict()
# TODO: Assign the largest difference, not the last one.
owner = commit_info['author']
sheriff = utils.GetSheriffForAutorollCommit(
commit_info['author'], commit_info['message'])
cc_list.add(commit_info['author'])
values_a = self.state.ResultValues(change_a)
values_b = self.state.ResultValues(change_b)
difference = _FormatDifferenceForBug(commit_info, values_a, values_b,
self.state.metric)
difference_details.append(difference)
# Header.
if len(differences) == 1:
status = 'Found a significant difference after 1 commit.'
else:
status = ('Found significant differences after each of %d commits.' %
len(differences))
title = '<b>%s %s</b>' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
# Body.
body = '\n\n'.join(difference_details)
if sheriff:
owner = sheriff
body += '\n\nAssigning to sheriff %s because "%s" is a roll.' % (
sheriff, commit_info['subject'])
# Footer.
footer = ('Understanding performance regressions:\n'
' http://g.co/ChromePerformanceRegressions')
if differences:
footer += self._FormatDocumentationUrls()
# Bring it all together.
comment = '\n\n'.join((header, body, footer))
current_bug_status = self._GetBugStatus()
if (not current_bug_status or
current_bug_status in ['Untriaged', 'Unconfirmed', 'Available']):
# Set the bug status and owner if this bug is opened and unowned.
self._PostBugComment(comment, status='Assigned',
cc_list=sorted(cc_list), owner=owner)
else:
# Only update the comment and cc list if this bug is assigned or closed.
self._PostBugComment(comment, cc_list=sorted(cc_list))
def _FormatDocumentationUrls(self):
if not self.tags:
return ''
# TODO(simonhatch): Tags isn't the best way to get at this, but wait until
# we move this back into the dashboard so we have a better way of getting
# at the test path.
# crbug.com/876899
test_path = self.tags.get('test_path')
if not test_path:
return ''
test_suite = utils.TestKey('/'.join(test_path.split('/')[:3]))
docs = histogram.SparseDiagnostic.GetMostRecentDataByNamesSync(
test_suite, [reserved_infos.DOCUMENTATION_URLS.name])
if not docs:
return ''
docs = docs[reserved_infos.DOCUMENTATION_URLS.name].get('values')
footer = '\n\n%s:\n %s' % (docs[0][0], docs[0][1])
return footer
def _UpdateGerritIfNeeded(self):
if self.gerrit_server and self.gerrit_change_id:
gerrit_service.PostChangeComment(
self.gerrit_server,
self.gerrit_change_id,
'%s Job complete.\n\nSee results at: %s' % (_ROUND_PUSHPIN, self.url))
def Fail(self, exception=None):
if exception:
self.exception = exception
else:
self.exception = traceback.format_exc()
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
exc_info = sys.exc_info()
exc_message = ''
if exc_info[1]:
exc_message = sys.exc_info()[1].message
elif self.exception:
exc_message = self.exception.splitlines()[-1]
comment = '\n'.join((title, self.url, '', exc_message))
self.task = None
self._PostBugComment(comment)
def _Schedule(self, countdown=_TASK_INTERVAL):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=countdown)
except (apiproxy_errors.DeadlineExceededError, taskqueue.TransientError):
task = taskqueue.add(
queue_name='job-queue', url='/api/run/' + self.job_id,
name=task_name, countdown=countdown)
self.task = task.name
def _MaybeScheduleRetry(self):
if not hasattr(self, 'retry_count') or self.retry_count is None:
self.retry_count = 0
if self.retry_count >= _MAX_RECOVERABLE_RETRIES:
return False
self.retry_count += 1
# Back off exponentially
self._Schedule(countdown=_TASK_INTERVAL * (2 ** self.retry_count))
return True
def Run(self):
"""Runs this Job.
Loops through all Attempts and checks the status of each one, kicking off
tasks as needed. Does not block to wait for all tasks to finish. Also
compares adjacent Changes' results and adds any additional Attempts or
Changes as needed. If there are any incomplete tasks, schedules another
Run() call on the task queue.
"""
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.comparison_mode:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
self.retry_count = 0
except job_state.JobStateRecoverableError:
if not self._MaybeScheduleRetry():
self.Fail()
raise
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def AsDict(self, options=None):
d = {
'job_id': self.job_id,
'results_url': self.results_url,
'arguments': self.arguments,
'bug_id': self.bug_id,
'comparison_mode': self.comparison_mode,
'name': self.auto_name,
'user': self.user,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'difference_count': self.difference_count,
'exception': self.exception,
'status': self.status,
}
if not options:
return d
if OPTION_STATE in options:
d.update(self.state.AsDict())
if OPTION_TAGS in options:
d['tags'] = {'tags': self.tags}
return d
def _PostBugComment(self, *args, **kwargs):
if not self.bug_id:
return
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, *args, **kwargs)
def _GetBugStatus(self):
if not self.bug_id:
return None
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_data = issue_tracker.GetIssue(self.bug_id)
return issue_data.get('status')
class Issue(ndb.Model):
number = ndb.IntegerProperty(required=True)
updated_at = ndb.DateTimeProperty()
user = ndb.StringProperty()
state = ndb.StringProperty()
title = ndb.StringProperty()
comments_json = ndb.JsonProperty(compressed=True)
comments_etag = ndb.StringProperty()
pr_comments_json = ndb.JsonProperty(compressed=True)
pr_comments_etag = ndb.StringProperty()
files_json = ndb.JsonProperty(compressed=True)
files_etag = ndb.StringProperty()
pr_json = ndb.JsonProperty()
etag = ndb.StringProperty()
# Cached properties, while we migrate away from on-the-fly computed ones:
cached_commenters = ndb.PickleProperty()
cached_last_jenkins_outcome = ndb.StringProperty()
last_jenkins_comment = ndb.JsonProperty()
ASKED_TO_CLOSE_REGEX = re.compile(
r"""
(mind\s+closing\s+(this|it))|
(close\s+this\s+(issue|pr))
""", re.I | re.X)
_components = [
# (name, pr_title_regex, filename_regex)
("Core", "core", "^core/"),
("Scheduler", "schedul", "scheduler"),
("Python", "python|pyspark", "python"),
("YARN", "yarn", "yarn"),
("Mesos", "mesos", "mesos"),
("Web UI", "webui|(web ui)", "spark/ui/"),
("Build", "build", "(pom\.xml)|project"),
("Docs", "docs", "docs|README"),
("EC2", "ec2", "ec2"),
("SQL", "sql", "sql"),
("MLlib", "mllib", "mllib"),
("GraphX", "graphx|pregel", "graphx"),
("Streaming", "stream|flume|kafka|twitter|zeromq", "streaming"),
]
@property
def components(self):
"""
Returns the list of components used to classify this pull request.
Components are identified automatically based on the files that the pull request
modified and any tags added to the pull request's title (such as [GraphX]).
"""
components = []
title = ((self.pr_json and self.pr_json["title"]) or self.title or "")
modified_files = [f["filename"] for f in (self.files_json or [])]
for (component_name, pr_title_regex,
filename_regex) in Issue._components:
if re.search(pr_title_regex, title, re.IGNORECASE) or \
any(re.search(filename_regex, f, re.I) for f in modified_files):
components.append(component_name)
return components or ["Core"]
@property
def parsed_title(self):
"""
Get a parsed version of this PR's title, which identifies referenced JIRAs and metadata.
For example, given a PR titled
"[SPARK-975] [core] Visual debugger of stages and callstacks""
this will return
{'jiras': [975], 'title': 'Visual debugger of stages and callstacks', 'metadata': ''}
"""
return parse_pr_title((self.pr_json and self.pr_json["title"])
or self.title or "")
@property
def lines_added(self):
if self.pr_json:
return self.pr_json.get("additions")
else:
return ""
@property
def lines_deleted(self):
if self.pr_json:
return self.pr_json.get("deletions")
else:
return ""
@property
def lines_changed(self):
if self.lines_added != "":
return self.lines_added + self.lines_deleted
else:
return 0
@property
def is_mergeable(self):
return self.pr_json and self.pr_json["mergeable"]
@property
def commenters(self):
if self.cached_commenters is None:
self.cached_commenters = self._compute_commenters()
self.put()
return self.cached_commenters
@property
def last_jenkins_outcome(self):
if self.cached_last_jenkins_outcome is None:
(outcome, comment) = self._compute_last_jenkins_outcome()
self.cached_last_jenkins_outcome = outcome
self.last_jenkins_comment = comment
self.put()
return self.cached_last_jenkins_outcome
def _compute_commenters(self):
res = defaultdict(
dict) # Indexed by user, since we only display each user once.
excluded_users = set(("SparkQA", "AmplabJenkins"))
all_comments = sorted(
(self.comments_json or []) + (self.pr_comments_json or []),
key=lambda c: c['created_at'])
for comment in all_comments:
if is_jenkins_command(comment['body']):
continue # Skip comments that solely consist of Jenkins commands
user = comment['user']['login']
if user not in excluded_users:
user_dict = res[user]
user_dict['url'] = comment['html_url']
user_dict['avatar'] = comment['user']['avatar_url']
user_dict['date'] = comment['created_at'],
user_dict['body'] = comment['body']
# Display at most 10 lines of context for comments left on diffs:
user_dict['diff_hunk'] = '\n'.join(
comment.get('diff_hunk', '').split('\n')[-10:])
user_dict['said_lgtm'] = (user_dict.get('said_lgtm')
or re.search("lgtm", comment['body'],
re.I) is not None)
user_dict['asked_to_close'] = \
(user_dict.get('asked_to_close')
or Issue.ASKED_TO_CLOSE_REGEX.search(comment['body']) is not None)
return sorted(res.items(), key=lambda x: x[1]['date'], reverse=True)
def _compute_last_jenkins_outcome(self):
status = "Unknown"
jenkins_comment = None
for comment in (self.comments_json or []):
if contains_jenkins_command(comment['body']):
status = "Asked"
jenkins_comment = comment
elif comment['user']['login'] in ("SparkQA", "AmplabJenkins"):
body = comment['body'].lower()
jenkins_comment = comment
if "pass" in body:
status = "Pass"
elif "fail" in body:
status = "Fail"
elif "started" in body:
status = "Running"
elif "can one of the admins verify this patch?" in body:
status = "Verify"
elif "timed out" in body:
status = "Timeout"
else:
status = "Unknown" # So we display "Unknown" instead of an out-of-date status
return (status, jenkins_comment)
@classmethod
def get_or_create(cls, number):
key = str(ndb.Key("Issue", number).id())
return Issue.get_or_insert(key, number=number)
def update(self, oauth_token):
logging.debug("Updating pull request %i" % self.number)
# Record basic information about this pull request
issue_response = raw_github_request(PULLS_BASE + '/%i' % self.number,
oauth_token=oauth_token,
etag=self.etag)
if issue_response is None:
logging.debug("PR %i hasn't changed since last visit; skipping" %
self.number)
return
self.pr_json = json.loads(issue_response.content)
self.etag = issue_response.headers["ETag"]
updated_at = \
parse_datetime(self.pr_json['updated_at']).astimezone(tz.tzutc()).replace(tzinfo=None)
self.user = self.pr_json['user']['login']
self.updated_at = updated_at
self.state = self.pr_json['state']
comments_response = paginated_github_request(
ISSUES_BASE + '/%i/comments' % self.number,
oauth_token=oauth_token,
etag=self.comments_etag)
if comments_response is not None:
self.comments_json, self.comments_etag = comments_response
pr_comments_response = paginated_github_request(
PULLS_BASE + '/%i/comments' % self.number,
oauth_token=oauth_token,
etag=self.pr_comments_etag)
if pr_comments_response is not None:
self.pr_comments_json, self.pr_comments_etag = pr_comments_response
files_response = paginated_github_request(PULLS_BASE +
"/%i/files" % self.number,
oauth_token=oauth_token,
etag=self.files_etag)
if files_response is not None:
self.files_json, self.files_etag = files_response
self.cached_last_jenkins_outcome = None
self.last_jenkins_outcome # force recomputation of Jenkins outcome
self.cached_commenters = self._compute_commenters()
for issue_number in self.parsed_title['jiras']:
try:
link_issue_to_pr("SPARK-%s" % issue_number, self)
except:
logging.exception(
"Exception when linking to JIRA issue SPARK-%s" %
issue_number)
self.put() # Write our modifications back to the database
class Drone(ndb.Model):
"""Models an individual drone."""
drone_name = ndb.StringProperty(indexed=True)
range_in_kilometers = ndb.FloatProperty()
update_time = ndb.DateTimeProperty()
location_cost_map = ndb.PickleProperty()
class Job(ndb.Model):
"""A Pinpoint job."""
created = ndb.DateTimeProperty(required=True, auto_now_add=True)
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
updated = ndb.DateTimeProperty(required=True, auto_now_add=True)
# The name of the Task Queue task this job is running on. If it's present, the
# job is running. The task is also None for Task Queue retries.
task = ndb.StringProperty()
# The string contents of any Exception that was thrown to the top level.
# If it's present, the job failed.
exception = ndb.TextProperty()
# Request parameters.
arguments = ndb.JsonProperty(required=True)
# If True, the service should pick additional Changes to run (bisect).
# If False, only run the Changes explicitly added by the user.
auto_explore = ndb.BooleanProperty(required=True)
# The metric to use when determining whether to add additional Attempts or
# Changes to the Job. If None, the Job will use a fixed number of Attempts.
comparison_mode = msgprop.EnumProperty(ComparisonMode)
# TODO: The bug id is only used for posting bug comments when a job starts and
# completes. This probably should not be the responsibility of Pinpoint.
bug_id = ndb.IntegerProperty()
# Email of the job creator.
user = ndb.StringProperty()
state = ndb.PickleProperty(required=True, compressed=True)
tags = ndb.JsonProperty()
@classmethod
def New(cls,
arguments,
quests,
auto_explore,
comparison_mode=None,
user=None,
bug_id=None,
tags=None):
# Create job.
return cls(arguments=arguments,
auto_explore=auto_explore,
comparison_mode=comparison_mode,
user=user,
bug_id=bug_id,
tags=tags,
state=job_state.JobState(quests))
@property
def job_id(self):
return '%x' % self.key.id()
@property
def status(self):
if self.task:
return 'Running'
if self.exception:
return 'Failed'
return 'Completed'
@property
def url(self):
return 'https://%s/job/%s' % (os.environ['HTTP_HOST'], self.job_id)
def AddChange(self, change):
self.state.AddChange(change)
def Start(self):
self._Schedule()
title = _ROUND_PUSHPIN + ' Pinpoint job started.'
comment = '\n'.join((title, self.url))
self._PostBugComment(comment, send_email=False)
def _Complete(self):
try:
results2.ScheduleResults2Generation(self)
except taskqueue.Error:
pass
# Format bug comment.
if not self.auto_explore:
# There is no comparison metric.
title = "<b>%s Job complete. See results below.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# There is a comparison metric.
differences = tuple(self.state.Differences())
if not differences:
title = "<b>%s Couldn't reproduce a difference.</b>" % _ROUND_PUSHPIN
self._PostBugComment('\n'.join((title, self.url)))
return
# Include list of Changes.
owner = None
cc_list = set()
commit_details = []
for _, change in differences:
if change.patch:
commit_info = change.patch.AsDict()
else:
commit_info = change.last_commit.AsDict()
# TODO: Assign the largest difference, not the last one.
owner = commit_info['author']
cc_list.add(commit_info['author'])
if 'reviewers' in commit_info:
cc_list |= frozenset(commit_info['reviewers'])
commit_details.append(_FormatCommitForBug(commit_info))
# Header.
if len(differences) == 1:
status = 'Found a significant difference after 1 commit.'
else:
status = (
'Found significant differences after each of %d commits.' %
len(differences))
title = '<b>%s %s</b>' % (_ROUND_PUSHPIN, status)
header = '\n'.join((title, self.url))
# Body.
body = '\n\n'.join(commit_details)
# Footer.
footer = ('Understanding performance regressions:\n'
' http://g.co/ChromePerformanceRegressions')
# Bring it all together.
comment = '\n\n'.join((header, body, footer))
current_bug_status = self._GetBugStatus()
if (not current_bug_status or current_bug_status
in ['Untriaged', 'Unconfirmed', 'Available']):
# Set the bug status and owner if this bug is opened and unowned.
self._PostBugComment(comment,
status='Assigned',
cc_list=sorted(cc_list),
owner=owner)
else:
# Only update the comment and cc list if this bug is assigned or closed.
self._PostBugComment(comment, cc_list=sorted(cc_list))
def Fail(self):
self.exception = traceback.format_exc()
title = _CRYING_CAT_FACE + ' Pinpoint job stopped with an error.'
comment = '\n'.join((title, self.url, '', sys.exc_value.message))
self._PostBugComment(comment)
def _Schedule(self):
# Set a task name to deduplicate retries. This adds some latency, but we're
# not latency-sensitive. If Job.Run() works asynchronously in the future,
# we don't need to worry about duplicate tasks.
# https://github.com/catapult-project/catapult/issues/3900
task_name = str(uuid.uuid4())
try:
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
name=task_name,
countdown=_TASK_INTERVAL)
except apiproxy_errors.DeadlineExceededError:
task = taskqueue.add(queue_name='job-queue',
url='/api/run/' + self.job_id,
name=task_name,
countdown=_TASK_INTERVAL)
self.task = task.name
def Run(self):
self.exception = None # In case the Job succeeds on retry.
self.task = None # In case an exception is thrown.
try:
if self.auto_explore:
self.state.Explore()
work_left = self.state.ScheduleWork()
# Schedule moar task.
if work_left:
self._Schedule()
else:
self._Complete()
except BaseException:
self.Fail()
raise
finally:
# Don't use `auto_now` for `updated`. When we do data migration, we need
# to be able to modify the Job without changing the Job's completion time.
self.updated = datetime.datetime.now()
try:
self.put()
except (datastore_errors.Timeout,
datastore_errors.TransactionFailedError):
# Retry once.
self.put()
except datastore_errors.BadRequestError:
if self.task:
queue = taskqueue.Queue('job-queue')
queue.delete_tasks(taskqueue.Task(name=self.task))
self.task = None
# The _JobState is too large to fit in an ndb property.
# Load the Job from before we updated it, and fail it.
job = self.key.get(use_cache=False)
job.task = None
job.Fail()
job.updated = datetime.datetime.now()
job.put()
raise
def AsDict(self, options=None):
d = {
'job_id': self.job_id,
'arguments': self.arguments,
'auto_explore': self.auto_explore,
'bug_id': self.bug_id,
'user': self.user,
'created': self.created.isoformat(),
'updated': self.updated.isoformat(),
'exception': self.exception,
'status': self.status,
}
if not options:
return d
if OPTION_STATE in options:
d.update(self.state.AsDict())
if OPTION_TAGS in options:
d['tags'] = {'tags': self.tags}
return d
def _PostBugComment(self, *args, **kwargs):
if not self.bug_id:
return
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_tracker.AddBugComment(self.bug_id, *args, **kwargs)
def _GetBugStatus(self):
if not self.bug_id:
return None
issue_tracker = issue_tracker_service.IssueTrackerService(
utils.ServiceAccountHttp())
issue_data = issue_tracker.GetIssue(self.bug_id)
return issue_data.get('status')
class TryJob(internal_only_model.InternalOnlyModel):
"""Stores config and tracking info about a single try job."""
bot = ndb.StringProperty()
config = ndb.TextProperty()
bug_id = ndb.IntegerProperty()
email = ndb.StringProperty()
rietveld_issue_id = ndb.IntegerProperty()
rietveld_patchset_id = ndb.IntegerProperty()
master_name = ndb.StringProperty(default='ChromiumPerf', indexed=False)
buildbucket_job_id = ndb.StringProperty()
internal_only = ndb.BooleanProperty(default=False, indexed=True)
# Bisect run status (e.g., started, failed).
status = ndb.StringProperty(
default='pending',
choices=[
'pending', # Created, but job start has not been confirmed.
'started', # Job is confirmed started.
'failed', # Job terminated, red build.
'staled', # No updates from bots.
'completed', # Job terminated, green build.
'aborted', # Job terminated with abort (purple, early abort).
],
indexed=True)
# Last time this job was started.
last_ran_timestamp = ndb.DateTimeProperty()
job_type = ndb.StringProperty(
default='bisect',
choices=['bisect', 'bisect-fyi', 'perf-try'])
# job_name attribute is used by try jobs of bisect FYI.
job_name = ndb.StringProperty(default=None)
# Results data coming from bisect bots.
results_data = ndb.JsonProperty(indexed=False)
log_record_id = ndb.StringProperty(indexed=False)
# Sets of emails of users who has confirmed this TryJob result is bad.
bad_result_emails = ndb.PickleProperty()
def SetStarted(self):
self.status = 'started'
self.last_ran_timestamp = datetime.datetime.now()
self.put()
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'started')
def SetFailed(self):
self.status = 'failed'
self.put()
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'failed')
bisect_stats.UpdateBisectStats(self.bot, 'failed')
def SetStaled(self):
self.status = 'staled'
self.put()
logging.info('Updated status to staled')
# TODO(sullivan, dtu): what is the purpose of 'staled' status? Doesn't it
# just prevent updating jobs older than 24 hours???
# TODO(chrisphan): Add 'staled' state to bug_data and bisect_stats.
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'failed')
bisect_stats.UpdateBisectStats(self.bot, 'failed')
def SetCompleted(self):
logging.info('Updated status to completed')
self.status = 'completed'
self.put()
if self.bug_id:
bug_data.SetBisectStatus(self.bug_id, 'completed')
bisect_stats.UpdateBisectStats(self.bot, 'completed')
def GetCulpritCL(self):
if not self.results_data:
return None
# culprit_data can be undefined or explicitly set to None
culprit_data = self.results_data.get('culprit_data') or {}
return culprit_data.get('cl')
def GetConfigDict(self):
return json.loads(self.config.split('=', 1)[1])
def CheckFailureFromBuildBucket(self):
# Buildbucket job id is not always set.
if not self.buildbucket_job_id:
return
job_info = buildbucket_service.GetJobStatus(self.buildbucket_job_id)
data = job_info.get('build', {})
# Since the job is completed successfully, results_data must
# have been set appropriately by the bisector.
# The buildbucket job's 'status' and 'result' fields are documented here:
# https://goto.google.com/bb_status
if data.get('status') == 'COMPLETED' and data.get('result') == 'SUCCESS':
return
# Proceed if the job failed or cancelled
logging.info('Job failed. Buildbucket id %s', self.buildbucket_job_id)
data['result_details'] = json.loads(data['result_details_json'])
# There are various failure and cancellation reasons for a buildbucket
# job to fail as listed in https://goto.google.com/bb_status.
job_updates = {
'failure_reason': (data.get('cancelation_reason') or
data.get('failure_reason')),
'buildbot_log_url': data.get('url')
}
details = data.get('result_details')
if details:
properties = details.get('properties')
if properties:
job_updates['bisect_bot'] = properties.get('buildername')
job_updates['extra_result_code'] = properties.get(
'extra_result_code')
bisect_config = properties.get('bisect_config')
if bisect_config:
job_updates['try_job_id'] = bisect_config.get('try_job_id')
job_updates['bug_id'] = bisect_config.get('bug_id')
job_updates['command'] = bisect_config.get('command')
job_updates['test_type'] = bisect_config.get('test_type')
job_updates['metric'] = bisect_config.get('metric')
job_updates['good_revision'] = bisect_config.get('good_revision')
job_updates['bad_revision'] = bisect_config.get('bad_revision')
if not self.results_data:
self.results_data = {}
self.results_data.update(job_updates)
self.status = 'failed'
self.last_ran_timestamp = datetime.datetime.fromtimestamp(
float(data['updated_ts'])/1000000)
self.put()
logging.info('updated status to failed.')
