def testGetCurrentCheckingPointForTreeWithExistingData(self):
TreeClosure(tree_name='c', closed_time=datetime(2017, 04, 10, 10, 10)).put()
TreeClosure(tree_name='c',
closed_time=datetime(2017, 04, 11, 05, 05),
opened_time=datetime(2017, 04, 11, 05, 15)).put()
expected_checking_point = datetime(2017, 04, 11, 05, 15)
checking_point = collect_tree_closures._GetCurrentCheckingPointForTree('c')
self.assertEqual(expected_checking_point, checking_point)
def _GetCurrentCheckingPointForTree(tree_name):
"""Returns the timestamp from which to start the current checking."""
query = TreeClosure.query(
TreeClosure.tree_name == tree_name).order(-TreeClosure.closed_time)
result = query.get()
if result:
return result.opened_time
now = time_util.GetUTCNow()
return now - timedelta(days=_DEFAULT_BACKLOG_DAYS)
def _GenerateTreeClosureMetrics(tree_name, step_name, start_date, end_date):
query = TreeClosure.query(
TreeClosure.tree_name == tree_name, TreeClosure.step_name == step_name,
TreeClosure.closed_time >= start_date, TreeClosure.closed_time < end_date)
all_closures = list(query) # Run the query and convert results to a list.
# Interesting closures are automatically closed but manually opened.
closures = filter(lambda c: c.auto_closed and not c.auto_opened, all_closures)
flakes = filter(lambda c: c.possible_flake, closures)
reverts = filter(lambda c: c.has_revert, closures)
return {
'total': len(all_closures),
'manually_closed_or_auto_opened': len(all_closures) - len(closures),
'flakes': len(flakes),
'reverts': len(reverts),
'others': len(closures) - len(flakes) - len(reverts),
}
def _CreateTreeClosure(tree_name, statuses, first_open_status):
"""Creates a TreeClosure based on the given statuses."""
closed_time = statuses[0].time
opened_time = first_open_status.time
latest_action_time = statuses[-1].time
auto_closed = statuses[0].automatic
auto_opened = first_open_status.automatic
possible_flake = False
has_revert = False
for s in statuses[1:]:
message = s.message.lower()
for signal in ('flake', 'flaky', 'bot failure'):
if signal in message:
possible_flake = True
if 'revert' in message:
has_revert = True
master_name, builder_name, build_id, step_name = _ExtractFailureInfo(
statuses[0].message)
key_str_id = '%s-%s' % (tree_name,
time_util.ConvertToTimestamp(closed_time))
return TreeClosure(
tree_name=tree_name,
statuses=statuses,
closed_time=closed_time,
opened_time=opened_time,
latest_action_time=latest_action_time,
auto_closed=auto_closed,
auto_opened=auto_opened,
possible_flake=possible_flake,
has_revert=has_revert,
master_name=master_name,
builder_name=builder_name,
build_id=build_id,
step_name=step_name,
key=ndb.Key(TreeClosure, key_str_id),
)
def testGenerateTreeClosureMetrics(self):
ndb.put_multi([
TreeClosure(
tree_name='skia',
closed_time=datetime(2017, 03, 15),
auto_closed=True,
auto_opened=False,
possible_flake=True,
has_revert=False,
step_name='compile',
),
TreeClosure(
tree_name='chromium',
closed_time=datetime(2017, 03, 15),
auto_closed=True,
auto_opened=False,
possible_flake=True,
has_revert=False,
step_name='bot_update',
),
TreeClosure(
tree_name='chromium',
closed_time=datetime(2017, 03, 15, 10, 10, 10),
auto_closed=True,
auto_opened=True,
possible_flake=False,
has_revert=False,
step_name='compile',
),
TreeClosure(
tree_name='chromium',
closed_time=datetime(2017, 03, 15, 17, 10, 10),
auto_closed=True,
auto_opened=False,
possible_flake=False,
has_revert=False,
step_name='compile',
),
TreeClosure(
tree_name='chromium',
closed_time=datetime(2017, 03, 15, 19, 10, 10),
auto_closed=True,
auto_opened=False,
possible_flake=True,
has_revert=False,
step_name='compile',
),
TreeClosure(
tree_name='chromium',
closed_time=datetime(2017, 03, 16, 21, 10, 10),
auto_closed=True,
auto_opened=False,
possible_flake=False,
has_revert=True,
step_name='compile',
),
TreeClosure(
tree_name='chromium',
closed_time=datetime(2017, 04, 15, 17, 10, 10),
auto_closed=True,
auto_opened=False,
possible_flake=False,
has_revert=False,
step_name='compile',
),
])