def testCollectPinpointResults(self):
state = [
StateItem('a100', job1='completed', job2='completed'),
StateItem('a200', job3='completed', job4='running'),
StateItem('a300', job5='running', job6='running')
]
# Write some fake "previous" results for first revision.
df = pd.DataFrame({'revision': ['a100']})
df.to_csv(pinboard.RevisionResultsFile(state[0]), index=False)
self.subprocess.check_output.side_effect = [
'job4: completed\n', 'job5: running\njob6: failed\n',
'getting csv data ...\n'
]
expected_state = [
StateItem('a100', job1='completed', job2='completed'),
StateItem('a200', job3='completed', job4='completed'),
StateItem('a300', job5='running', job6='failed')
]
pinboard.CollectPinpointResults(state)
self.assertEqual(state, expected_state)
self.subprocess.check_output.assert_has_calls([
mock.call(['vpython', pinboard.PINPOINT_CLI, 'status', 'job4'],
universal_newlines=True),
mock.call(
['vpython', pinboard.PINPOINT_CLI, 'status', 'job5', 'job6'],
universal_newlines=True),
mock.call([
'vpython', pinboard.PINPOINT_CLI, 'get-csv', '--output',
pinboard.RevisionResultsFile(state[1]), '--', 'job3', 'job4'
])
])
def testGetRevisionResults_simple(self):
item = StateItem('2a66ba', timestamp='2019-03-17T23:50:16-07:00')
csv = [
'change,benchmark,story,name,unit,mean\n',
'2a66ba,loading,story1,Total:duration,ms_smallerIsBetter,300.0\n',
'2a66ba,loading,story2,Total:duration,ms_smallerIsBetter,400.0\n',
'2a66ba+patch,loading,story1,Total:duration,ms_smallerIsBetter,100.0\n',
'2a66ba+patch,loading,story2,Total:duration,ms_smallerIsBetter,200.0\n',
'2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0\n'
]
expected_results = [
('without_patch', 0.35, '2018-03-17T12:00:00'),
('with_patch', 0.15, '2019-03-17T12:00:00'),
]
filename = pinboard.RevisionResultsFile(item)
with open(filename, 'w') as f:
f.writelines(csv)
with mock.patch('cli_tools.pinboard.pinboard.ACTIVE_STORIES',
new=['story1', 'story2']):
df = pinboard.GetRevisionResults(item)
self.assertEqual(len(df.index), 2) # Only two rows of output.
self.assertTrue((df['revision'] == '2a66ba').all())
self.assertTrue((df['benchmark'] == 'loading').all())
self.assertTrue((df['name'] == 'Total:duration').all())
self.assertTrue((df['count'] == 2).all())
df = df.set_index('label', verify_integrity=True)
for label, value, timestamp in expected_results:
self.assertEqual(df.loc[label, 'mean'], value)
self.assertEqual(df.loc[label, 'timestamp'],
pd.Timestamp(timestamp))
def testGetRevisionResults_empty(self):
item = StateItem('2a66ba', timestamp='2019-03-17T23:50:16-07:00')
csv = [
'change,benchmark,story,name,unit,mean\n',
'2a66ba,loading,story1,Other:metric,count_smallerIsBetter,1.0\n']
filename = pinboard.RevisionResultsFile(item)
with open(filename, 'w') as f:
f.writelines(csv)
df = pinboard.GetRevisionResults(item)
self.assertEqual(len(df.index), 1) # Only one row of output.
row = df.iloc[0]
self.assertEqual(row['revision'], '2a66ba')
self.assertEqual(row['count'], 0)
