def testStoryRunSucceeded(self):
run = story_run.StoryRun(self.stories[0])
self.assertTrue(run.ok)
self.assertFalse(run.failed)
self.assertFalse(run.skipped)
run = story_run.StoryRun(self.stories[0])
run.AddValue(scalar.ScalarValue(self.stories[0], 'a', 's', 1))
self.assertTrue(run.ok)
self.assertFalse(run.failed)
self.assertFalse(run.skipped)
def testStoryRunSucceeded(self):
run = story_run.StoryRun(self.story)
self.assertTrue(run.ok)
self.assertFalse(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, None)
run = story_run.StoryRun(self.story)
self.assertTrue(run.ok)
self.assertFalse(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, None)
def testStoryRunFailed(self):
run = story_run.StoryRun(self.stories[0])
run.AddValue(failure.FailureValue.FromMessage(self.stories[0], 'test'))
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
run = story_run.StoryRun(self.stories[0])
run.AddValue(scalar.ScalarValue(self.stories[0], 'a', 's', 1))
run.AddValue(failure.FailureValue.FromMessage(self.stories[0], 'test'))
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
def testStoryRunFailed(self):
run = story_run.StoryRun(self.story)
run.SetFailed('abc')
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, 'abc')
run = story_run.StoryRun(self.story)
run.SetFailed('something is wrong')
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, 'something is wrong')
def testStoryRunSucceeded(self):
run = story_run.StoryRun(self.story)
self.assertTrue(run.ok)
self.assertFalse(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, None)
run = story_run.StoryRun(self.story)
run.AddValue(scalar.ScalarValue(
self.story, 'a', 's', 1,
improvement_direction=improvement_direction.UP))
self.assertTrue(run.ok)
self.assertFalse(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, None)
def WillRunPage(self, page, story_run_index=0):
assert not self.finalized, 'Results are finalized, cannot run more stories.'
assert not self._current_story_run, 'Did not call DidRunPage.'
self._current_story_run = story_run.StoryRun(
page, test_prefix=self.benchmark_name, index=story_run_index,
intermediate_dir=self._intermediate_dir)
self._progress_reporter.WillRunStory(self)
def WillRunPage(self, page, storyset_repeat_counter=0,
story_repeat_counter=0):
assert not self._current_page_run, 'Did not call DidRunPage.'
self._current_page_run = story_run.StoryRun(page)
self._progress_reporter.WillRunPage(self)
self.iteration_info.WillRunStory(
page, storyset_repeat_counter, story_repeat_counter)
def CreateStoryRun(self, story, story_run_index=0):
"""A context manager to delimit the capture of results for a new story run.
Args:
story: The story to be run.
story_run_index: An optional integer indicating the number of times this
same story has been already run as part of the same benchmark run.
"""
assert not self.finalized, 'Results are finalized, cannot run more stories.'
assert not self._current_story_run, 'Already running a story'
self._current_story_run = story_run.StoryRun(
story,
test_prefix=self.benchmark_name,
index=story_run_index,
intermediate_dir=self._intermediate_dir)
artifact_logger.RegisterArtifactImplementation(self._current_story_run)
try:
with self.CreateArtifact(DIAGNOSTICS_NAME) as f:
json.dump({'diagnostics': self._diagnostics}, f, indent=4)
self._progress_reporter.WillRunStory(self._current_story_run)
yield self._current_story_run
finally:
self._current_story_run.Finish()
self._WriteJsonLine(self._current_story_run.AsDict())
self._progress_reporter.DidRunStory(self._current_story_run)
self._all_story_runs.append(self._current_story_run)
self._current_story_run = None
def testAddMeasurements(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
run = story_run.StoryRun(self.story, intermediate_dir=tempdir)
run.AddMeasurement('run_time', 'ms', [1, 2, 3])
run.AddMeasurement('foo_bars',
'count',
4,
description='number of foo_bars found')
run.Finish()
artifact = run.GetArtifact(story_run.MEASUREMENTS_NAME)
with open(artifact.local_path) as f:
measurements = json.load(f)
self.assertEqual(
measurements, {
'measurements': [{
'name': 'run_time',
'unit': 'ms',
'samples': [1, 2, 3],
}, {
'name': 'foo_bars',
'unit': 'count',
'samples': [4],
'description': 'number of foo_bars found'
}]
})
def testStoryRunFailed(self):
run = story_run.StoryRun(self.story)
run.SetFailed('abc')
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, 'abc')
run = story_run.StoryRun(self.story)
run.AddValue(scalar.ScalarValue(
self.story, 'a', 's', 1,
improvement_direction=improvement_direction.UP))
run.SetFailed('something is wrong')
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
self.assertEquals(run.failure_str, 'something is wrong')
def WillRunPage(self, page, story_run_index=0):
assert not self._current_story_run, 'Did not call DidRunPage.'
self._current_story_run = story_run.StoryRun(
page,
test_prefix=self.benchmark_name,
index=story_run_index,
output_dir=self._output_dir)
self._progress_reporter.WillRunStory(self)
def testStoryRunSkipped(self):
run = story_run.StoryRun(self.story)
run.SetFailed('oops')
run.Skip('test', is_expected=True)
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
self.assertTrue(run.is_expected)
self.assertEquals(run.failure_str, 'oops')
run = story_run.StoryRun(self.story)
run.Skip('test', is_expected=False)
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
self.assertFalse(run.is_expected)
self.assertEquals(run.failure_str, None)
def testStoryRunFailed(self):
run = story_run.StoryRun(self.stories[0])
run.SetFailed()
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
run = story_run.StoryRun(self.stories[0])
run.AddValue(
scalar.ScalarValue(self.stories[0],
'a',
's',
1,
improvement_direction=improvement_direction.UP))
run.SetFailed()
self.assertFalse(run.ok)
self.assertTrue(run.failed)
self.assertFalse(run.skipped)
def testCreateArtifact(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
run = story_run.StoryRun(self.story, tempdir)
with run.CreateArtifact('logs') as log_file:
log_file.write('hi\n')
filename = run.GetArtifact('logs').local_path
with open(filename) as f:
self.assertEqual(f.read(), 'hi\n')
def WillRunPage(self, page, story_run_index=0):
assert not self.finalized, 'Results are finalized, cannot run more stories.'
assert not self._current_story_run, 'Did not call DidRunPage.'
self._current_story_run = story_run.StoryRun(
page, test_prefix=self.benchmark_name, index=story_run_index,
intermediate_dir=self._intermediate_dir)
with self.CreateArtifact(DIAGNOSTICS_NAME) as f:
json.dump({'diagnostics': self._diagnostics}, f, indent=4)
self._progress_reporter.WillRunStory(self)
def testCreateArtifact(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
run = story_run.StoryRun(self.story, intermediate_dir=tempdir)
with run.CreateArtifact('logs.txt') as log_file:
log_file.write('hi\n')
artifact = run.GetArtifact('logs.txt')
with open(artifact.local_path) as f:
self.assertEqual(f.read(), 'hi\n')
self.assertEqual(artifact.content_type, 'text/plain')
def testStoryRunSkipped(self):
run = story_run.StoryRun(self.stories[0])
run.AddValue(failure.FailureValue.FromMessage(self.stories[0], 'test'))
run.AddValue(skip.SkipValue(self.stories[0], 'test'))
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
run = story_run.StoryRun(self.stories[0])
run.AddValue(
scalar.ScalarValue(self.stories[0],
'a',
's',
1,
improvement_direction=improvement_direction.UP))
run.AddValue(skip.SkipValue(self.stories[0], 'test'))
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
def testCreateArtifact(self):
with tempfile_ext.NamedTemporaryDirectory(
prefix='artifact_tests') as tempdir:
run = story_run.StoryRun(self.story, tempdir)
with run.CreateArtifact('logs', '', '') as log_file:
filename = log_file.name
log_file.write('hi\n')
with open(filename) as f:
self.assertEqual(f.read(), 'hi\n')
def testStoryRunSkipped(self):
run = story_run.StoryRun(self.story)
run.SetFailed('oops')
run.Skip('test', is_expected=True)
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
self.assertTrue(run.is_expected)
self.assertEquals(run.failure_str, 'oops')
run = story_run.StoryRun(self.story)
run.AddValue(scalar.ScalarValue(
self.story, 'a', 's', 1,
improvement_direction=improvement_direction.UP))
run.Skip('test', is_expected=False)
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
self.assertFalse(run.is_expected)
self.assertEquals(run.failure_str, None)
def testAddArtifactBasic(self, make_patch, move_patch):
run = story_run.StoryRun(self.story, _abs_join('foo'))
run.AddArtifact('artifact_name',
_abs_join('foo', 'artifacts', 'bar.log'))
move_patch.assert_not_called()
make_patch.assert_called_with(_abs_join('foo', 'artifacts'))
self.assertEqual(run._artifacts, {
'artifact_name': os.path.join('artifacts', 'bar.log'),
})
def testAsDict(self, time_module):
time_module.time.side_effect = [1234567890.987, 1234567900.987]
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
run = story_run.StoryRun(
story=TestStory(name='http://example.com'),
test_prefix='benchmark',
intermediate_dir=tempdir)
with run.CreateArtifact('logs.txt') as log_file:
log_file.write('hello\n')
run.SetTbmMetrics(['metric1', 'metric2'])
run.Finish()
entry = run.AsDict()
self.assertEqual(
entry, {
'testResult': {
'testPath':
'benchmark/http%3A%2F%2Fexample.com',
'status':
'PASS',
'isExpected':
True,
'startTime':
'2009-02-13T23:31:30.987000Z',
'runDuration':
'10.00s',
'artifacts': {
'logs.txt': {
'filePath': mock.ANY,
'contentType': 'text/plain'
}
},
'tags': [
{
'key': 'tbmv2',
'value': 'metric1'
},
{
'key': 'tbmv2',
'value': 'metric2'
},
],
}
})
# Log file is in the {intermediate_dir}/ directory, and file name
# extension is preserved.
logs_file = entry['testResult']['artifacts']['logs.txt'][
'filePath']
intermediate_dir = os.path.join(tempdir, '')
self.assertTrue(logs_file.startswith(intermediate_dir))
self.assertTrue(logs_file.endswith('.txt'))
def testAddMeasurementValidation(self):
run = story_run.StoryRun(self.story)
with self.assertRaises(TypeError):
run.AddMeasurement(name=None, unit='ms', samples=[1, 2, 3])
with self.assertRaises(TypeError):
run.AddMeasurement(name='run_time', unit=7, samples=[1, 2, 3])
with self.assertRaises(TypeError):
run.AddMeasurement(name='run_time',
unit='ms',
samples=[1, None, 3])
with self.assertRaises(TypeError):
run.AddMeasurement(name='run_time',
unit='ms',
samples=[1, 2, 3],
description=['not', 'a', 'string'])
def testIterArtifacts(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
run = story_run.StoryRun(self.story, tempdir)
with run.CreateArtifact('log/log1'):
pass
with run.CreateArtifact('trace/trace1'):
pass
with run.CreateArtifact('trace/trace2'):
pass
all_artifacts = list(run.IterArtifacts())
self.assertEqual(3, len(all_artifacts))
logs = list(run.IterArtifacts('log'))
self.assertEqual(1, len(logs))
traces = list(run.IterArtifacts('trace'))
self.assertEqual(2, len(traces))
def testAsDict(self, time_module):
time_module.time.side_effect = [1234567890.987, 1234567900.987]
run = story_run.StoryRun(self.story)
run.AddValue(
scalar.ScalarValue(self.story,
'a',
's',
1,
improvement_direction=improvement_direction.UP))
run.Finish()
self.assertEquals(
run.AsDict(), {
'testRun': {
'testName': 'foo',
'status': 'PASS',
'startTime': '2009-02-13T23:31:30.987000Z',
'endTime': '2009-02-13T23:31:40.987000Z'
}
})
def testAsDict(self, time_module):
time_module.time.side_effect = [1234567890.987,
1234567900.987]
run = story_run.StoryRun(
story=TestStory(name='http://example.com'), test_prefix='benchmark')
run.AddValue(scalar.ScalarValue(
self.story, 'a', 's', 1,
improvement_direction=improvement_direction.UP))
run.Finish()
self.assertEquals(
run.AsDict(),
{
'testResult': {
'testPath': 'benchmark/http%3A%2F%2Fexample.com',
'status': 'PASS',
'isExpected': True,
'startTime': '2009-02-13T23:31:30.987000Z',
'runDuration': '10.00s'
}
}
)
def testIterArtifacts(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
run = story_run.StoryRun(self.story, output_dir=tempdir)
with run.CreateArtifact('log/log1.foo'):
pass
with run.CreateArtifact('trace/trace1.json'):
pass
with run.CreateArtifact('trace/trace2.json'):
pass
all_artifacts = list(run.IterArtifacts())
self.assertEqual(3, len(all_artifacts))
logs = list(run.IterArtifacts('log'))
self.assertEqual(1, len(logs))
# Falls back to 'application/octet-stream' due to unknown extension.
self.assertEqual('application/octet-stream', logs[0].content_type)
traces = list(run.IterArtifacts('trace'))
self.assertEqual(2, len(traces))
self.assertTrue(all(t.content_type == 'application/json' for t in traces))
def WillRunPage(self, page):
assert not self._current_page_run, 'Did not call DidRunPage.'
self._current_page_run = story_run.StoryRun(page)
self._progress_reporter.WillRunPage(self)
def testAddMeasurementTwiceRaises(self):
run = story_run.StoryRun(self.story)
run.AddMeasurement('foo_bars', 'ms', [1])
with self.assertRaises(AssertionError):
run.AddMeasurement('foo_bars', 'ms', [2])
def testAddMeasurementRaisesAfterFinish(self):
run = story_run.StoryRun(self.story)
run.AddMeasurement('run_time', 'ms', [1, 2, 3])
run.Finish()
with self.assertRaises(AssertionError):
run.AddMeasurement('foo_bars', 'count', 4)
def WillRunPage(self, page, storyset_repeat_counter=0):
assert not self._current_page_run, 'Did not call DidRunPage.'
self._current_page_run = story_run.StoryRun(page, self._output_dir)
self._progress_reporter.WillRunPage(self)
self.telemetry_info.WillRunStory(page, storyset_repeat_counter)
