def testCreateSameMeasurementsForDifferentTokens(self):
test_path = 'test/path'
token1 = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
token1.PopulateMeasurements([test_path])
token2 = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
token2.PopulateMeasurements([test_path])
measurement1 = upload_completion_token.Measurement.get_by_id(
test_path, parent=token1.key)
measurement2 = upload_completion_token.Measurement.get_by_id(
test_path, parent=token2.key)
self.assertNotEqual(measurement1, measurement2)
def testMeasurementUpdateStateByIdAsync(self):
test_path = 'test/path'
token_id = str(uuid.uuid4())
target_state = upload_completion_token.State.COMPLETED
token_key = upload_completion_token.Token(id=token_id).put()
upload_completion_token.Measurement(id=test_path,
parent=token_key).put()
upload_completion_token.Measurement.UpdateStateByIdAsync(
None, token_id, target_state).wait()
upload_completion_token.Measurement.UpdateStateByIdAsync(
test_path, None, target_state).wait()
upload_completion_token.Measurement.UpdateStateByIdAsync(
'expired', token_id, target_state).wait()
measurement = upload_completion_token.Measurement.get_by_id(
test_path, parent=token_key)
self.assertNotEqual(measurement.state, target_state)
self.assertEqual(measurement.error_message, None)
upload_completion_token.Measurement.UpdateStateByIdAsync(
test_path, token_id, target_state).wait()
measurement = upload_completion_token.Measurement.get_by_id(
test_path, parent=token_key)
self.assertEqual(measurement.state, target_state)
self.assertEqual(measurement.error_message, None)
def testPutAndUpdate(self):
token_id = str(uuid.uuid4())
gcs_file = 'path/%s.gcs' % str(uuid.uuid4())
token = upload_completion_token.Token(
id=token_id, temporary_staging_file_path=gcs_file).put().get()
self.assertEqual(token_id, token.key.id())
self.assertEqual(gcs_file, token.temporary_staging_file_path)
self.assertEqual(token.state, upload_completion_token.State.PENDING)
# Sleep for 1 second, so update_time change is visible.
sleep_time = datetime.timedelta(seconds=1)
time.sleep(sleep_time.total_seconds())
new_state = upload_completion_token.State.PROCESSING
token.UpdateStateAsync(new_state).wait()
changed_token = upload_completion_token.Token.get_by_id(token_id)
self.assertEqual(token_id, changed_token.key.id())
self.assertEqual(gcs_file, changed_token.temporary_staging_file_path)
self.assertTrue(
(changed_token.update_time - changed_token.creation_time) >= sleep_time)
self.assertEqual(changed_token.state, new_state)
new_state = upload_completion_token.State.COMPLETED
changed_token.UpdateStateAsync(new_state).wait()
changed_token = upload_completion_token.Token.get_by_id(token_id)
self.assertEqual(changed_token.state, new_state)
def testCreateSameMeasurementsForDifferentTokens(self):
test_path = 'test/path'
token1 = upload_completion_token.Token(
id=str(uuid.uuid4())).put().get()
token1.AddMeasurement(test_path, True).wait()
token2 = upload_completion_token.Token(
id=str(uuid.uuid4())).put().get()
token2.AddMeasurement(test_path, True).wait()
measurement1 = upload_completion_token.Measurement.GetByPath(
test_path, token1.key.id())
measurement2 = upload_completion_token.Measurement.GetByPath(
test_path, token2.key.id())
self.assertNotEqual(measurement1, measurement2)
def testUpdateObjectState(self):
target_state = upload_completion_token.State.COMPLETED
upload_completion_token.Token.UpdateObjectState(None, target_state)
token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
upload_completion_token.Token.UpdateObjectState(token, target_state)
self.assertEqual(token.state, target_state)
def testUpdateObjectStateAsync_ErrorMessage(self):
target_state = upload_completion_token.State.FAILED
target_message = 'Some message'
token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
upload_completion_token.Token.UpdateObjectStateAsync(
token, target_state, target_message).wait()
self.assertEqual(token.state, target_state)
self.assertEqual(token.error_message, target_message)
def testMeasurementUpdateStateByIdAsync_ErrorMessageFaile(self):
test_path = 'test/path'
token_id = str(uuid.uuid4())
token_key = upload_completion_token.Token(id=token_id).put()
upload_completion_token.Measurement(id=test_path,
parent=token_key).put()
upload_completion_token.Measurement.UpdateStateByIdAsync(
test_path, token_id, upload_completion_token.State.FAILED,
'Some message').wait()
def testMeasurementUpdateStateByPathAsync_ErrorMessageFail(self):
test_path = 'test/path'
token_id = str(uuid.uuid4())
token_key = upload_completion_token.Token(id=token_id).put()
upload_completion_token.Measurement(id=str(uuid.uuid4()),
test_path=test_path,
token=token_key).put()
upload_completion_token.Measurement.UpdateStateByPathAsync(
test_path, token_id, upload_completion_token.State.COMPLETED,
'Some message').check_success()
def _CreateUploadCompletionToken(self, temporary_staging_file_path=None):
token_info = {
'token': str(uuid.uuid4()),
'file': temporary_staging_file_path,
}
token = upload_completion_token.Token(
id=token_info['token'],
temporary_staging_file_path=temporary_staging_file_path,
)
token.put()
logging.info('Upload completion token created. Token id: %s',
token_info['token'])
return token, token_info
def testGet_SuccessLimitedInfo(self):
token_id = str(uuid.uuid4())
token = upload_completion_token.Token(id=token_id).put().get()
token.AddMeasurement('Chromium/win7/suite/metric1', False).wait()
token.AddMeasurement('Chromium/win7/suite/metric2', True).wait()
expected = {
'token': token_id,
'file': None,
'created': str(token.creation_time),
'lastUpdated': str(token.update_time),
'state': 'PROCESSING',
}
response = self.GetLimitedInfoRequest(token_id)
self.assertEqual(response, expected)
def testStatusUpdateWithExpiredMeasurement(self):
token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
measurement1, measurement2 = token.PopulateMeasurements(
['test/1', 'test/2'])
measurement1.key.delete()
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
token.UpdateStateAsync(upload_completion_token.State.COMPLETED).wait()
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
measurement2.state = upload_completion_token.State.COMPLETED
measurement2.put()
self.assertEqual(token.state, upload_completion_token.State.COMPLETED)
def testStatusUpdateWithExpiredMeasurement(self):
token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
measurement1 = token.AddMeasurement('test/1', True).get_result()
measurement2 = token.AddMeasurement('test/2', False).get_result()
measurement1.key.delete()
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
token.UpdateState(upload_completion_token.State.COMPLETED)
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
measurement2.state = upload_completion_token.State.COMPLETED
measurement2.put()
self.assertEqual(token.state, upload_completion_token.State.COMPLETED)
def testMeasurementUpdateStateByIdAsync_ErrorMessage(self):
test_path = 'test/path'
token_id = str(uuid.uuid4())
target_state = upload_completion_token.State.FAILED
target_message = 'Some message'
token_key = upload_completion_token.Token(id=token_id).put()
upload_completion_token.Measurement(id=test_path,
parent=token_key).put()
upload_completion_token.Measurement.UpdateStateByIdAsync(
test_path, token_id, target_state, target_message).wait()
measurement = upload_completion_token.Measurement.get_by_id(
test_path, parent=token_key)
self.assertEqual(measurement.state, target_state)
self.assertEqual(measurement.error_message, target_message)
def testGet_SuccessWithTokenErrorMessage(self):
token_id = str(uuid.uuid4())
token = upload_completion_token.Token(
id=token_id,
state_=upload_completion_token.State.FAILED,
error_message='Some error').put().get()
expected = {
'token': token_id,
'file': None,
'created': str(token.creation_time),
'lastUpdated': str(token.update_time),
'state': 'FAILED',
'error_message': 'Some error',
}
response = self.GetFullInfoRequest(token_id)
self.assertEqual(response, expected)
def testGet_SuccessWithMeasurements(self):
token_id = str(uuid.uuid4())
test_path1 = 'Chromium/win7/suite/metric1'
test_path2 = 'Chromium/win7/suite/metric2'
token = upload_completion_token.Token(id=token_id).put().get()
measurement1 = token.AddMeasurement(test_path1, False).get_result()
measurement2 = token.AddMeasurement(test_path2, True).get_result()
measurement1.state = upload_completion_token.State.COMPLETED
measurement1.put()
measurement2.state = upload_completion_token.State.FAILED
measurement2.error_message = 'Some error'
measurement1.put()
expected = {
'token':
token_id,
'file':
None,
'created':
str(token.creation_time),
'lastUpdated':
str(token.update_time),
'state':
'PROCESSING',
'measurements': [
{
'name': test_path1,
'state': 'COMPLETED',
'monitored': False,
'lastUpdated': str(measurement1.update_time),
},
{
'name': test_path2,
'state': 'FAILED',
'error_message': 'Some error',
'monitored': True,
'lastUpdated': str(measurement2.update_time),
},
]
}
response = self.GetFullInfoRequest(token_id)
expected['measurements'].sort()
response['measurements'].sort()
self.assertEqual(response, expected)
def testGet_SuccessWithMeasurementsAndAssociatedHistogram(self):
owners_diagnostic = generic_set.GenericSet(['owner_name'])
owners_diagnostic.guid = str(uuid.uuid4())
histogram.SparseDiagnostic(id=owners_diagnostic.guid,
data=owners_diagnostic.AsDict(),
name=reserved_infos.OWNERS.name,
test=None,
start_revision=1,
end_revision=999).put().get()
hs = self.GetTestHistogram(owners_diagnostic).put().get()
token_id = str(uuid.uuid4())
test_path = 'Chromium/win7/suite/metric1'
token = upload_completion_token.Token(id=token_id).put().get()
measurement = token.AddMeasurement(test_path, True).get_result()
measurement.histogram = hs.key
measurement.put()
expected = {
'token':
token_id,
'file':
None,
'created':
str(token.creation_time),
'lastUpdated':
str(token.update_time),
'state':
'PROCESSING',
'measurements': [
{
'name': test_path,
'state': 'PROCESSING',
'monitored': True,
'lastUpdated': str(measurement.update_time),
},
]
}
response = json.loads(
self.testapp.get('/uploads/%s?additional_info=measurements' %
token_id,
status=200).body)
self.assertEqual(response, expected)
def _CreateHistogramWithMeasurementAndAdd(self, status=200):
token_id = str(uuid.uuid4())
test_path = 'Chromium/win7/suite/metric'
token = upload_completion_token.Token(id=token_id).put().get()
token.AddMeasurement(test_path, False).wait()
token.UpdateState(upload_completion_token.State.COMPLETED)
graph_data.Bot(key=ndb.Key('Master', 'Chromium', 'Bot', 'win7'),
internal_only=False).put()
params = [{
'data': TEST_HISTOGRAM,
'test_path': test_path,
'benchmark_description': None,
'revision': 123,
'token': token_id
}]
upload_data = json.dumps(params)
self.testapp.post('/add_histograms_queue', upload_data, status=status)
return upload_completion_token.Token.get_by_id(token_id)
def testGet_Success(self):
token_id = str(uuid.uuid4())
token = upload_completion_token.Token(
id=token_id, temporary_staging_file_path='file/path').put().get()
expected = {
'token': token_id,
'file': 'file/path',
'created': str(token.creation_time),
'lastUpdated': str(token.update_time),
'state': 'PENDING'
}
response = self.GetFullInfoRequest(token_id)
self.assertEqual(response, expected)
token.UpdateState(upload_completion_token.State.COMPLETED)
expected['state'] = 'COMPLETED'
expected['lastUpdated'] = str(token.update_time)
response = self.GetFullInfoRequest(token_id)
self.assertEqual(response, expected)
def testPostMultipleHistogram_MeasurementExpired(self, mock_log):
test_path1 = 'Chromium/win7/suite/metric1'
test_path2 = 'Chromium/win7/suite/metric2'
token_id = str(uuid.uuid4())
token = upload_completion_token.Token(id=token_id).put().get()
token.AddMeasurement(test_path1, False).wait()
measurement2 = token.AddMeasurement(test_path2, False).get_result()
token.UpdateState(upload_completion_token.State.COMPLETED)
measurement2.key.delete()
measurement2 = upload_completion_token.Measurement.GetByPath(
test_path2, token_id)
self.assertEqual(measurement2, None)
graph_data.Bot(key=ndb.Key('Master', 'Chromium', 'Bot', 'win7'),
internal_only=False).put()
params = [
{
'data': TEST_HISTOGRAM,
'test_path': test_path1,
'benchmark_description': None,
'revision': 123,
'token': token_id
},
{
'data': TEST_HISTOGRAM,
'test_path': test_path2,
'benchmark_description': None,
'revision': 5,
'token': token_id
},
]
upload_data = json.dumps(params)
self.testapp.post('/add_histograms_queue', upload_data)
token = upload_completion_token.Token.get_by_id(token_id)
self.assertEqual(token.state, upload_completion_token.State.COMPLETED)
mock_log.assert_called_once_with(
'Upload completion token measurement could not be found. '
'Token id: %s, measurement test path: %s', token_id, test_path2)
def testPostMultipleHistogram_MeasrementExpired(self):
test_path1 = 'Chromium/win7/suite/metric1'
test_path2 = 'Chromium/win7/suite/metric2'
token_id = str(uuid.uuid4())
token = upload_completion_token.Token(id=token_id).put().get()
_, measurement2 = token.PopulateMeasurements([test_path1, test_path2])
token.UpdateStateAsync(upload_completion_token.State.COMPLETED).wait()
measurement2.key.delete()
measurement2 = upload_completion_token.Measurement.get_by_id(
test_path2, parent=token.key)
self.assertEqual(measurement2, None)
graph_data.Bot(
key=ndb.Key('Master', 'Chromium', 'Bot', 'win7'),
internal_only=False).put()
params = [
{
'data': TEST_HISTOGRAM,
'test_path': test_path1,
'benchmark_description': None,
'revision': 123,
'token': token_id
},
{
'data': TEST_HISTOGRAM,
'test_path': test_path2,
'benchmark_description': None,
'revision': 5,
'token': token_id
},
]
upload_data = json.dumps(params)
self.testapp.post('/add_histograms_queue', upload_data)
token = upload_completion_token.Token.get_by_id(token_id)
self.assertEqual(token.state, upload_completion_token.State.COMPLETED)
def testStatusUpdateWithMeasurements(self):
token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
self.assertEqual(token.state, upload_completion_token.State.PENDING)
measurement1 = token.AddMeasurement('test/1', True).get_result()
measurement2 = token.AddMeasurement('test/2', False).get_result()
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
token.UpdateStateAsync(upload_completion_token.State.PROCESSING).wait()
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
measurement1.state = upload_completion_token.State.FAILED
measurement1.put()
self.assertEqual(token.state, upload_completion_token.State.PROCESSING)
token.UpdateStateAsync(upload_completion_token.State.COMPLETED).wait()
measurement2.state = upload_completion_token.State.COMPLETED
measurement2.put()
self.assertEqual(token.state, upload_completion_token.State.FAILED)
measurement1.state = upload_completion_token.State.COMPLETED
measurement1.put()
self.assertEqual(token.state, upload_completion_token.State.COMPLETED)
def testGet_SuccessWithMeasurements(self):
token_id = str(uuid.uuid4())
test_path1 = 'Chromium/win7/suite/metric1'
test_path2 = 'Chromium/win7/suite/metric2'
token = upload_completion_token.Token(id=token_id).put().get()
measurement1, _ = token.PopulateMeasurements([test_path1, test_path2])
measurement1.state = upload_completion_token.State.COMPLETED
measurement1.put()
expected = {
'token':
token_id,
'file':
None,
'created':
str(token.creation_time),
'lastUpdated':
str(token.update_time),
'state':
'PROCESSING',
'measurements': [
{
'name': test_path1,
'state': 'COMPLETED',
},
{
'name': test_path2,
'state': 'PROCESSING',
},
]
}
response = self.GetInfoRequest(token_id)
expected['measurements'].sort()
response['measurements'].sort()
self.assertEqual(response, expected)
def testPopulateMeasurementsMultipleTimes(self): token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get() token.PopulateMeasurements(['test/1', 'test/2']) token.PopulateMeasurements(['test/3', 'test/4'])
def testGet_SuccessWithMeasurementsDimentionsAssociatedHistogram(self):
owners_diagnostic = generic_set.GenericSet(['owner_name'])
commit_position_diagnostic = generic_set.GenericSet([123])
owners_diagnostic.guid = str(uuid.uuid4())
commit_position_diagnostic.guid = str(uuid.uuid4())
histogram.SparseDiagnostic(id=owners_diagnostic.guid,
data=owners_diagnostic.AsDict(),
name=reserved_infos.OWNERS.name,
test=None,
start_revision=1,
end_revision=999).put().get()
hs = self.GetTestHistogram(owners_diagnostic,
commit_position_diagnostic).put().get()
token_id = str(uuid.uuid4())
test_path = 'Chromium/win7/suite/metric1'
token = upload_completion_token.Token(id=token_id).put().get()
measurement = token.AddMeasurement(test_path, True).get_result()
measurement.histogram = hs.key
measurement.put()
expected = {
'token':
token_id,
'file':
None,
'created':
str(token.creation_time),
'lastUpdated':
str(token.update_time),
'state':
'PROCESSING',
'measurements': [
{
'name':
test_path,
'state':
'PROCESSING',
'monitored':
True,
'lastUpdated':
str(measurement.update_time),
'dimensions': [
{
'name': reserved_infos.OWNERS.name,
'value': list(owners_diagnostic),
},
{
'name':
reserved_infos.CHROMIUM_COMMIT_POSITIONS.name,
'value': list(commit_position_diagnostic),
},
]
},
]
}
response = self.GetFullInfoRequest(token_id)
expected['measurements'][0]['dimensions'].sort()
response['measurements'][0]['dimensions'].sort()
self.assertEqual(response, expected)
def testGet_InvalidUser(self):
token_id = str(uuid.uuid4())
upload_completion_token.Token(id=token_id).put().get()
self.SetCurrentUser('*****@*****.**')
self.GetFullInfoRequest(token_id, status=403)
def testGet_SuccessWithMeasurementsAndAssociatedHistogram(self):
owners_diagnostic = generic_set.GenericSet(['owner_name'])
commit_position_diagnostic = generic_set.GenericSet([123])
irrelevant_diagnostic = generic_set.GenericSet([42])
owners_diagnostic.guid = str(uuid.uuid4())
commit_position_diagnostic.guid = str(uuid.uuid4())
histogram.SparseDiagnostic(id=owners_diagnostic.guid,
data=owners_diagnostic.AsDict(),
name=reserved_infos.OWNERS.name,
test=None,
start_revision=1,
end_revision=999).put().get()
hs = histogram.Histogram(
id=str(uuid.uuid4()),
data={
'allBins': {
'1': [1],
'3': [1],
'4': [1]
},
'binBoundaries': [1, [1, 1000, 20]],
'diagnostics': {
reserved_infos.CHROMIUM_COMMIT_POSITIONS.name:
commit_position_diagnostic.AsDict(),
reserved_infos.OWNERS.name:
owners_diagnostic.guid,
'irrelevant_diagnostic':
irrelevant_diagnostic.AsDict(),
},
'name': 'foo',
'running': [3, 3, 0.5972531564093516, 2, 1, 6, 2],
'sampleValues': [1, 2, 3],
'unit': 'count_biggerIsBetter'
},
test=None,
revision=123,
internal_only=True).put().get()
token_id = str(uuid.uuid4())
test_path = 'Chromium/win7/suite/metric1'
token = upload_completion_token.Token(id=token_id).put().get()
measurement = token.AddMeasurement(test_path, True).get_result()
measurement.histogram = hs.key
measurement.put()
expected = {
'token':
token_id,
'file':
None,
'created':
str(token.creation_time),
'lastUpdated':
str(token.update_time),
'state':
'PROCESSING',
'measurements': [
{
'name':
test_path,
'state':
'PROCESSING',
'monitored':
True,
'lastUpdated':
str(measurement.update_time),
'dimensions': [
{
'name': reserved_infos.OWNERS.name,
'value': list(owners_diagnostic),
},
{
'name':
reserved_infos.CHROMIUM_COMMIT_POSITIONS.name,
'value': list(commit_position_diagnostic),
},
]
},
]
}
response = self.GetFullInfoRequest(token_id)
expected['measurements'][0]['dimensions'].sort()
response['measurements'][0]['dimensions'].sort()
self.assertEqual(response, expected)
def testUpdateObjectStateAsyncWith_ErrorMessageFail(self):
token = upload_completion_token.Token(id=str(uuid.uuid4())).put().get()
upload_completion_token.Token.UpdateObjectStateAsync(
token, upload_completion_token.State.FAILED,
'Some message').wait()
