def test_same_signature_multiple_performance_frameworks(
test_repository, perf_job):
framework_names = ["some-perf-framework", "another-perf-framework"]
for framework_name in framework_names:
PerformanceFramework.objects.create(name=framework_name, enabled=True)
datum = sample_perf_datum(framework_name)
# the perf data adapter expects deserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps(
{'performance_data': submit_datum['blob']})
store_performance_artifact(perf_job, submit_datum)
# we should have 2 performance signature objects, one for each framework
# and one datum for each signature
for framework_name in framework_names:
s = PerformanceSignature.objects.get(
framework__name=framework_name,
repository=test_repository,
suite="some-perf-suite",
test="some-perf-test",
)
d = PerformanceDatum.objects.get(signature=s)
assert d.value == 20.0
def _generate_perf_data_range(test_repository,
generic_reference_data,
create_perf_framework=True,
enable_framework=True,
add_suite_value=False,
extra_suite_metadata=None,
extra_subtest_metadata=None,
reverse_push_range=False):
framework_name = "cheezburger"
if create_perf_framework:
PerformanceFramework.objects.create(name=framework_name, enabled=enable_framework)
now = int(time.time())
push_range = range(30)
if reverse_push_range:
push_range = reversed(push_range)
for (i, value) in zip(push_range, [1]*15 + [2]*15):
push_time = datetime.datetime.fromtimestamp(now+i)
push = Push.objects.create(
repository=test_repository,
revision='abcdefgh%s' % i,
author='*****@*****.**',
time=push_time)
job = create_generic_job('myguid%s' % i, test_repository,
push.id, generic_reference_data)
datum = {
'job_guid': 'fake_job_guid',
'name': 'test',
'type': 'test',
'blob': {
'framework': {'name': framework_name},
'suites': [
{
'name': 'cheezburger metrics',
'subtests': [
{
'name': 'test1',
'value': value
}
]
}
]
}
}
if add_suite_value:
datum['blob']['suites'][0]['value'] = value
if extra_suite_metadata:
datum['blob']['suites'][0].update(extra_suite_metadata)
if extra_subtest_metadata:
datum['blob']['suites'][0]['subtests'][0].update(
extra_subtest_metadata)
# the perf data adapter expects unserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps({
'performance_data': submit_datum['blob']
})
store_performance_artifact(job, submit_datum)
def test_measurement_unit_can_be_updated(test_repository, later_perf_push,
perf_job, generic_reference_data,
sample_perf_artifact,
sample_perf_artifact_with_new_unit):
_, submit_datum = _prepare_test_data(sample_perf_artifact)
store_performance_artifact(perf_job, submit_datum)
_, updated_submit_datum = _prepare_test_data(
sample_perf_artifact_with_new_unit)
later_job = create_generic_job('lateguid', test_repository,
later_perf_push.id, generic_reference_data)
store_performance_artifact(later_job, updated_submit_datum)
summary_signature = PerformanceSignature.objects.get(
suite='cheezburger metrics', test='')
updated_subtest_signature = PerformanceSignature.objects.get(
suite='cheezburger metrics', test='test1')
assert summary_signature.measurement_unit == UPDATED_MEASUREMENT_UNIT
assert updated_subtest_signature.measurement_unit == UPDATED_MEASUREMENT_UNIT
# no side effects when parent/sibling signatures
# change measurement units
not_changed_subtest_signature = PerformanceSignature.objects.get(
suite='cheezburger metrics', test='test2')
assert not_changed_subtest_signature.measurement_unit == MEASUREMENT_UNIT
def test_ingest_workflow(test_repository, perf_push, later_perf_push, perf_job,
generic_reference_data, sample_perf_artifact):
perf_datum, submit_datum = _prepare_test_data(sample_perf_artifact)
store_performance_artifact(perf_job, submit_datum)
assert 8 == PerformanceSignature.objects.all().count()
assert 1 == PerformanceFramework.objects.all().count()
framework = PerformanceFramework.objects.first()
assert FRAMEWORK_NAME == framework.name
for suite in perf_datum['suites']:
# verify summary, then subtests
_verify_signature(test_repository.name,
perf_datum['framework']['name'], suite['name'], '',
'my_option_hash', 'my_platform',
suite.get('lowerIsBetter',
True), suite.get('extraOptions'),
suite.get('unit'), perf_push.time)
_verify_datum(suite['name'], '', suite['value'], perf_push.time)
for subtest in suite['subtests']:
_verify_signature(test_repository.name,
perf_datum['framework']['name'], suite['name'],
subtest['name'], 'my_option_hash', 'my_platform',
subtest.get('lowerIsBetter',
True), suite.get('extraOptions'),
suite.get('unit'), perf_push.time)
_verify_datum(suite['name'], subtest['name'], subtest['value'],
perf_push.time)
def store_job_artifacts(artifact_data):
"""
Store a list of job artifacts. All of the datums in artifact_data need
to be in the following format:
{
'type': 'json',
'name': 'my-artifact-name',
# blob can be any kind of structured data
'blob': { 'stuff': [1, 2, 3, 4, 5] },
'job_guid': 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
}
"""
for artifact in artifact_data:
# Determine what type of artifact we have received
if artifact:
artifact_name = artifact.get('name')
if not artifact_name:
logger.error(
"load_job_artifacts: Unnamed job artifact, skipping")
continue
job_guid = artifact.get('job_guid')
if not job_guid:
logger.error(
"load_job_artifacts: Artifact '%s' with no "
"job guid set, skipping",
artifact_name,
)
continue
try:
job = Job.objects.get(guid=job_guid)
except Job.DoesNotExist:
logger.error('load_job_artifacts: No job_id for guid %s',
job_guid)
continue
if artifact_name == 'performance_data':
store_performance_artifact(job, artifact)
elif artifact_name == 'text_log_summary':
try:
store_text_log_summary_artifact(job, artifact)
except IntegrityError:
logger.warning(
"Couldn't insert text log information "
"for job with guid %s, this probably "
"means the job was already parsed",
job_guid,
)
else:
logger.warning(
"Unknown artifact type: %s submitted with job %s",
artifact_name, job.guid)
else:
logger.error(
'store_job_artifacts: artifact type %s not understood',
artifact_name)
def store_job_artifacts(artifact_data):
"""
Store a list of job artifacts. All of the datums in artifact_data need
to be in the following format:
{
'type': 'json',
'name': 'my-artifact-name',
# blob can be any kind of structured data
'blob': { 'stuff': [1, 2, 3, 4, 5] },
'job_guid': 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
}
"""
for artifact in artifact_data:
# Determine what type of artifact we have received
if artifact:
artifact_name = artifact.get('name')
if not artifact_name:
logger.error("load_job_artifacts: Unnamed job artifact, skipping")
continue
job_guid = artifact.get('job_guid')
if not job_guid:
logger.error("load_job_artifacts: Artifact '%s' with no "
"job guid set, skipping", artifact_name)
continue
try:
job = Job.objects.get(guid=job_guid)
except Job.DoesNotExist:
logger.error('load_job_artifacts: No job_id for guid %s', job_guid)
continue
if artifact_name == 'performance_data':
store_performance_artifact(job, artifact)
elif artifact_name == 'Job Info':
store_job_info_artifact(job, artifact)
elif artifact_name == 'text_log_summary':
try:
store_text_log_summary_artifact(job, artifact)
except IntegrityError:
logger.warning("Couldn't insert text log information "
"for job with guid %s, this probably "
"means the job was already parsed",
job_guid)
elif artifact_name == 'buildapi':
buildbot_request_id = json.loads(artifact['blob']).get(
'request_id')
if buildbot_request_id:
JobDetail.objects.update_or_create(
job=job,
title='buildbot_request_id',
value=str(buildbot_request_id))
else:
logger.warning("Unknown artifact type: %s submitted with job %s",
artifact_name, job.guid)
else:
logger.error('store_job_artifacts: artifact type %s not understood', artifact_name)
def _generate_perf_data_range(test_repository,
generic_reference_data,
create_perf_framework=True,
enable_framework=True,
add_suite_value=False,
extra_suite_metadata=None,
extra_subtest_metadata=None,
reverse_push_range=False):
framework_name = "cheezburger"
if create_perf_framework:
PerformanceFramework.objects.create(name=framework_name,
enabled=enable_framework)
now = int(time.time())
push_range = range(30)
if reverse_push_range:
push_range = reversed(push_range)
for (i, value) in zip(push_range, [1] * 15 + [2] * 15):
push_time = datetime.datetime.fromtimestamp(now + i)
push = Push.objects.create(repository=test_repository,
revision='abcdefgh%s' % i,
author='*****@*****.**',
time=push_time)
job = create_generic_job('myguid%s' % i, test_repository, push.id,
generic_reference_data)
datum = {
'job_guid': 'fake_job_guid',
'name': 'test',
'type': 'test',
'blob': {
'framework': {
'name': framework_name
},
'suites': [{
'name': 'cheezburger metrics',
'subtests': [{
'name': 'test1',
'value': value
}]
}]
}
}
if add_suite_value:
datum['blob']['suites'][0]['value'] = value
if extra_suite_metadata:
datum['blob']['suites'][0].update(extra_suite_metadata)
if extra_subtest_metadata:
datum['blob']['suites'][0]['subtests'][0].update(
extra_subtest_metadata)
# the perf data adapter expects unserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps(
{'performance_data': submit_datum['blob']})
store_performance_artifact(job, submit_datum)
def test_hash_remains_unchanged(test_repository, perf_job, sample_perf_artifact):
_, submit_datum = _prepare_test_data(sample_perf_artifact)
store_performance_artifact(perf_job, submit_datum)
summary_signature = PerformanceSignature.objects.get(
suite='cheezburger metrics', test='')
# Ensure we don't inadvertently change the way we generate signature hashes.
assert summary_signature.signature_hash == 'f451f0c9000a7f99e5dc2f05792bfdb0e11d0cac'
subtest_signatures = PerformanceSignature.objects.filter(
parent_signature=summary_signature).values_list('signature_hash', flat=True)
assert len(subtest_signatures) == 3
def test_timestamp_can_be_updated(test_repository, perf_job, later_perf_push,
generic_reference_data,
sample_perf_artifact):
_, submit_datum = _prepare_test_data(sample_perf_artifact)
store_performance_artifact(perf_job, submit_datum)
# send another datum, a little later, verify that signature is changed accordingly
later_job = create_generic_job('lateguid', test_repository,
later_perf_push.id, generic_reference_data)
store_performance_artifact(later_job, submit_datum)
signature = PerformanceSignature.objects.get(suite='cheezburger metrics',
test='test1')
assert signature.last_updated == later_perf_push.time
def _generate_perf_data_range(
test_repository,
generic_reference_data,
create_perf_framework=True,
enable_framework=True,
suite_provides_value=False,
extra_suite_metadata=None,
extra_subtest_metadata=None,
reverse_push_range=False,
job_tier=None,
):
framework_name = "some-perf-framework"
if create_perf_framework:
PerformanceFramework.objects.create(name=framework_name,
enabled=enable_framework)
now = int(time.time())
push_range = range(30)
if reverse_push_range:
push_range = reversed(push_range)
for (i, value) in zip(push_range, [1] * 15 + [2] * 15):
push_time = datetime.datetime.fromtimestamp(now + i)
push = Push.objects.create(
repository=test_repository,
revision=f"abcdefgh{i}",
author="*****@*****.**",
time=push_time,
)
job = create_generic_job(f"myguid{i}",
test_repository,
push.id,
generic_reference_data,
tier=job_tier)
datum = sample_perf_datum(framework_name, value)
if suite_provides_value:
datum['blob']['suites'][0]['value'] = value
if extra_suite_metadata:
datum['blob']['suites'][0].update(extra_suite_metadata)
if extra_subtest_metadata:
datum['blob']['suites'][0]['subtests'][0].update(
extra_subtest_metadata)
# the perf data adapter expects deserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps(
{'performance_data': submit_datum['blob']})
store_performance_artifact(job, submit_datum)
def test_same_signature_multiple_performance_frameworks(
test_repository, perf_job):
framework_names = ['cheezburger1', 'cheezburger2']
for framework_name in framework_names:
PerformanceFramework.objects.create(name=framework_name, enabled=True)
datum = {
'job_guid': 'fake_job_guid',
'name': 'test',
'type': 'test',
'blob': {
'framework': {
'name': framework_name
},
'suites': [{
'name':
'cheezburger metrics',
'subtests': [{
'name': 'test1',
'value': 20.0,
'unit': 'ms'
}],
}],
},
}
# the perf data adapter expects unserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps(
{'performance_data': submit_datum['blob']})
store_performance_artifact(perf_job, submit_datum)
# we should have 2 performance signature objects, one for each framework
# and one datum for each signature
for framework_name in framework_names:
s = PerformanceSignature.objects.get(
framework__name=framework_name,
repository=test_repository,
suite='cheezburger metrics',
test='test1',
)
d = PerformanceDatum.objects.get(signature=s)
assert d.value == 20.0
def test_changing_extra_options_decouples_perf_signatures(
test_repository, later_perf_push, perf_job, generic_reference_data,
sample_perf_artifact):
updated_perf_artifact = copy.deepcopy(sample_perf_artifact)
updated_perf_artifact['blob']['suites'][0]['extraOptions'] = [
'different-extra-options'
]
later_job = create_generic_job('lateguid', test_repository,
later_perf_push.id, generic_reference_data)
_, submit_datum = _prepare_test_data(sample_perf_artifact)
_, updated_submit_datum = _prepare_test_data(updated_perf_artifact)
store_performance_artifact(perf_job, submit_datum)
initial_signature_amount = PerformanceSignature.objects.all().count()
store_performance_artifact(later_job, updated_submit_datum)
# Perfherder treats perf data with new properties as entirely new data.
# Thus, it creates new & separate signatures for them.
assert initial_signature_amount < PerformanceSignature.objects.all().count(
)
def test_same_signature_multiple_performance_frameworks(test_repository,
perf_job):
framework_names = ['cheezburger1', 'cheezburger2']
for framework_name in framework_names:
PerformanceFramework.objects.create(name=framework_name, enabled=True)
datum = {
'job_guid': 'fake_job_guid',
'name': 'test',
'type': 'test',
'blob': {
'framework': {'name': framework_name},
'suites': [
{
'name': 'cheezburger metrics',
'subtests': [
{
'name': 'test1',
'value': 20.0,
}
]
}
]
}
}
# the perf data adapter expects unserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps({
'performance_data': submit_datum['blob']
})
store_performance_artifact(perf_job, submit_datum)
# we should have 2 performance signature objects, one for each framework
# and one datum for each signature
for framework_name in framework_names:
s = PerformanceSignature.objects.get(framework__name=framework_name,
repository=test_repository,
suite='cheezburger metrics',
test='test1')
d = PerformanceDatum.objects.get(signature=s)
assert d.value == 20.0
def test_load_generic_data(test_repository, perf_push, perf_job,
generic_reference_data):
framework_name = 'cheezburger'
PerformanceFramework.objects.get_or_create(name=framework_name,
enabled=True)
datum = {
'job_guid': 'fake_job_guid',
'name': 'test',
'type': 'test',
'blob': {
'framework': {
'name': framework_name
},
'suites': [{
'name':
'cheezburger metrics',
'extraOptions': ['shell', 'e10s'],
'lowerIsBetter':
True,
'value':
10.0,
'subtests': [{
'name': 'test1',
'value': 20.0,
'lowerIsBetter': True
}, {
'name': 'test2',
'value': 30.0,
'lowerIsBetter': False
}, {
'name': 'test3',
'value': 40.0
}]
}, {
'name': 'cheezburger metrics 2',
'lowerIsBetter': False,
'value': 10.0,
'subtests': [{
'name': 'test1',
'value': 20.0
}]
}, {
'name': 'cheezburger metrics 3',
'value': 10.0,
'subtests': [{
'name': 'test1',
'value': 20.0
}]
}]
}
}
# the perf data adapter expects unserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps(
{'performance_data': submit_datum['blob']})
store_performance_artifact(perf_job, submit_datum)
assert 8 == PerformanceSignature.objects.all().count()
assert 1 == PerformanceFramework.objects.all().count()
framework = PerformanceFramework.objects.all()[0]
assert framework_name == framework.name
perf_datum = datum['blob']
for suite in perf_datum['suites']:
# verify summary, then subtests
_verify_signature(test_repository.name,
perf_datum['framework']['name'], suite['name'], '',
'my_option_hash', 'my_platform',
suite.get('lowerIsBetter', True),
suite.get('extraOptions'), perf_push.time)
_verify_datum(suite['name'], '', suite['value'], perf_push.time)
for subtest in suite['subtests']:
_verify_signature(test_repository.name,
perf_datum['framework']['name'], suite['name'],
subtest['name'], 'my_option_hash', 'my_platform',
subtest.get('lowerIsBetter', True),
suite.get('extraOptions'), perf_push.time)
_verify_datum(suite['name'], subtest['name'], subtest['value'],
perf_push.time)
summary_signature = PerformanceSignature.objects.get(
suite=perf_datum['suites'][0]['name'], test='')
subtest_signatures = PerformanceSignature.objects.filter(
parent_signature=summary_signature).values_list('signature_hash',
flat=True)
assert len(subtest_signatures) == 3
# send another datum, a little later, verify that signature's
# `last_updated` is changed accordingly
later_timestamp = datetime.datetime.fromtimestamp(int(time.time()) + 5)
later_push = Push.objects.create(repository=test_repository,
revision='1234abcd12',
author='*****@*****.**',
time=later_timestamp)
later_job = create_generic_job('lateguid', test_repository, later_push.id,
generic_reference_data)
store_performance_artifact(later_job, submit_datum)
signature = PerformanceSignature.objects.get(
suite=perf_datum['suites'][0]['name'],
test=perf_datum['suites'][0]['subtests'][0]['name'])
assert signature.last_updated == later_timestamp
def test_load_generic_data(test_repository,
perf_push, perf_job, generic_reference_data):
framework_name = 'cheezburger'
PerformanceFramework.objects.get_or_create(name=framework_name, enabled=True)
datum = {
'job_guid': 'fake_job_guid',
'name': 'test',
'type': 'test',
'blob': {
'framework': {'name': framework_name},
'suites': [
{
'name': 'cheezburger metrics',
'extraOptions': ['shell', 'e10s'],
'lowerIsBetter': True,
'value': 10.0,
'subtests': [
{
'name': 'test1',
'value': 20.0,
'lowerIsBetter': True
},
{
'name': 'test2',
'value': 30.0,
'lowerIsBetter': False
},
{
'name': 'test3',
'value': 40.0
}
]
},
{
'name': 'cheezburger metrics 2',
'lowerIsBetter': False,
'value': 10.0,
'subtests': [
{
'name': 'test1',
'value': 20.0
}
]
},
{
'name': 'cheezburger metrics 3',
'value': 10.0,
'subtests': [
{
'name': 'test1',
'value': 20.0
}
]
}
]
}
}
# the perf data adapter expects unserialized performance data
submit_datum = copy.copy(datum)
submit_datum['blob'] = json.dumps({
'performance_data': submit_datum['blob']
})
store_performance_artifact(perf_job, submit_datum)
assert 8 == PerformanceSignature.objects.all().count()
assert 1 == PerformanceFramework.objects.all().count()
framework = PerformanceFramework.objects.all()[0]
assert framework_name == framework.name
perf_datum = datum['blob']
for suite in perf_datum['suites']:
# verify summary, then subtests
_verify_signature(test_repository.name,
perf_datum['framework']['name'],
suite['name'],
'',
'my_option_hash',
'my_platform',
suite.get('lowerIsBetter', True),
suite.get('extraOptions'),
perf_push.time)
_verify_datum(suite['name'], '', suite['value'], perf_push.time)
for subtest in suite['subtests']:
_verify_signature(test_repository.name,
perf_datum['framework']['name'],
suite['name'],
subtest['name'],
'my_option_hash',
'my_platform',
subtest.get('lowerIsBetter', True),
suite.get('extraOptions'),
perf_push.time)
_verify_datum(suite['name'], subtest['name'], subtest['value'],
perf_push.time)
summary_signature = PerformanceSignature.objects.get(
suite=perf_datum['suites'][0]['name'], test='')
subtest_signatures = PerformanceSignature.objects.filter(
parent_signature=summary_signature).values_list('signature_hash', flat=True)
assert len(subtest_signatures) == 3
# send another datum, a little later, verify that signature's
# `last_updated` is changed accordingly
later_timestamp = datetime.datetime.fromtimestamp(int(time.time()) + 5)
later_push = Push.objects.create(
repository=test_repository,
revision='1234abcd12',
author='*****@*****.**',
time=later_timestamp)
later_job = create_generic_job('lateguid', test_repository,
later_push.id, 2, generic_reference_data)
store_performance_artifact(later_job, submit_datum)
signature = PerformanceSignature.objects.get(
suite=perf_datum['suites'][0]['name'],
test=perf_datum['suites'][0]['subtests'][0]['name'])
assert signature.last_updated == later_timestamp
def store_job_artifacts(artifact_data):
"""
Store a list of job artifacts. All of the datums in artifact_data need
to be in the following format:
{
'type': 'json',
'name': 'my-artifact-name',
# blob can be any kind of structured data
'blob': { 'stuff': [1, 2, 3, 4, 5] },
'job_guid': 'd22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33'
}
"""
for index, artifact in enumerate(artifact_data):
# Determine what type of artifact we have received
if artifact:
artifact_name = artifact.get('name')
if not artifact_name:
logger.error("load_job_artifacts: Unnamed job artifact, "
"skipping")
continue
job_guid = artifact.get('job_guid')
if not job_guid:
logger.error("load_job_artifacts: Artifact '{}' with no "
"job guid set, skipping".format(
artifact_name))
continue
try:
job = Job.objects.get(guid=job_guid)
except Job.DoesNotExist:
logger.error(
('load_job_artifacts: No job_id for '
'guid {}'.format(job_guid)))
continue
if artifact_name == 'performance_data':
store_performance_artifact(job, artifact)
elif artifact_name == 'Job Info':
store_job_info_artifact(job, artifact)
elif artifact_name == 'text_log_summary':
try:
store_text_log_summary_artifact(job, artifact)
except IntegrityError:
logger.warning("Couldn't insert text log information "
"for job with guid %s, this probably "
"means the job was already parsed",
job_guid)
elif artifact_name == 'buildapi':
buildbot_request_id = json.loads(artifact['blob']).get(
'request_id')
if buildbot_request_id:
JobDetail.objects.update_or_create(
job=job,
title='buildbot_request_id',
value=str(buildbot_request_id))
else:
logger.warning("Unknown artifact type: %s submitted with job %s",
artifact_name, job.guid)
else:
logger.error(
('store_job_artifacts: artifact type '
'{} not understood'.format(artifact_name)))
