def test_alert_change_type_absolute(test_repository,
failure_classifications,
generic_reference_data,
test_perf_signature, new_value,
expected_num_alerts):
# modify the test signature to say that we alert on absolute value
# (as opposed to percentage change)
test_perf_signature.alert_change_type = PerformanceSignature.ALERT_ABS
test_perf_signature.alert_threshold = 0.3
test_perf_signature.save()
base_time = time.time() # generate it based off current time
INTERVAL = 30
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, 1, 0.5, INTERVAL/2)
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, (INTERVAL/2) + 1, new_value,
INTERVAL/2)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == expected_num_alerts
assert PerformanceAlertSummary.objects.count() == expected_num_alerts
def test_detect_alerts_in_series_with_retriggers(
test_repository, failure_classifications,
generic_reference_data, test_perf_signature):
# sometimes we detect an alert in the middle of a series
# where there are retriggers, make sure we handle this case
# gracefully by generating a sequence where the regression
# "appears" in the middle of a series with the same push
# to make sure things are calculated correctly
# (in this case, we're moving from consistent 0.5 to a 0.5/1.0
# mix)
base_time = time.time() # generate it based off current time
for i in range(20):
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, 1, 0.5, 1)
for i in range(5):
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, 2, 0.5, 1)
for i in range(15):
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, 2, 1.0, 1)
generate_new_alerts_in_series(test_perf_signature)
_verify_alert(1, 2, 1, test_perf_signature, 0.5, 0.875, True,
PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
def test_no_alerts_with_old_data(
test_repository,
test_issue_tracker,
failure_classifications,
generic_reference_data,
test_perf_signature,
):
base_time = 0 # 1970, too old!
INTERVAL = 30
_generate_performance_data(
test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time,
1,
0.5,
int(INTERVAL / 2),
)
_generate_performance_data(
test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time,
int(INTERVAL / 2) + 1,
1.0,
int(INTERVAL / 2),
)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 0
assert PerformanceAlertSummary.objects.count() == 0
def test_custom_alert_threshold(
test_repository, failure_classifications,
generic_reference_data, test_perf_signature):
test_perf_signature.alert_threshold = 200.0
test_perf_signature.save()
# under default settings, this set of data would generate
# 2 alerts, but we'll set an artificially high threshold
# of 200% that should only generate 1
INTERVAL = 60
base_time = time.time()
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, 1, 0.5, INTERVAL/3)
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, (INTERVAL/3) + 1, 0.6, INTERVAL/3)
_generate_performance_data(test_repository, test_perf_signature,
generic_reference_data,
base_time, 2*(INTERVAL/3) + 1, 2.0, INTERVAL/3)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
def test_custom_alert_threshold(
test_project, test_repository, test_perf_signature):
test_perf_signature.alert_threshold = 200.0
test_perf_signature.save()
# under default settings, this set of data would generate
# 2 alerts, but we'll set an artificially high threshold
# of 200% that should only generate 1
INTERVAL = 60
now = time.time()
for (t, v) in zip([i for i in range(INTERVAL)],
([0.5 for i in range(INTERVAL/3)] +
[0.6 for i in range(INTERVAL/3)] +
[2.0 for i in range(INTERVAL/3)])):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=t,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
def handle(self, *args, **options):
if not options['project']:
raise CommandError("Must specify at least one project with " "--project")
for project in options['project']:
repository = models.Repository.objects.get(name=project)
signatures = PerformanceSignature.objects.filter(repository=repository)
if options['signature']:
signatures_to_process = signatures.filter(signature_hash__in=options['signature'])
else:
hashes_to_ignore = set()
# if doing everything, only handle series which are not a
# subtest of another (we should alert only the parent series
# in that case)
for signature in signatures:
# Don't alert on subtests which have a summary
hashes_to_ignore.update(
signature.extra_properties.get('subtest_signatures', [])
)
signatures_to_process = [
signature
for signature in signatures
if signature.signature_hash not in hashes_to_ignore
]
for signature in signatures_to_process:
generate_new_alerts_in_series(signature)
def test_detect_alerts_in_series_with_retriggers(
test_project, test_repository, test_perf_signature):
# sometimes we detect an alert in the middle of a series
# where there are retriggers, make sure we handle this case
# gracefully by generating a sequence where the regression
# "appears" in the middle of a series with the same resultset
# to make sure things are calculated correctly
now = time.time()
for (t, j, v) in zip(
([1 for i in range(30)] +
[2 for i in range(60)]),
[i for i in range(90)],
([0.5 for i in range(50)] +
[1.0 for i in range(40)])
):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=j,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
_verify_alert(1, 2, 1, test_perf_signature, 0.5, 1.0, True,
PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
def test_custom_alert_threshold(test_repository, test_issue_tracker,
failure_classifications,
generic_reference_data, test_perf_signature):
test_perf_signature.alert_threshold = 200.0
test_perf_signature.save()
# under default settings, this set of data would generate
# 2 alerts, but we'll set an artificially high threshold
# of 200% that should only generate 1
INTERVAL = 60
base_time = time.time()
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 1, 0.5, int(INTERVAL / 3))
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time,
int(INTERVAL / 3) + 1, 0.6, int(INTERVAL / 3))
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 2 * int(INTERVAL / 3) + 1, 2.0,
int(INTERVAL / 3))
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
def handle(self, *args, **options):
if not options['project']:
raise CommandError("Must specify at least one project with "
"--project")
for project in options['project']:
repository = models.Repository.objects.get(name=project)
signatures = PerformanceSignature.objects.filter(
repository=repository)
if options['signature']:
signatures_to_process = signatures.filter(
signature_hash__in=options['signature'])
else:
hashes_to_ignore = set()
# if doing everything, only handle series which are not a
# subtest of another (we should alert only the parent series
# in that case)
for signature in signatures:
# Don't alert on subtests which have a summary
hashes_to_ignore.update(
signature.extra_properties.get('subtest_signatures',
[]))
signatures_to_process = [signature for signature in signatures
if signature.signature_hash not in
hashes_to_ignore]
for signature in signatures_to_process:
generate_new_alerts_in_series(signature)
def test_alert_change_type_absolute(test_repository, test_issue_tracker,
failure_classifications,
generic_reference_data,
test_perf_signature, new_value,
expected_num_alerts):
# modify the test signature to say that we alert on absolute value
# (as opposed to percentage change)
test_perf_signature.alert_change_type = PerformanceSignature.ALERT_ABS
test_perf_signature.alert_threshold = 0.3
test_perf_signature.save()
base_time = time.time() # generate it based off current time
INTERVAL = 30
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 1, 0.5, int(INTERVAL / 2))
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time,
int(INTERVAL / 2) + 1, new_value,
int(INTERVAL / 2))
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == expected_num_alerts
assert PerformanceAlertSummary.objects.count() == expected_num_alerts
def test_detect_alerts_in_series_with_retriggers(
test_project, test_repository, test_perf_signature):
# sometimes we detect an alert in the middle of a series
# where there are retriggers, make sure we handle this case
# gracefully by generating a sequence where the regression
# "appears" in the middle of a series with the same resultset
# to make sure things are calculated correctly
now = time.time()
for (t, j, v) in zip(
([1 for i in range(30)] +
[2 for i in range(60)]),
[i for i in range(90)],
([0.5 for i in range(50)] +
[1.0 for i in range(40)])
):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=j,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
_verify_alert(1, 2, 1, test_perf_signature, 0.5, 1.0, True)
def test_detect_alerts_in_series_with_retriggers(test_repository,
test_issue_tracker,
failure_classifications,
generic_reference_data,
test_perf_signature):
# sometimes we detect an alert in the middle of a series
# where there are retriggers, make sure we handle this case
# gracefully by generating a sequence where the regression
# "appears" in the middle of a series with the same push
# to make sure things are calculated correctly
# (in this case, we're moving from consistent 0.5 to a 0.5/1.0
# mix)
base_time = time.time() # generate it based off current time
for i in range(20):
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 1, 0.5, 1)
for i in range(5):
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 2, 0.5, 1)
for i in range(15):
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 2, 1.0, 1)
generate_new_alerts_in_series(test_perf_signature)
_verify_alert(1, 2, 1, test_perf_signature, 0.5, 0.875, True,
PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
def test_no_alerts_with_old_data(
test_project, test_repository, test_perf_signature):
base_time = 0 # 1970, too old!
INTERVAL = 30
_generate_performance_data(test_repository, test_perf_signature,
base_time, 1, 0.5, INTERVAL/2)
_generate_performance_data(test_repository, test_perf_signature,
base_time, (INTERVAL/2) + 1, 1.0, INTERVAL/2)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 0
assert PerformanceAlertSummary.objects.count() == 0
def test_no_alerts_with_old_data(test_project, test_repository,
test_perf_signature):
base_time = 0 # 1970, too old!
INTERVAL = 30
_generate_performance_data(test_repository, test_perf_signature, base_time,
1, 0.5, INTERVAL / 2)
_generate_performance_data(test_repository, test_perf_signature, base_time,
(INTERVAL / 2) + 1, 1.0, INTERVAL / 2)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 0
assert PerformanceAlertSummary.objects.count() == 0
def test_no_alerts_with_old_data(
test_project, test_repository, test_perf_signature):
INTERVAL = 30
for (t, v) in zip([i for i in range(INTERVAL)],
([0.5 for i in range(INTERVAL/2)] +
[1.0 for i in range(INTERVAL/2)])):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=t,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 0
assert PerformanceAlertSummary.objects.count() == 0
def test_no_alerts_with_old_data(
test_project, test_repository, test_perf_signature):
INTERVAL = 30
for (t, v) in zip([i for i in range(INTERVAL)],
([0.5 for i in range(INTERVAL/2)] +
[1.0 for i in range(INTERVAL/2)])):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=t,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 0
assert PerformanceAlertSummary.objects.count() == 0
def test_no_alerts_with_old_data(
test_repository, test_issue_tracker,
failure_classifications, generic_reference_data, test_perf_signature):
base_time = 0 # 1970, too old!
INTERVAL = 30
_generate_performance_data(test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time, 1, 0.5, int(INTERVAL / 2))
_generate_performance_data(test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time, int(INTERVAL / 2) + 1, 1.0, int(INTERVAL / 2))
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 0
assert PerformanceAlertSummary.objects.count() == 0
def test_custom_alert_threshold(test_project, test_repository,
test_perf_signature, jm):
test_perf_signature.alert_threshold = 200.0
test_perf_signature.save()
# under default settings, this set of data would generate
# 2 alerts, but we'll set an artificially high threshold
# of 200% that should only generate 1
INTERVAL = 60
base_time = time.time()
_generate_performance_data(test_repository, test_perf_signature, base_time,
1, 0.5, INTERVAL / 3)
_generate_performance_data(test_repository, test_perf_signature, base_time,
(INTERVAL / 3) + 1, 0.6, INTERVAL / 3)
_generate_performance_data(test_repository, test_perf_signature, base_time,
2 * (INTERVAL / 3) + 1, 2.0, INTERVAL / 3)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
def test_detect_alerts_in_series_with_retriggers(test_project, test_repository,
test_perf_signature):
# sometimes we detect an alert in the middle of a series
# where there are retriggers, make sure we handle this case
# gracefully by generating a sequence where the regression
# "appears" in the middle of a series with the same push
# to make sure things are calculated correctly
base_time = time.time() # generate it based off current time
for i in range(30):
_generate_performance_data(test_repository, test_perf_signature,
base_time, 1, 0.5, 1)
for i in range(20):
_generate_performance_data(test_repository, test_perf_signature,
base_time, 2, 0.5, 1)
for i in range(40):
_generate_performance_data(test_repository, test_perf_signature,
base_time, 2, 1.0, 1)
generate_new_alerts_in_series(test_perf_signature)
_verify_alert(1, 2, 1, test_perf_signature, 0.5, 1.0, True,
PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
def test_detect_alerts_in_series_with_retriggers(
test_project, test_repository, test_perf_signature):
# sometimes we detect an alert in the middle of a series
# where there are retriggers, make sure we handle this case
# gracefully by generating a sequence where the regression
# "appears" in the middle of a series with the same push
# to make sure things are calculated correctly
base_time = time.time() # generate it based off current time
for i in range(30):
_generate_performance_data(test_repository, test_perf_signature,
base_time, 1, 0.5, 1)
for i in range(20):
_generate_performance_data(test_repository, test_perf_signature,
base_time, 2, 0.5, 1)
for i in range(40):
_generate_performance_data(test_repository, test_perf_signature,
base_time, 2, 1.0, 1)
generate_new_alerts_in_series(test_perf_signature)
_verify_alert(1, 2, 1, test_perf_signature, 0.5, 1.0, True,
PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
def test_detect_alerts_in_series(test_project, test_repository,
test_perf_signature):
INTERVAL = 30
now = time.time()
for (t, v) in zip(
[i for i in range(INTERVAL)],
([0.5
for i in range(INTERVAL / 2)] + [1.0 for i in range(INTERVAL / 2)])):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=t,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL / 2), (INTERVAL / 2) - 1, test_perf_signature,
0.5, 1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
# verify that no new alerts generated if we rerun
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL / 2), (INTERVAL / 2) - 1, test_perf_signature,
0.5, 1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
# add data to generate a new alert
for (t, v) in zip([i for i in range(INTERVAL, INTERVAL * 2)],
[2.0 for i in range(INTERVAL)]):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=0,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 2
assert PerformanceAlertSummary.objects.count() == 2
_verify_alert(2, INTERVAL, INTERVAL - 1, test_perf_signature, 1.0, 2.0,
True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
def test_detect_alerts_in_series(test_project, test_repository,
test_perf_signature):
INTERVAL = 30
now = time.time()
for (t, v) in zip([i for i in range(INTERVAL)],
([0.5 for i in range(INTERVAL/2)] +
[1.0 for i in range(INTERVAL/2)])):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=t,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL/2), (INTERVAL/2)-1, test_perf_signature, 0.5,
1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
# verify that no new alerts generated if we rerun
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL/2), (INTERVAL/2)-1, test_perf_signature, 0.5,
1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
# add data to generate a new alert
for (t, v) in zip([i for i in range(INTERVAL, INTERVAL*2)],
[2.0 for i in range(INTERVAL)]):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=0,
signature=test_perf_signature,
push_timestamp=datetime.datetime.fromtimestamp(now + t),
value=v)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 2
assert PerformanceAlertSummary.objects.count() == 2
_verify_alert(2, INTERVAL, INTERVAL-1, test_perf_signature, 1.0, 2.0,
True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED)
def test_detect_alerts_in_series(test_repository,
test_issue_tracker,
failure_classifications,
generic_reference_data,
test_perf_signature):
base_time = time.time() # generate it based off current time
INTERVAL = 30
_generate_performance_data(test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time, 1, 0.5, int(INTERVAL / 2))
_generate_performance_data(test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time, int(INTERVAL / 2) + 1, 1.0, int(INTERVAL / 2))
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL/2)+1, (INTERVAL/2), test_perf_signature, 0.5,
1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
# verify that no new alerts generated if we rerun
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL/2)+1, (INTERVAL/2), test_perf_signature, 0.5,
1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
# add data that should be enough to generate a new alert if we rerun
_generate_performance_data(test_repository,
test_perf_signature,
test_issue_tracker,
generic_reference_data,
base_time, (INTERVAL+1), 2.0, INTERVAL)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 2
assert PerformanceAlertSummary.objects.count() == 2
_verify_alert(2, INTERVAL+1, INTERVAL, test_perf_signature, 1.0, 2.0,
True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
def test_detect_alerts_in_series(test_repository, test_issue_tracker,
failure_classifications,
generic_reference_data, test_perf_signature):
base_time = time.time() # generate it based off current time
INTERVAL = 30
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, 1, 0.5, int(INTERVAL / 2))
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time,
int(INTERVAL / 2) + 1, 1.0, int(INTERVAL / 2))
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL / 2) + 1, (INTERVAL / 2), test_perf_signature,
0.5, 1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
# verify that no new alerts generated if we rerun
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
_verify_alert(1, (INTERVAL / 2) + 1, (INTERVAL / 2), test_perf_signature,
0.5, 1.0, True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
# add data that should be enough to generate a new alert if we rerun
_generate_performance_data(test_repository, test_perf_signature,
test_issue_tracker, generic_reference_data,
base_time, (INTERVAL + 1), 2.0, INTERVAL)
generate_new_alerts_in_series(test_perf_signature)
assert PerformanceAlert.objects.count() == 2
assert PerformanceAlertSummary.objects.count() == 2
_verify_alert(2, INTERVAL + 1, INTERVAL, test_perf_signature, 1.0, 2.0,
True, PerformanceAlert.UNTRIAGED,
PerformanceAlertSummary.UNTRIAGED, None)
def generate_alerts(signature_id):
newrelic.agent.add_custom_parameter("signature_id", signature_id)
signature = PerformanceSignature.objects.get(id=signature_id)
generate_new_alerts_in_series(signature)
def generate_alerts(signature_id):
signature = PerformanceSignature.objects.get(id=signature_id)
generate_new_alerts_in_series(signature)
def test_detect_alerts_in_series(test_project, test_repository):
framework = PerformanceFramework.objects.create(
name='test_talos')
option = Option.objects.create(name='opt')
option_collection = OptionCollection.objects.create(
option_collection_hash='my_option_hash',
option=option)
platform = MachinePlatform.objects.create(
os_name='win',
platform='win7',
architecture='x86',
active_status='active')
signature = PerformanceSignature.objects.create(
repository=test_repository,
signature_hash=(40*'t'),
framework=framework,
platform=platform,
option_collection=option_collection,
suite='mysuite',
test='mytest'
)
INTERVAL = 30
for (t, v) in zip([i for i in range(INTERVAL)],
([0.5 for i in range(INTERVAL/2)] +
[1.0 for i in range(INTERVAL/2)])):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=0,
signature=signature,
push_timestamp=datetime.datetime.fromtimestamp(t),
value=v)
generate_new_alerts_in_series(signature)
def verify_alert(alertid, expected_result_set_id,
expected_prev_result_set_id,
expected_signature, expected_prev_value,
expected_new_value, is_regression):
alert = PerformanceAlert.objects.get(id=alertid)
assert alert.prev_value == expected_prev_value
assert alert.new_value == expected_new_value
assert alert.series_signature == expected_signature
assert alert.is_regression == is_regression
summary = PerformanceAlertSummary.objects.get(id=alertid)
assert summary.result_set_id == expected_result_set_id
assert summary.prev_result_set_id == expected_prev_result_set_id
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
verify_alert(1, (INTERVAL/2), (INTERVAL/2)-1, signature, 0.5, 1.0, True)
# verify that no new alerts generated if we rerun
generate_new_alerts_in_series(signature)
assert PerformanceAlert.objects.count() == 1
assert PerformanceAlertSummary.objects.count() == 1
verify_alert(1, (INTERVAL/2), (INTERVAL/2)-1, signature, 0.5, 1.0, True)
# add data to generate a new alert
for (t, v) in zip([i for i in range(INTERVAL, INTERVAL*2)],
[2.0 for i in range(INTERVAL)]):
PerformanceDatum.objects.create(
repository=test_repository,
result_set_id=t,
job_id=0,
signature=signature,
push_timestamp=datetime.datetime.fromtimestamp(t),
value=v)
generate_new_alerts_in_series(signature)
assert PerformanceAlert.objects.count() == 2
assert PerformanceAlertSummary.objects.count() == 2
verify_alert(2, INTERVAL, INTERVAL-1, signature, 1.0, 2.0, True)