def check_json(self, filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for TalosAnalyzer
FORE_WINDOW = 12
BACK_WINDOW = 12
THRESHOLD = 7
MACHINE_THRESHOLD = 15
MACHINE_HISTORY_SIZE = 5
payload = SampleData.get_perf_data(os.path.join('graphs', filename))
runs = payload['test_runs']
data = [
PerfDatum(r[2],
r[3],
testrun_id=r[0],
machine_id=r[6],
testrun_timestamp=r[2],
buildid=r[1][1],
revision=r[1][2]) for r in runs
]
a = TalosAnalyzer()
a.addData(data)
results = a.analyze_t(BACK_WINDOW, FORE_WINDOW, THRESHOLD,
MACHINE_THRESHOLD, MACHINE_HISTORY_SIZE)
regression_timestamps = [
d.testrun_timestamp for d in results if d.state == 'regression'
]
self.assertEqual(regression_timestamps, expected_timestamps)
def check_json(self, filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for Analyzer
FORE_WINDOW = 12
BACK_WINDOW = 12
THRESHOLD = 7
payload = SampleData.get_perf_data(os.path.join('graphs', filename))
runs = payload['test_runs']
a = Analyzer()
for r in runs:
a.add_data(r[2], r[3], testrun_id=r[0],
revision_id=r[1][2])
results = a.analyze_t(BACK_WINDOW, FORE_WINDOW, THRESHOLD)
regression_timestamps = [d.push_timestamp for d in results if
d.state == 'regression']
self.assertEqual(regression_timestamps, expected_timestamps)
def test_detect_changes_historical_data(filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for Analyzer
FORE_WINDOW = 12
MIN_BACK_WINDOW = 12
MAX_BACK_WINDOW = 24
THRESHOLD = 7
payload = SampleData.get_perf_data(os.path.join('graphs', filename))
runs = payload['test_runs']
data = [RevisionDatum(r[2], r[2], [r[3]]) for r in runs]
results = detect_changes(data,
min_back_window=MIN_BACK_WINDOW,
max_back_window=MAX_BACK_WINDOW,
fore_window=FORE_WINDOW,
t_threshold=THRESHOLD)
regression_timestamps = [d.push_timestamp for d in results if
d.change_detected]
assert regression_timestamps == expected_timestamps
def check_json(self, filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for Analyzer
FORE_WINDOW = 12
MIN_BACK_WINDOW = 12
MAX_BACK_WINDOW = 24
THRESHOLD = 7
payload = SampleData.get_perf_data(os.path.join('graphs', filename))
runs = payload['test_runs']
data = []
for r in runs:
data.append(Datum(r[2], r[3], testrun_id=r[0],
revision_id=r[1][2]))
results = detect_changes(data, min_back_window=MIN_BACK_WINDOW,
max_back_window=MAX_BACK_WINDOW,
fore_window=FORE_WINDOW,
t_threshold=THRESHOLD)
regression_timestamps = [d.push_timestamp for d in results if
d.state == 'regression']
self.assertEqual(regression_timestamps, expected_timestamps)
def check_json(self, filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for TalosAnalyzer
FORE_WINDOW = 12
BACK_WINDOW = 12
THRESHOLD = 7
MACHINE_THRESHOLD = 15
MACHINE_HISTORY_SIZE = 5
payload = SampleData.get_perf_data(os.path.join('graphs', filename))
runs = payload['test_runs']
data = [PerfDatum(r[2], r[3], testrun_id=r[0], machine_id=r[6],
testrun_timestamp=r[2], buildid=r[1][1],
revision=r[1][2]) for r in runs]
a = TalosAnalyzer()
a.addData(data)
results = a.analyze_t(BACK_WINDOW, FORE_WINDOW, THRESHOLD,
MACHINE_THRESHOLD, MACHINE_HISTORY_SIZE)
regression_timestamps = [d.testrun_timestamp for d in results if
d.state == 'regression']
self.assertEqual(regression_timestamps, expected_timestamps)
def check_json(self, filename, expected_timestamps):
"""Parse JSON produced by http://graphs.mozilla.org/api/test/runs"""
# Configuration for Analyzer
FORE_WINDOW = 12
MIN_BACK_WINDOW = 12
MAX_BACK_WINDOW = 24
THRESHOLD = 7
payload = SampleData.get_perf_data(os.path.join('graphs', filename))
runs = payload['test_runs']
data = []
for r in runs:
data.append(Datum(r[2], r[3], testrun_id=r[0],
revision_id=r[1][2]))
results = detect_changes(data,
min_back_window=MIN_BACK_WINDOW,
max_back_window=MAX_BACK_WINDOW,
fore_window=FORE_WINDOW,
t_threshold=THRESHOLD)
regression_timestamps = [
d.push_timestamp for d in results if d.state == 'regression'
]
self.assertEqual(regression_timestamps, expected_timestamps)
