def get_alerts(client, revinfo, appname, context):
numbers = client.query("select time, percentile(value, 95) from coldlaunch.visuallyLoaded where appName = '%s' and context = '%s' and device='%s' and branch='%s' and memory='%s' and time > now() - 7d group by time(1000u) order asc;" % (appname, context, DEVICE, BRANCH, MEMORY))
perf_data = []
values = []
ret = []
if not numbers:
return ret
for (timestamp, value) in numbers[0]['points']:
if not timestamp in revinfo:
continue
else:
gaia_rev = revinfo[timestamp][0]
gecko_rev = revinfo[timestamp][1]
perf_data.append(B2GPerfDatum(timestamp, value, gaia_revision=gaia_rev, revision=gecko_rev))
ta = TalosAnalyzer()
ta.addData(perf_data)
vals = ta.analyze_t()
for (i, r) in enumerate(vals):
if r.state == 'regression':
prevr = vals[i-1]
ret.append({ 'push_timestamp': r.push_timestamp,
'confidence': r.t,
'gecko_revision': r.revision,
'gaia_revision': r.gaia_revision,
'prev_gecko_revision': prevr.revision,
'prev_gaia_revision': prevr.gaia_revision,
'oldavg': r.historical_stats['avg'],
'newavg': r.forward_stats['avg'] })
return ret
def handleSeries(self, s):
if self.config.has_option('os', s.os_name):
s.os_name = self.config.get('os', s.os_name)
# Check if we should skip this test
ignore_tests = []
if self.config.has_option('main', 'ignore_tests'):
for i in self.config.get('main', 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
if self.config.has_option(s.branch_name, 'ignore_tests'):
for i in self.config.get(s.branch_name, 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
for i in ignore_tests:
if re.search(i, s.test_name):
log.debug("Skipping %s %s %s", s.branch_name, s.os_name, s.test_name)
return
log.info("Processing %s %s %s", s.branch_name, s.os_name, s.test_name)
# Get all the test data for all machines running this combination
t = time.time()
data = self.source.getTestData(s, options.start_time, self.data_type)
log.debug("%.2f to fetch data", time.time() - t)
if data:
m = max(d.testrun_id for d in data)
if self.last_run < m:
log.debug("Setting last_run to %s", m)
self.last_run = m
self.updateTimes(s.branch_name, data)
a = TalosAnalyzer()
a.addData(data)
analysis_gen = a.analyze_t(self.back_window, self.fore_window,
self.threshold, machine_threshold=self.machine_threshold,
machine_history_size=self.machine_history_size)
if s.branch_name not in self.warning_history:
self.warning_history[s.branch_name] = {}
if s.os_name not in self.warning_history[s.branch_name]:
self.warning_history[s.branch_name][s.os_name] = {}
if s.test_name not in self.warning_history[s.branch_name][s.os_name]:
self.warning_history[s.branch_name][s.os_name][s.test_name] = []
warnings = self.warning_history[s.branch_name][s.os_name][s.test_name]
series_data = self.processSeries(analysis_gen, warnings)
for d, skip, last_good in series_data:
self.handleData(s, d, d.state, skip, last_good)
if self.config.has_option('main', 'graph_dir'):
self.outputGraphs(s, series_data)
def get_alerts(client, appname, context):
numbers = client.query(
"select time, mean(value) from coldlaunch.visuallyLoaded where appName = '%s' and context = '%s' and device='flame-kk' and branch='master' and memory='319' and time > '2015-03-31' group by time(1s) order ASC;"
% (appname, context))
revinfo = client.query(
"select time, text from events where device='flame-kk' and branch='master' and memory='319' and time > '2015-03-31' group by time(1s) order ASC;"
)
sequence_rev_list = []
for (timestamp, sequence_number, text) in revinfo[0]['points']:
(gaia_revision,
gecko_revision) = re.match("^Gaia: (.*)<br/>Gecko: (.*)$",
text).groups()
sequence_rev_list.append((timestamp, gaia_revision, gecko_revision))
perf_data = []
values = []
for (timestamp, value) in numbers[0]['points']:
# we sometimes have duplicate test runs for the same timestamp? or
# the revision info is otherwise missing. this is probably bad, and
# resulting in inaccurate data...
for (sequence_timestamp, gaia_revision,
gecko_revision) in sequence_rev_list:
if sequence_timestamp <= timestamp:
revinfo = (sequence_timestamp, gaia_revision, gecko_revision)
else:
break
perf_data.append(
B2GPerfDatum(revinfo[0],
value,
gaia_revision=revinfo[1],
revision=revinfo[2]))
ta = TalosAnalyzer()
ta.addData(perf_data)
vals = ta.analyze_t()
ret = []
for (i, r) in enumerate(vals):
if r.state == 'regression':
prevr = vals[i - 1]
ret.append({
'push_timestamp': r.push_timestamp,
'confidence': r.t,
'gecko_revision': r.revision,
'gaia_revision': r.gaia_revision,
'prev_gecko_revision': prevr.revision,
'prev_gaia_revision': prevr.gaia_revision,
'oldavg': r.historical_stats['avg'],
'newavg': r.forward_stats['avg']
})
return ret
def get_alerts(client, revinfo, appname, context):
numbers = client.query(
"select time, percentile(value, 95) from coldlaunch.visuallyLoaded where appName = '%s' and context = '%s' and device='%s' and branch='%s' and memory='%s' and time > now() - 7d group by time(1000u) order asc;"
% (appname, context, DEVICE, BRANCH, MEMORY))
perf_data = []
values = []
ret = []
if not numbers:
return ret
for (timestamp, value) in numbers[0]['points']:
if not timestamp in revinfo:
continue
else:
gaia_rev = revinfo[timestamp][0]
gecko_rev = revinfo[timestamp][1]
perf_data.append(
B2GPerfDatum(timestamp,
value,
gaia_revision=gaia_rev,
revision=gecko_rev))
ta = TalosAnalyzer()
ta.addData(perf_data)
vals = ta.analyze_t()
for (i, r) in enumerate(vals):
if r.state == 'regression':
prevr = vals[i - 1]
ret.append({
'push_timestamp': r.push_timestamp,
'confidence': r.t,
'gecko_revision': r.revision,
'gaia_revision': r.gaia_revision,
'prev_gecko_revision': prevr.revision,
'prev_gaia_revision': prevr.gaia_revision,
'oldavg': r.historical_stats['avg'],
'newavg': r.forward_stats['avg']
})
return ret
def handleSeries(self, s):
if self.config.has_option('os', s.os_name):
s.os_name = self.config.get('os', s.os_name)
# Check if we should skip this test
ignore_tests = []
if self.config.has_option('main', 'ignore_tests'):
for i in self.config.get('main', 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
if self.config.has_option(s.branch_name, 'ignore_tests'):
for i in self.config.get(s.branch_name, 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
for i in ignore_tests:
if re.search(i, s.test_name):
log.debug("Skipping %s %s %s", s.branch_name, s.os_name, s.test_name)
return
log.info("Processing %s %s %s", s.branch_name, s.os_name, s.test_name)
# Get all the test data for all machines running this combination
t = time.time()
data = self.source.getTestData(s, options.start_time)
log.debug("%.2f to fetch data", time.time() - t)
if data:
m = max(d.testrun_id for d in data)
if self.last_run < m:
log.debug("Setting last_run to %s", m)
self.last_run = m
self.updateTimes(s.branch_name, data)
a = TalosAnalyzer()
a.addData(data)
analysis_gen = a.analyze_t(self.back_window, self.fore_window,
self.threshold, self.machine_threshold,
self.machine_history_size)
if s.branch_name not in self.warning_history:
self.warning_history[s.branch_name] = {}
if s.os_name not in self.warning_history[s.branch_name]:
self.warning_history[s.branch_name][s.os_name] = {}
if s.test_name not in self.warning_history[s.branch_name][s.os_name]:
self.warning_history[s.branch_name][s.os_name][s.test_name] = []
warnings = self.warning_history[s.branch_name][s.os_name][s.test_name]
last_good = None
last_err = None
last_err_good = None
# Uncomment this for debugging!
#cutoff = self.options.start_time
cutoff = time.time() - 7*24*3600
series_data = []
for d, state in analysis_gen:
skip = False
if d.timestamp < cutoff:
continue
if state != "good":
# Skip warnings about regressions we've already
# warned people about
if (d.buildid, d.timestamp) in warnings:
skip = True
else:
warnings.append((d.buildid, d.timestamp))
if state == "machine":
machine_name = self.source.getMachineName(d.machine_id)
if 'bad_machines' not in self.warning_history:
self.warning_history['bad_machines'] = {}
# When did we last warn about this machine?
if self.warning_history['bad_machines'].get(machine_name, 0) > time.time() - 7*24*3600:
skip = True
else:
# If it was over a week ago, then send another warning
self.warning_history['bad_machines'][machine_name] = time.time()
if not last_err:
last_err = d
last_err_good = last_good
elif last_err_good == last_good:
skip = True
else:
last_err = None
last_good = d
series_data.append((s, d, state, skip, last_good))
self.handleData(s, d, state, skip, last_good)
if self.config.has_option('main', 'graph_dir'):
self.outputGraphs(s, series_data)
values = []
for (timestamp, sequence_number, value) in clocknumbers[0]['points']:
if prev_timestamp and prev_timestamp == timestamp:
values.append(value)
elif prev_timestamp:
# add everything to perf data
avg = float(sum(values)) / len(values)
perf_data.append(
PerfDatum(prev_timestamp, 0, prev_timestamp, avg, None,
prev_timestamp))
# start again
values = [value]
prev_timestamp = timestamp
else:
# first value
prev_timestamp = timestamp
values = [value]
ta = TalosAnalyzer()
ta.addData(perf_data)
for r in ta.analyze_t(5, 5, 3):
# if r.state == 'regression':
if sequence_rev_map.get(r.timestamp):
print "date: %s" % datetime.datetime.fromtimestamp(
r.timestamp).strftime('%Y-%m-%d %H:%M:%S')
print "confidence (higher is more confident): %s" % r.t
print "revision: %s" % sequence_rev_map[r.timestamp]
print "old average: %s" % r.historical_stats['avg']
print "new average: %s" % r.forward_stats['avg']
print "================================="
def handleSeries(self, s):
if self.config.has_option('os', s.os_name):
s.os_name = self.config.get('os', s.os_name)
# Check if we should skip this test
ignore_tests = []
if self.config.has_option('main', 'ignore_tests'):
for i in self.config.get('main', 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
if self.config.has_option(s.branch_name, 'ignore_tests'):
for i in self.config.get(s.branch_name, 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
for i in ignore_tests:
if re.search(i, s.test_name):
log.debug("Skipping %s %s %s", s.branch_name, s.os_name, s.test_name)
return
log.info("Processing %s %s %s", s.branch_name, s.os_name, s.test_name)
# Get all the test data for all machines running this combination
t = time.time()
data = self.source.getTestData(s, options.start_time)
log.debug("%.2f to fetch data", time.time() - t)
# Add it to our dashboard data
sevenDaysAgo = time.time() - 7*24*60*60
importantTests = []
for t in re.split(r"(?<!\\),", self.config.get("dashboard", "tests")):
t = t.replace("\\,", ",").strip()
importantTests.append(t)
if data:
m = max(d.testrun_id for d in data)
if self.last_run < m:
log.debug("Setting last_run to %s", m)
self.last_run = m
if s.test_name in importantTests and len(data) > 0:
# We want to merge the Tp3 (Memset) and Tp3 (RSS) results together
# for the dashboard, since they're just different names for the
# same thing on different platforms
test_name = s.test_name
if test_name == "Tp3 (Memset)":
test_name = "Tp3 (RSS)"
elif test_name == "Tp4 (Memset)":
test_name = "Tp4 (RSS)"
self.dashboard_data.setdefault(s.branch_name, {})
self.dashboard_data[s.branch_name].setdefault(test_name, {'_testid': s.test_id})
self.dashboard_data[s.branch_name][test_name].setdefault(s.os_name, {'_platformid': s.os_id, '_graphURL': self.makeChartUrl(s)})
_d = self.dashboard_data[s.branch_name][test_name][s.os_name]
for d in data:
if d.timestamp < sevenDaysAgo:
continue
machine_name = self.source.getMachineName(d.machine_id)
if machine_name not in _d:
_d[machine_name] = {
'results': [],
'stats': [],
}
results = _d[machine_name]['results']
results.append(d.timestamp)
results.append(d.value)
for machine_name in _d:
if machine_name.startswith("_"):
continue
results = _d[machine_name]['results']
values = [results[i+1] for i in range(0, len(results), 2)]
_d[machine_name]['stats'] = [avg(values), max(values), min(values)]
self.updateTimes(s.branch_name, data)
a = TalosAnalyzer()
a.addData(data)
analysis_gen = a.analyze_t(self.back_window, self.fore_window,
self.threshold, self.machine_threshold,
self.machine_history_size)
if s.branch_name not in self.warning_history:
self.warning_history[s.branch_name] = {}
if s.os_name not in self.warning_history[s.branch_name]:
self.warning_history[s.branch_name][s.os_name] = {}
if s.test_name not in self.warning_history[s.branch_name][s.os_name]:
self.warning_history[s.branch_name][s.os_name][s.test_name] = []
warnings = self.warning_history[s.branch_name][s.os_name][s.test_name]
last_good = None
last_err = None
last_err_good = None
# Uncomment this for debugging!
#cutoff = self.options.start_time
cutoff = time.time() - 7*24*3600
series_data = []
for d, state in analysis_gen:
skip = False
if d.timestamp < cutoff:
continue
if state != "good":
# Skip warnings about regressions we've already
# warned people about
if (d.buildid, d.timestamp) in warnings:
skip = True
else:
warnings.append((d.buildid, d.timestamp))
if state == "machine":
machine_name = self.source.getMachineName(d.machine_id)
if 'bad_machines' not in self.warning_history:
self.warning_history['bad_machines'] = {}
# When did we last warn about this machine?
if self.warning_history['bad_machines'].get(machine_name, 0) > time.time() - 7*24*3600:
skip = True
else:
# If it was over a week ago, then send another warning
self.warning_history['bad_machines'][machine_name] = time.time()
if not last_err:
last_err = d
last_err_good = last_good
elif last_err_good == last_good:
skip = True
else:
last_err = None
last_good = d
series_data.append((s, d, state, skip, last_good))
self.handleData(s, d, state, skip, last_good)
if self.config.has_option('main', 'graph_dir'):
self.outputGraphs(s, series_data)
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import phclient
import sys
from analyze import PerfDatum, TalosAnalyzer
pc = phclient.Client()
(projectname, signature) = (sys.argv[1], sys.argv[2])
s = pc.get_series(projectname, signature, time_interval=phclient.TimeInterval.NINETY_DAYS)
perf_data = []
for (result_set_id, timestamp, geomean) in zip(
s['result_set_id'], s['push_timestamp'], s['geomean']):
perf_data.append(PerfDatum(result_set_id, 0, timestamp, geomean, None,
timestamp))
ta = TalosAnalyzer()
ta.addData(perf_data)
for r in ta.analyze_t(5, 5, 2):
if r.state == 'regression':
print (r.testrun_id, r.t, pc.get_revision(projectname, r.testrun_id)[0:12])
def handleSeries(self, s):
if self.config.has_option('os', s.os_name):
s.os_name = self.config.get('os', s.os_name)
# Check if we should skip this test
ignore_tests = []
if self.config.has_option('main', 'ignore_tests'):
for i in self.config.get('main', 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
if self.config.has_option(s.branch_name, 'ignore_tests'):
for i in self.config.get(s.branch_name, 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
for i in ignore_tests:
if re.search(i, s.test_name):
log.debug("Skipping %s %s %s", s.branch_name, s.os_name,
s.test_name)
return
log.info("Processing %s %s %s", s.branch_name, s.os_name, s.test_name)
# Get all the test data for all machines running this combination
t = time.time()
data = self.source.getTestData(s, options.start_time)
log.debug("%.2f to fetch data", time.time() - t)
if data:
m = max(d.testrun_id for d in data)
if self.last_run < m:
log.debug("Setting last_run to %s", m)
self.last_run = m
self.updateTimes(s.branch_name, data)
a = TalosAnalyzer()
a.addData(data)
analysis_gen = a.analyze_t(self.back_window, self.fore_window,
self.threshold, self.machine_threshold,
self.machine_history_size)
if s.branch_name not in self.warning_history:
self.warning_history[s.branch_name] = {}
if s.os_name not in self.warning_history[s.branch_name]:
self.warning_history[s.branch_name][s.os_name] = {}
if s.test_name not in self.warning_history[s.branch_name][s.os_name]:
self.warning_history[s.branch_name][s.os_name][s.test_name] = []
warnings = self.warning_history[s.branch_name][s.os_name][s.test_name]
last_good = None
last_err = None
last_err_good = None
# Uncomment this for debugging!
#cutoff = self.options.start_time
cutoff = time.time() - 7 * 24 * 3600
series_data = []
for d, state in analysis_gen:
skip = False
if d.timestamp < cutoff:
continue
if state != "good":
# Skip warnings about regressions we've already
# warned people about
if (d.buildid, d.timestamp) in warnings:
skip = True
else:
warnings.append((d.buildid, d.timestamp))
if state == "machine":
machine_name = self.source.getMachineName(d.machine_id)
if 'bad_machines' not in self.warning_history:
self.warning_history['bad_machines'] = {}
# When did we last warn about this machine?
if self.warning_history['bad_machines'].get(
machine_name, 0) > time.time() - 7 * 24 * 3600:
skip = True
else:
# If it was over a week ago, then send another warning
self.warning_history['bad_machines'][
machine_name] = time.time()
if not last_err:
last_err = d
last_err_good = last_good
elif last_err_good == last_good:
skip = True
else:
last_err = None
last_good = d
series_data.append((s, d, state, skip, last_good))
self.handleData(s, d, state, skip, last_good)
if self.config.has_option('main', 'graph_dir'):
self.outputGraphs(s, series_data)
def handleSeries(self, s):
if self.config.has_option('os', s.os_name):
s.os_name = self.config.get('os', s.os_name)
# Check if we should skip this test
ignore_tests = []
if self.config.has_option('main', 'ignore_tests'):
for i in self.config.get('main', 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
if self.config.has_option(s.branch_name, 'ignore_tests'):
for i in self.config.get(s.branch_name, 'ignore_tests').split(','):
i = i.strip()
if i:
ignore_tests.append(i)
for i in ignore_tests:
if re.search(i, s.test_name):
log.debug("Skipping %s %s %s", s.branch_name, s.os_name,
s.test_name)
return
log.info("Processing %s %s %s", s.branch_name, s.os_name, s.test_name)
# Get all the test data for all machines running this combination
t = time.time()
data = self.source.getTestData(s, options.start_time, self.data_type)
log.debug("%.2f to fetch data", time.time() - t)
if data:
m = max(d.testrun_id for d in data)
if self.last_run < m:
log.debug("Setting last_run to %s", m)
self.last_run = m
self.updateTimes(s.branch_name, data)
a = TalosAnalyzer()
a.addData(data)
analysis_gen = a.analyze_t(
self.back_window,
self.fore_window,
self.threshold,
machine_threshold=self.machine_threshold,
machine_history_size=self.machine_history_size)
if s.branch_name not in self.warning_history:
self.warning_history[s.branch_name] = {}
if s.os_name not in self.warning_history[s.branch_name]:
self.warning_history[s.branch_name][s.os_name] = {}
if s.test_name not in self.warning_history[s.branch_name][s.os_name]:
self.warning_history[s.branch_name][s.os_name][s.test_name] = []
warnings = self.warning_history[s.branch_name][s.os_name][s.test_name]
series_data = self.processSeries(analysis_gen, warnings)
for d, skip, last_good in series_data:
self.handleData(s, d, d.state, skip, last_good)
if self.config.has_option('main', 'graph_dir'):
self.outputGraphs(s, series_data)