def all_fails(classifier):
"""Find all the the fails in the integrated gate.
This attempts to find all the build jobs in the integrated gate
so we can figure out how good we are doing on total classification.
"""
all_fails = {}
query = ('filename:"console.html" '
'AND message:"Finished: FAILURE" '
'AND build_queue:"gate"')
results = classifier.hits_by_query(query, size=30000)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
for result in facets[build]:
# If the job is on the exclude list, skip
if re.search(EXCLUDED_JOBS_REGEX, result.build_name):
continue
# not perfect, but basically an attempt to show the integrated
# gate. Would be nice if there was a zuul attr for this in es.
if re.search("(^openstack/|devstack|grenade)", result.project):
name = result.build_name
timestamp = dp.parse(result.timestamp)
log = result.log_url.split("console.html")[0]
all_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,
'build_uuid': result.build_uuid
}
return all_fails
def _count_fails_per_build_name(hits):
facets = er_results.FacetSet()
counts = collections.defaultdict(int)
facets.detect_facets(hits, ["build_status", "build_name", "build_uuid"])
if "FAILURE" in facets:
for build_name in facets["FAILURE"]:
counts[build_name] += 1
return counts
def _status_count(results):
counts = {}
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_status", "build_uuid"])
for key in facets:
counts[key] = len(facets[key])
return counts
def test_facet_multi_level(self):
data = load_sample(1226337)
result_set = results.ResultSet(data)
facets = results.FacetSet()
facets.detect_facets(result_set, ["build_status", "build_uuid"])
self.assertEqual(len(facets.keys()), 2)
self.assertEqual(len(facets['FAILURE'].keys()), 12)
self.assertEqual(len(facets['SUCCESS'].keys()), 3)
def _failed_jobs(results):
failed_jobs = []
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_status", "build_uuid"])
if "FAILURE" in facets:
for build in facets["FAILURE"]:
for result in facets["FAILURE"][build]:
failed_jobs.append("%s.%s" % (build, result.build_name))
return failed_jobs
def test_facet_histogram(self):
data = load_sample(1226337)
result_set = results.ResultSet(data)
facets = results.FacetSet()
facets.detect_facets(result_set,
["timestamp", "build_status", "build_uuid"])
self.assertEqual(len(facets.keys()), 14)
print facets[1382104800000].keys()
self.assertEqual(facets[1382104800000].keys(), ["FAILURE"])
self.assertEqual(len(facets[1382104800000]["FAILURE"]), 2)
self.assertEqual(facets[1382101200000].keys(), ["FAILURE"])
def test_facet_one_level(self):
data = load_sample(1218391)
result_set = results.ResultSet(data)
facets = results.FacetSet()
facets.detect_facets(result_set, ["build_uuid"])
self.assertEqual(len(facets.keys()), 20)
facets = results.FacetSet()
facets.detect_facets(result_set, ["build_status"])
self.assertEqual(facets.keys(), ['FAILURE'])
data = load_sample(1226337)
result_set = results.ResultSet(data)
facets = results.FacetSet()
facets.detect_facets(result_set, ["build_status"])
self.assertEqual(len(facets.keys()), 2)
self.assertIn('FAILURE', facets.keys())
self.assertIn('SUCCESS', facets.keys())
self.assertEqual(len(facets['FAILURE']), 202)
self.assertEqual(len(facets['SUCCESS']), 27)
def all_fails(classifier):
"""Find all the fails in the integrated gate.
This attempts to find all the build jobs in the integrated gate
so we can figure out how good we are doing on total classification.
"""
all_fails = {}
results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY, size=30000)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
for result in facets[build]:
# not perfect, but basically an attempt to show the integrated
# gate. Would be nice if there was a zuul attr for this in es.
if re.search(er_config.INCLUDED_PROJECTS_REGEX, result.project):
all_fails["%s.%s" % (build, result.build_name)] = False
return all_fails
def all_fails(classifier):
"""Find all the the fails in the integrated gate.
This attempts to find all the build jobs in the integrated gate
so we can figure out how good we are doing on total classification.
"""
all_fails = {}
query = ('filename:"console.html" '
'AND message:"Finished: FAILURE" '
'AND build_queue:"gate"')
results = classifier.hits_by_query(query, size=30000)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
for result in facets[build]:
# not perfect, but basically an attempt to show the integrated
# gate. Would be nice if there was a zuul attr for this in es.
if re.search("(^openstack/|devstack|grenade)", result.project):
all_fails["%s.%s" % (build, result.build_name)] = False
return all_fails
def main():
parser = argparse.ArgumentParser(description='Generate data for graphs.')
parser.add_argument(dest='queries', help='path to query file')
parser.add_argument('-o', dest='output', help='output filename')
parser.add_argument('-q',
dest='queue',
help='limit results to a specific query')
parser.add_argument('-v',
dest='verbose',
action='store_true',
default=False,
help='print out details as we go')
args = parser.parse_args()
classifier = er.Classifier(args.queries)
buglist = []
# if you don't hate timezones, you don't program enough
epoch = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
ts = datetime.utcnow().replace(tzinfo=pytz.utc)
# rawnow is useful for sending to javascript
rawnow = int(((ts - epoch).total_seconds()) * 1000)
ts = datetime(ts.year, ts.month, ts.day, ts.hour).replace(tzinfo=pytz.utc)
# ms since epoch
now = int(((ts - epoch).total_seconds()) * 1000)
# number of days to match to, this should be the same as we are
# indexing in logstash
days = 10
# How far back to start in the graphs
start = now - (days * 24 * STEP)
# ER timeframe for search
timeframe = days * 24 * STEP / 1000
last_indexed = int(
((classifier.most_recent() - epoch).total_seconds()) * 1000)
behind = now - last_indexed
# the data we're going to return, including interesting headers
jsondata = {
'now': rawnow,
'last_indexed': last_indexed,
'behind': behind,
'buglist': []
}
for query in classifier.queries:
if args.queue:
query['query'] = query['query'] + (' AND build_queue:"%s"' %
args.queue)
if query.get('suppress-graph'):
continue
if args.verbose:
LOG.debug("Starting query for bug %s" % query['bug'])
logstash_query = qb.encode_logstash_query(query['query'],
timeframe=timeframe)
bug_data = get_launchpad_bug(query['bug'])
bug = dict(number=query['bug'],
query=query['query'],
logstash_query=logstash_query,
bug_data=bug_data,
fails=0,
fails24=0,
data=[])
buglist.append(bug)
results = classifier.hits_by_query(query['query'],
args.queue,
size=3000)
facets_for_fail = er_results.FacetSet()
facets_for_fail.detect_facets(results, ["build_status", "build_uuid"])
if "FAILURE" in facets_for_fail:
bug['fails'] = len(facets_for_fail['FAILURE'])
facets = er_results.FacetSet()
facets.detect_facets(results,
["build_status", "timestamp", "build_uuid"])
for status in facets.keys():
data = []
for ts in range(start, now, STEP):
if ts in facets[status]:
fails = len(facets[status][ts])
data.append([ts, fails])
# get the last 24 hr count as well, can't wait to have
# the pandas code and able to do it that way
if status == "FAILURE" and ts > (now - (24 * STEP)):
bug['fails24'] += fails
else:
data.append([ts, 0])
bug["data"].append(dict(label=status, data=data))
# the sort order is a little odd, but basically sort by failures in
# the last 24 hours, then with all failures for ones that we haven't
# seen in the last 24 hours.
buglist = sorted(buglist,
key=lambda bug: -(bug['fails24'] * 100000 + bug['fails']))
jsondata['buglist'] = buglist
out = open(args.output, 'w')
out.write(json.dumps(jsondata))
out.close()
def all_fails(classifier, config=None):
"""Find all the the fails in the integrated gate.
This attempts to find all the build jobs in the integrated gate
so we can figure out how good we are doing on total classification.
"""
config = config or er_config.Config()
integrated_fails = {}
other_fails = {}
all_fails = {}
results = classifier.hits_by_query(config.all_fails_query,
size=config.uncat_search_size)
facets = er_results.FacetSet()
facets.detect_facets(results, ["build_uuid"])
for build in facets:
for result in facets[build]:
# If the job is on the exclude list, skip
if re.search(config.excluded_jobs_regex, result.build_name):
continue
integrated_gate_projects = [
'openstack/cinder',
'openstack/glance',
'openstack/keystone',
'openstack/neutron',
'openstack/nova',
'openstack/requirements',
'openstack/tempest',
'openstack-dev/devstack',
'openstack-dev/grenade',
'openstack-infra/devstack-gate',
]
if result.project in integrated_gate_projects:
name = result.build_name
timestamp = dp.parse(result.timestamp)
if 'console.html' in result.log_url:
log = result.log_url.split('console.html')[0]
elif 'job-output.txt' in result.log_url:
log = result.log_url.split('job-output.txt')[0]
integrated_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,
'build_uuid': result.build_uuid
}
else:
# not perfect, but basically an attempt to show the integrated
# gate. Would be nice if there was a zuul attr for this in es.
if re.search(config.included_projects_regex, result.project):
name = result.build_name
timestamp = dp.parse(result.timestamp)
if 'console.html' in result.log_url:
log = result.log_url.split('console.html')[0]
elif 'job-output.txt' in result.log_url:
log = result.log_url.split('job-output.txt')[0]
other_fails["%s.%s" % (build, name)] = {
'log': log,
'timestamp': timestamp,
'build_uuid': result.build_uuid
}
LOG.debug("Found failure: %s build_uuid: %s project %s",
len(all_fails), result.build_uuid, result.project)
all_fails = {'integrated_gate': integrated_fails, 'others': other_fails}
return all_fails
def main():
parser = argparse.ArgumentParser(description='Generate data for graphs.')
parser.add_argument(dest='queries', help='path to query file')
parser.add_argument('-o',
dest='output',
help='output filename. Omit for stdout')
parser.add_argument('-q',
dest='queue',
help='limit results to a build queue regex')
parser.add_argument('--es-query-suffix',
help='further limit results with an '
'elastic search query suffix. This will be ANDed '
'to all queries. '
'For example, to limit all queries to a '
'specific branch use: '
' --es-query-suffix "build_branch:\\"stable/'
'liberty\\""')
parser.add_argument('-c',
'--conf',
help="Elastic Recheck Configuration "
"file to use for data_source options such as "
"elastic search url, logstash url, and database "
"uri.")
parser.add_argument('-v',
dest='verbose',
action='store_true',
default=False,
help='print out details as we go')
args = parser.parse_args()
config = er_conf.Config(config_file=args.conf)
classifier = er.Classifier(args.queries, config=config)
buglist = []
# if you don't hate timezones, you don't program enough
epoch = datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc)
ts = datetime.utcnow().replace(tzinfo=pytz.utc)
# rawnow is useful for sending to javascript
rawnow = int(((ts - epoch).total_seconds()) * 1000)
ts = datetime(ts.year, ts.month, ts.day, ts.hour).replace(tzinfo=pytz.utc)
# ms since epoch
now = int(((ts - epoch).total_seconds()) * 1000)
# number of days to match to, this should be the same as we are
# indexing in logstash
days = 10
# How far back to start in the graphs
start = now - (days * 24 * STEP)
# ER timeframe for search
timeframe = days * 24 * STEP / 1000
last_indexed = int(
((classifier.most_recent() - epoch).total_seconds()) * 1000)
behind = now - last_indexed
# the data we're going to return, including interesting headers
jsondata = {
'now': rawnow,
'last_indexed': last_indexed,
'behind': behind,
'buglist': []
}
# Get the cluster health for the header
es = pyelasticsearch.ElasticSearch(config.es_url)
jsondata['status'] = es.health()['status']
for query in classifier.queries:
if args.queue:
query['query'] += ' AND build_queue:%s' % args.queue
if args.es_query_suffix:
query['query'] += ' AND (%s)' % args.es_query_suffix
if query.get('suppress-graph'):
continue
if args.verbose:
LOG.debug("Starting query for bug %s" % query['bug'])
logstash_query = qb.encode_logstash_query(query['query'],
timeframe=timeframe)
logstash_url = ("%s/#/dashboard/file/logstash.json?%s" %
(config.ls_url, logstash_query))
bug_data = get_launchpad_bug(query['bug'])
bug = dict(number=query['bug'],
query=query['query'],
logstash_url=logstash_url,
bug_data=bug_data,
fails=0,
fails24=0,
data=[],
voting=(False if query.get('allow-nonvoting') else True))
buglist.append(bug)
try:
results = classifier.hits_by_query(query['query'],
args.queue,
size=3000,
days=days)
except pyelasticsearch.exceptions.InvalidJsonResponseError:
LOG.exception(
"Invalid Json while collecting metrics for query %s" %
query['query'])
continue
except requests.exceptions.ReadTimeout:
LOG.exception("Timeout while collecting metrics for query %s" %
query['query'])
continue
except pyelasticsearch.exceptions.ElasticHttpError as ex:
LOG.error('Error from elasticsearch query for bug %s: %s',
query['bug'], ex)
continue
facets_for_fail = er_results.FacetSet()
facets_for_fail.detect_facets(results, ["build_status", "build_uuid"])
if "FAILURE" in facets_for_fail:
bug['fails'] = len(facets_for_fail['FAILURE'])
facets = er_results.FacetSet()
facets.detect_facets(results,
["build_status", "timestamp", "build_uuid"])
for status in facets.keys():
data = []
for ts in range(start, now, STEP):
if ts in facets[status]:
fails = len(facets[status][ts])
data.append([ts, fails])
# get the last 24 hr count as well, can't wait to have
# the pandas code and able to do it that way
if status == "FAILURE" and ts > (now - (24 * STEP)):
bug['fails24'] += fails
else:
data.append([ts, 0])
bug["data"].append(dict(label=status, data=data))
# the sort order is a little odd, but basically sort by failures in
# the last 24 hours, then with all failures for ones that we haven't
# seen in the last 24 hours.
buglist = sorted(buglist,
key=lambda bug: -(bug['fails24'] * 100000 + bug['fails']))
jsondata['buglist'] = buglist
if args.output:
out = open(args.output, 'w')
else:
out = sys.stdout
try:
out.write(json.dumps(jsondata))
finally:
out.close()