def test_get_run_metadata(self):
run_a = api.create_run()
run_b = api.create_run()
api.add_run_metadata({'not_a_key': 'not_a_value'}, run_b.id)
a_metadata = api.get_run_metadata(run_a.uuid)
b_metadata = api.get_run_metadata(run_b.uuid)
self.assertEqual([], a_metadata)
self.assertEqual(1, len(b_metadata))
b_metadata = b_metadata[0].to_dict()
self.assertEqual(run_b.id, b_metadata['run_id'])
self.assertEqual('not_a_key', b_metadata['key'])
self.assertEqual('not_a_value', b_metadata['value'])
def test_get_run_metadata(self):
run_a = api.create_run()
run_b = api.create_run()
api.add_run_metadata({'not_a_key': 'not_a_value'}, run_b.id)
a_metadata = api.get_run_metadata(run_a.uuid)
b_metadata = api.get_run_metadata(run_b.uuid)
self.assertEqual([], a_metadata)
self.assertEqual(1, len(b_metadata))
b_metadata = b_metadata[0].to_dict()
self.assertEqual(run_b.id, b_metadata['run_id'])
self.assertEqual('not_a_key', b_metadata['key'])
self.assertEqual('not_a_value', b_metadata['value'])
def test_delete_old_runs(self):
run_a = api.create_run(run_at=datetime.datetime(
1914, 6, 28, 10, 45, 0))
run_b = api.create_run()
api.add_run_metadata({'key': 'value'}, run_b.id)
api.add_run_metadata({'key': 'not_so_much_a_value'}, run_a.id)
api.delete_old_runs()
runs = api.get_all_runs()
self.assertEqual(1, len(runs))
self.assertEqual(1, api.get_session().query(
models.RunMetadata.id).count())
self.assertEqual(run_b.id, runs[0].id)
self.assertEqual(1, len(api.get_run_metadata(run_b.uuid)))
self.assertEqual(0, len(api.get_run_metadata(run_a.uuid)))
def test_delete_old_runs(self):
run_a = api.create_run(
run_at=datetime.datetime(1914, 6, 28, 10, 45, 0))
run_b = api.create_run()
api.add_run_metadata({'key': 'value'}, run_b.id)
api.add_run_metadata({'key': 'not_so_much_a_value'}, run_a.id)
api.delete_old_runs()
runs = api.get_all_runs()
self.assertEqual(1, len(runs))
self.assertEqual(
1,
api.get_session().query(models.RunMetadata.id).count())
self.assertEqual(run_b.id, runs[0].id)
self.assertEqual(1, len(api.get_run_metadata(run_b.uuid)))
self.assertEqual(0, len(api.get_run_metadata(run_a.uuid)))
def get_subunit_results(build_uuid,
dataset_name,
sample_interval,
db_uri,
build_name='tempest-full',
data_path=None,
s3=None):
engine = create_engine(db_uri)
Session = sessionmaker(bind=engine)
session = Session()
runs = api.get_runs_by_key_value('build_uuid', build_uuid, session=session)
results = []
for run in runs:
# Check if we are interested in this build at all
meta = api.get_run_metadata(run.uuid, session=session)
build_names = [x.value for x in meta if x.key == 'build_name']
if len(build_names) >= 1:
# Skip build_names that aren't selected
if not build_name == build_names[0]:
continue
db_build_name = build_names[0]
else:
continue
# NOTE(mtreinish): Only be concerned with single node to start
if 'multinode' in db_build_name:
continue
result = _get_data_for_run(run,
sample_interval,
session,
data_path=data_path,
s3=s3)
if result:
results.append(result)
session.close()
return results
def _get_recent_runs_data(run_metadata_key, value, detail=False):
num_runs = flask.request.args.get('num_runs', 10)
with session_scope() as session:
results = api.get_recent_runs_by_key_value_metadata(
run_metadata_key, value, num_runs, session)
runs = []
for result in results:
if detail:
run = result.to_dict()
else:
if result.passes > 0 and result.fails == 0:
status = 'success'
elif result.fails > 0:
status = 'fail'
else:
continue
run = {
'id': result.uuid,
'status': status,
'start_date': result.run_at.isoformat(),
'link': result.artifacts,
}
run_meta = api.get_run_metadata(result.uuid, session)
for meta in run_meta:
if meta.key == 'build_name':
run['build_name'] = meta.value
break
runs.append(run)
return runs
def _get_recent_runs_data(run_metadata_key, value, detail=False):
num_runs = flask.request.args.get('num_runs', 10)
with session_scope() as session:
results = api.get_recent_runs_by_key_value_metadata(
run_metadata_key, value, num_runs, session)
runs = []
for result in results:
if detail:
run = result.to_dict()
else:
if result.passes > 0 and result.fails == 0:
status = 'success'
elif result.fails > 0:
status = 'fail'
else:
continue
run = {
'id': result.uuid,
'status': status,
'start_date': result.run_at.isoformat(),
'link': result.artifacts,
}
run_meta = api.get_run_metadata(result.uuid, session)
for meta in run_meta:
if meta.key == 'build_name':
run['build_name'] = meta.value
break
runs.append(run)
return runs
def get_metadata(self):
if self._run_id:
session = self.session_factory()
metadata = db_api.get_run_metadata(self._run_id, session=session)
for meta in metadata:
if meta.key == 'stestr_run_meta':
return meta.value
return None
def get_recent_failed_runs_rss(run_metadata_key, value):
run_metadata_key = parse.unquote(run_metadata_key)
value = parse.unquote(value)
url = request.url
if run_metadata_key not in feeds:
feeds[run_metadata_key] = {value: _gen_feed(url,
run_metadata_key,
value)}
feeds["last runs"][run_metadata_key] = {value: None}
elif value not in feeds[run_metadata_key]:
feeds[run_metadata_key][value] = _gen_feed(url,
run_metadata_key,
value)
feeds["last runs"][run_metadata_key][value] = None
fg = feeds[run_metadata_key][value]
with session_scope() as session:
failed_runs = api.get_recent_failed_runs_by_run_metadata(
run_metadata_key, value,
start_date=feeds["last runs"][run_metadata_key][value],
session=session)
if failed_runs:
last_run = sorted([x.run_at for x in failed_runs])[-1]
if feeds["last runs"][run_metadata_key][value] == last_run:
return feeds[run_metadata_key][value].rss_str()
feeds["last runs"][run_metadata_key][value] = last_run
else:
count = api.get_runs_counts_by_run_metadata(
run_metadata_key, value, session=session)
if count == 0:
msg = 'No matching runs found with %s=%s' % (
run_metadata_key, value)
return abort(make_response(msg, 404))
for run in failed_runs:
meta = api.get_run_metadata(run.uuid, session=session)
uuid = [x.value for x in meta if x.key == 'build_uuid'][0]
build_name = [x.value for x in meta if x.key == 'build_name'][0]
entry = fg.add_entry()
entry.id(uuid)
entry.title('Failed Run %s/%s' % (build_name, uuid[:7]))
entry.published(pytz.utc.localize(run.run_at))
entry.link({'href': run.artifacts, 'rel': 'alternate'})
metadata_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'g/%s/%s' % (run_metadata_key, value))
job_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'job/%s' % build_name)
content = '<ul>'
content += '<li><a href="%s">Metadata page</a></li>\n' % (
metadata_url)
content += '<li><a href="%s">Job Page</a></li>' % (job_url)
content += '</ul>'
entry.description(content)
response = make_response(feeds[run_metadata_key][value].rss_str())
response.headers['Content-Type'] = 'application/xml; charset=utf-8'
return response
def get_recent_failed_runs_rss(run_metadata_key, value):
run_metadata_key = parse.unquote(run_metadata_key)
value = parse.unquote(value)
url = request.url
if run_metadata_key not in feeds:
feeds[run_metadata_key] = {value: _gen_feed(url,
run_metadata_key,
value)}
feeds["last runs"][run_metadata_key] = {value: None}
elif value not in feeds[run_metadata_key]:
feeds[run_metadata_key][value] = _gen_feed(url,
run_metadata_key,
value)
feeds["last runs"][run_metadata_key][value] = None
fg = feeds[run_metadata_key][value]
with session_scope() as session:
failed_runs = api.get_recent_failed_runs_by_run_metadata(
run_metadata_key, value,
start_date=feeds["last runs"][run_metadata_key][value],
session=session)
if failed_runs:
last_run = sorted([x.run_at for x in failed_runs])[-1]
if feeds["last runs"][run_metadata_key][value] == last_run:
return feeds[run_metadata_key][value].rss_str()
feeds["last runs"][run_metadata_key][value] = last_run
else:
msg = 'No Failed Runs for run metadata %s: %s' % (
run_metadata_key, value)
return abort(make_response(msg, 400))
for run in failed_runs:
meta = api.get_run_metadata(run.uuid, session=session)
uuid = [x.value for x in meta if x.key == 'build_uuid'][0]
entry = fg.add_entry()
entry.id(uuid)
entry.title('Failed Run %s' % uuid)
entry.published(pytz.utc.localize(run.run_at))
entry.link({'href': run.artifacts, 'rel': 'alternate'})
build_name = [x.value for x in meta if x.key == 'build_name'][0]
# TODO(mtreinish): Make this html
metadata_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'g/%s/%s' % (run_metadata_key, value))
job_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'job/%s' % build_name)
content = 'Metadata page: %s\n' % metadata_url
content += '\nJob Page %s' % job_url
entry.description(content)
return feeds[run_metadata_key][value].rss_str()
def _get_metadata(run_id):
"""Returns a dict of run_metadata objects associated with a run_id
:param run_id:
:return:
"""
engine = create_engine('mysql://*****:*****@logstash.openstack.org' +
':3306/subunit2sql')
Session = sessionmaker(bind=engine)
session = Session()
metadata = api.get_run_metadata(run_id,session=session)
ret_list = []
for meta in metadata:
ret_list.append(meta.to_dict())
return ret_list
def _get_recent(status):
with session_scope() as session:
failed_runs = api.get_recent_failed_runs(num_runs, session)
job_names = {}
for run in failed_runs:
metadata = api.get_run_metadata(run, session=session)
short_uuid = None
change_num = None
patch_num = None
for meta in metadata:
if meta.key == 'build_short_uuid':
short_uuid = meta.value
elif meta.key == 'build_change':
change_num = meta.value
elif meta.key == 'build_patchset':
patch_num = meta.value
elif meta.key == 'build_name':
job_names[run] = meta.value
global classifier
if classifier:
# NOTE(mtreinish): If the required metadata fields
# aren't present skip ES lookup
if not short_uuid or not change_num or not patch_num:
continue
query_thread = threading.Thread(
target=_populate_bug_dict,
args=(change_num, patch_num, short_uuid, run))
query_threads.append(query_thread)
query_thread.start()
test_runs = api.get_test_runs_by_status_for_run_ids(
status, failed_runs, session=session, include_run_id=True)
output = []
for run in test_runs:
run['start_time'] = run['start_time'].isoformat()
run['stop_time'] = run['stop_time'].isoformat()
run['job_name'] = job_names.get(run['uuid'])
output.append(run)
for thread in query_threads:
thread.join()
return {'test_runs': output, 'bugs': bug_dict}
def _get_recent(status):
with session_scope() as session:
failed_runs = api.get_recent_failed_runs(num_runs, session)
global classifier
if classifier:
for run in failed_runs:
metadata = api.get_run_metadata(run, session=session)
short_uuid = None
change_num = None
patch_num = None
for meta in metadata:
if meta.key == 'build_short_uuid':
short_uuid = meta.value
elif meta.key == 'build_change':
change_num = meta.value
elif meta.key == 'build_patchset':
patch_num = meta.value
# NOTE(mtreinish): If the required metadata fields aren't
# present skip ES lookup
if not short_uuid or not change_num or not patch_num:
continue
query_thread = threading.Thread(
target=_populate_bug_dict, args=(change_num, patch_num,
short_uuid, run))
query_threads.append(query_thread)
query_thread.start()
test_runs = api.get_test_runs_by_status_for_run_ids(
status, failed_runs, session=session, include_run_id=True)
output = []
for run in test_runs:
run['start_time'] = run['start_time'].isoformat()
run['stop_time'] = run['stop_time'].isoformat()
output.append(run)
for thread in query_threads:
thread.join()
return {'test_runs': output, 'bugs': bug_dict}
def _get_data(test_id, start_date, stop_date):
with session_scope() as session:
db_test_runs = api.get_test_runs_by_test_test_id(
test_id, session=session, start_date=start_date,
stop_date=stop_date)
if not db_test_runs:
# NOTE(mtreinish) if no data is returned from the DB just
# return an empty set response, the test_run_aggregator
# function assumes data is present.
return {'numeric': {}, 'data': {}, 'failed_runs': {}}
test_runs =\
test_run_aggregator.convert_test_runs_list_to_time_series_dict(
db_test_runs, datetime_resolution)
failed_run_ids = [
x.run_id for x in db_test_runs if x.status == 'fail']
failed_runs = api.get_runs_by_ids(failed_run_ids, session=session)
job_names = {}
providers = {}
failed_uuids = [x.uuid for x in failed_runs]
split_uuids = []
if len(failed_uuids) <= 10:
split_uuids = [[x] for x in failed_uuids]
else:
for i in range(0, len(failed_uuids), 10):
end = i + 10
split_uuids.append(failed_uuids[i:end])
for uuids in split_uuids:
change_dict = {}
for uuid in uuids:
metadata = api.get_run_metadata(uuid, session=session)
short_uuid = None
change_num = None
patch_num = None
for meta in metadata:
if meta.key == 'build_short_uuid':
short_uuid = meta.value
elif meta.key == 'build_change':
change_num = meta.value
elif meta.key == 'build_patchset':
patch_num = meta.value
elif meta.key == 'build_name':
job_names[uuid] = meta.value
elif meta.key == 'node_provider':
providers[uuid] = meta.value
# NOTE(mtreinish): If the required metadata fields
# aren't present skip ES lookup
if not short_uuid or not change_num or not patch_num:
continue
global classifier
if classifier:
change_dict[uuid] = {
'change_num': change_num,
'patch_num': patch_num,
'short_uuid': short_uuid,
}
query_thread = threading.Thread(
target=_populate_bug_dict, args=[change_dict])
query_threads.append(query_thread)
query_thread.start()
output = []
for thread in query_threads:
thread.join()
for run in failed_runs:
temp_run = {}
temp_run['provider'] = providers.get(run.uuid)
temp_run['job_name'] = job_names.get(run.uuid)
temp_run['run_at'] = run.run_at.isoformat()
temp_run['artifacts'] = run.artifacts
temp_run['bugs'] = bug_dict.get(run.uuid, [])
output.append(temp_run)
test_runs['failed_runs'] = output
return test_runs
def get_recent_failed_runs_rss(run_metadata_key, value):
run_metadata_key = parse.unquote(run_metadata_key)
value = parse.unquote(value)
url = request.url
if run_metadata_key not in feeds:
feeds[run_metadata_key] = {
value: _gen_feed(url, run_metadata_key, value)
}
feeds["last runs"][run_metadata_key] = {value: None}
elif value not in feeds[run_metadata_key]:
feeds[run_metadata_key][value] = _gen_feed(url, run_metadata_key,
value)
feeds["last runs"][run_metadata_key][value] = None
fg = feeds[run_metadata_key][value]
with session_scope() as session:
failed_runs = api.get_recent_failed_runs_by_run_metadata(
run_metadata_key,
value,
start_date=feeds["last runs"][run_metadata_key][value],
session=session)
if failed_runs:
last_run = sorted([x.run_at for x in failed_runs])[-1]
if feeds["last runs"][run_metadata_key][value] == last_run:
return feeds[run_metadata_key][value].rss_str()
feeds["last runs"][run_metadata_key][value] = last_run
else:
count = api.get_runs_counts_by_run_metadata(run_metadata_key,
value,
session=session)
if count == 0:
msg = 'No matching runs found with %s=%s' % (run_metadata_key,
value)
return abort(make_response(msg, 404))
for run in failed_runs:
meta = api.get_run_metadata(run.uuid, session=session)
failing_test_runs = api.get_failing_from_run(run.id,
session=session)
uuid = [x.value for x in meta if x.key == 'build_uuid'][0]
build_name = [x.value for x in meta if x.key == 'build_name'][0]
entry = fg.add_entry()
entry.id(uuid)
entry.title('Failed Run %s/%s' % (build_name, uuid[:7]))
entry.published(pytz.utc.localize(run.run_at))
entry.link({'href': run.artifacts, 'rel': 'alternate'})
metadata_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'g/%s/%s' % (run_metadata_key, value))
job_url = rss_opts['frontend_url'] + '/#/' + parse.quote(
'job/%s' % build_name)
content = '<ul>'
content += '<li><a href="%s">Metadata page</a></li>\n' % (
metadata_url)
content += '<li><a href="%s">Job Page</a></li>' % (job_url)
content += '</ul>'
content += '<h3>Failed tests</h3>'
content += '<ul>'
for failing_test_run in failing_test_runs:
content += '<li><a href="%s">%s</a></li>' % (
rss_opts['frontend_url'] + '/#/test/' +
failing_test_run.test.test_id,
failing_test_run.test.test_id)
content += '</ul>'
entry.description(content)
response = make_response(feeds[run_metadata_key][value].rss_str())
response.headers['Content-Type'] = 'application/xml; charset=utf-8'
return response
def _get_data(test_id, start_date, stop_date):
with session_scope() as session:
db_test_runs = api.get_test_runs_by_test_test_id(
test_id,
session=session,
start_date=start_date,
stop_date=stop_date)
if not db_test_runs:
# NOTE(mtreinish) if no data is returned from the DB just
# return an empty set response, the test_run_aggregator
# function assumes data is present.
return {'numeric': {}, 'data': {}, 'failed_runs': {}}
test_runs =\
test_run_aggregator.convert_test_runs_list_to_time_series_dict(
db_test_runs, datetime_resolution)
failed_run_ids = [
x.run_id for x in db_test_runs if x.status == 'fail'
]
failed_runs = api.get_runs_by_ids(failed_run_ids, session=session)
job_names = {}
providers = {}
failed_uuids = [x.uuid for x in failed_runs]
split_uuids = []
if len(failed_uuids) <= 10:
split_uuids = [[x] for x in failed_uuids]
else:
for i in range(0, len(failed_uuids), 10):
end = i + 10
split_uuids.append(failed_uuids[i:end])
for uuids in split_uuids:
change_dict = {}
for uuid in uuids:
metadata = api.get_run_metadata(uuid, session=session)
short_uuid = None
change_num = None
patch_num = None
for meta in metadata:
if meta.key == 'build_short_uuid':
short_uuid = meta.value
elif meta.key == 'build_change':
change_num = meta.value
elif meta.key == 'build_patchset':
patch_num = meta.value
elif meta.key == 'build_name':
job_names[uuid] = meta.value
elif meta.key == 'node_provider':
providers[uuid] = meta.value
# NOTE(mtreinish): If the required metadata fields
# aren't present skip ES lookup
if not short_uuid or not change_num or not patch_num:
continue
global classifier
if classifier:
change_dict[uuid] = {
'change_num': change_num,
'patch_num': patch_num,
'short_uuid': short_uuid,
}
query_thread = threading.Thread(target=_populate_bug_dict,
args=[change_dict])
query_threads.append(query_thread)
query_thread.start()
output = []
for thread in query_threads:
thread.join()
for run in failed_runs:
temp_run = {}
temp_run['provider'] = providers.get(run.uuid)
temp_run['job_name'] = job_names.get(run.uuid)
temp_run['run_at'] = run.run_at.isoformat()
temp_run['artifacts'] = run.artifacts
temp_run['bugs'] = bug_dict.get(run.uuid, [])
output.append(temp_run)
test_runs['failed_runs'] = output
return test_runs
def _get_result_for_run(run,
session,
use_db=True,
get_tests=False,
data_path=None,
s3=None):
# First try to get the data from disk
metadata_folder = get_data_path(data_path=data_path, s3=s3)
metadata_folder.append('.metadata')
use_s3 = (metadata_folder[0] == 's3:')
stream_or_file = False
# If s3 is None, get a vanilla s3 client just for the exceptions
s3 = s3 or get_s3_client()
if use_s3:
object_key = os.sep.join(metadata_folder[3:] + [run.uuid + '.json.gz'])
try:
s3.head_object(Bucket=metadata_folder[2], Key=object_key)
def stream_or_file():
return s3.get_object(Bucket=metadata_folder[2],
Key=object_key)['Body']
except s3.exceptions.ClientError:
# Not found, continue
pass
else:
os.makedirs(os.sep.join(metadata_folder), exist_ok=True)
result_file = os.sep.join(metadata_folder + [run.uuid + '.json.gz'])
if os.path.isfile(result_file):
def stream_or_file():
return result_file
# If cached
if stream_or_file:
try:
with gzip.open(stream_or_file(), mode='rt') as f:
if use_db:
# When using remote let me know if loading from cache
print("%s: metadata found in cache" % run.uuid)
return json.loads(f.read())
except IOError as ioe:
# Something went wrong opening the file, so we won't load
# this run.
print('Run %s found in the dataset, however: %s', (run.uuid, ioe))
return None
# If no local cache, and use_db is False, return nothing
if not use_db:
print("No local data for %s, use_db set to false" % run.uuid)
return None
# If no local cache, get data from the DB
result = {}
# We may need the list of tests
if get_tests:
test_runs = api.get_test_runs_by_run_id(run.uuid, session=session)
tests = []
for test_run in test_runs:
test = {'status': test_run.status}
start_time = test_run.start_time
start_time = start_time.replace(
microsecond=test_run.start_time_microsecond)
stop_time = test_run.stop_time
stop_time = stop_time.replace(
microsecond=test_run.stop_time_microsecond)
test['start_time'] = start_time
test['stop_time'] = stop_time
tests.append(test)
# Setup run metadata
if run.fails > 0 or run.passes == 0:
result['status'] = 1 # Failed
else:
result['status'] = 0 # Passed
result['artifact'] = run.artifacts
# Get extra run metadata
metadata = api.get_run_metadata(run.uuid, session)
for md in metadata:
result[md['key']] = md['value']
# Cache the json file, without tests
if use_s3:
s3.put_object(Bucket=metadata_folder[2],
Key=object_key,
Body=gzip.compress(json.dumps(result).encode()))
else:
with gzip.open(result_file, mode='wb') as local_cache:
local_cache.write(json.dumps(result).encode())
print("%s: metadata cached from URL" % run.uuid)
# Adding the tests after caching
if get_tests:
result['tests'] = tests
return result
