def from_s3(overwrite=False):
"""Import test results from S3 and insert it to database."""
s3 = S3.factory(bucket='juju-qa-data', directory='cwr')
ds = Datastore()
for key in s3.list(filter_fun=_filter_fun):
if not overwrite and not doc_needs_update(key):
app.logger.debug('Skip importing {}'.format(key.name))
continue
app.logger.info('Import new data from {}'.format(key.name))
job_name = get_meta_data(key.name, 'job_name')
build_number = get_meta_data(key.name, 'build_number')
uploader_number = get_meta_data(key.name, 'uploader_build_number')
build_info = json.loads(key.get_contents_as_string())
artifacts = get_artifacts(build_info)
test_result_path = get_test_path(
artifacts, job_name, build_number, uploader_number)
if not test_result_path:
app.logger.error(
"Test result file not found for key: {} ".format(key.name))
continue
svg_path = make_path(
artifacts, job_name, build_number, uploader_number, "result.svg")
html_path = make_path(
artifacts, job_name, build_number, uploader_number, "result.html")
json_path = make_path(
artifacts, job_name, build_number, uploader_number, "result.json")
test_key = s3.get(test_result_path)
test = json.loads(test_key.get_contents_as_string())
doc = make_doc(build_info, test, job_name, key, artifacts, svg_path,
html_path, json_path)
ds.update({'_id': _get_id(key)}, doc)
def generate_chart_data(self):
ds = Datastore()
test_ids = list(ds.get_test_ids(
bundle=self.name, date=self.bundle.get('date')))
if test_ids:
test_ids.reverse()
provider_names = self.get_provider_names(test_ids)
datasets = self._create_initial_datasets(provider_names)
title = None
units = None
direction = None
for test_id in test_ids:
tests = ds.get({'test_id': test_id['_id']})
title, units, direction = self._generate_datasets(
datasets, tests, provider_names)
if not title:
return None
self.calculate_avg_benchmark(datasets)
title = "{} Benchmark Chart (Units: {} Direction: {})".format(
title.title(), units, direction)
chart_data = {
'labels': [x['_id'][-5:] for x in test_ids],
'datasets': datasets,
'title': title,
}
return json.dumps(chart_data)
def from_s3(overwrite=False):
"""Import test results from S3 and insert it to database."""
s3 = S3.factory(bucket='juju-qa-data', directory='cwr')
ds = Datastore()
for key in s3.list(filter_fun=_filter_fun):
if not overwrite and not doc_needs_update(key):
app.logger.debug('Skip importing {}'.format(key.name))
continue
app.logger.info('Import new data from {}'.format(key.name))
job_name = get_meta_data(key.name, 'job_name')
build_number = get_meta_data(key.name, 'build_number')
uploader_number = get_meta_data(key.name, 'uploader_build_number')
build_info = json.loads(key.get_contents_as_string())
artifacts = get_artifacts(build_info)
test_result_path = get_test_path(artifacts, job_name, build_number,
uploader_number)
if not test_result_path:
app.logger.error("Test result file not found for key: {} ".format(
key.name))
continue
svg_path = get_svg_path(artifacts, job_name, build_number,
uploader_number)
test_key = s3.get(test_result_path)
test = json.loads(test_key.get_contents_as_string())
doc = make_doc(build_info, test, job_name, key, artifacts, svg_path)
ds.update({'_id': _get_id(key)}, doc)
def test_get_filter(self):
doc = make_doc()
doc2 = make_doc(2)
update_data(self.ds, doc)
update_data(self.ds, doc2)
ds = Datastore()
items = list(ds.get(filter={'_id': doc['_id']}))
self.assertEqual(items, [doc])
def test_get_one(self):
doc = make_doc()
doc2 = make_doc(2)
update_data(self.ds, doc)
update_data(self.ds, doc2)
ds = Datastore()
item = ds.get_one(filter={'_id': doc['_id']})
self.assertEqual(item, doc)
def test_get_by_bundle_name(self):
doc = make_doc(1)
doc2 = make_doc(2)
update_data(self.ds, doc)
update_data(self.ds, doc2)
ds = Datastore()
items = list(ds.get_by_bundle_name(doc['bundle_name']))
self.assertEqual(items, [doc])
def test_update(self):
doc = make_doc()
ds = Datastore()
with patch('cwrstatus.datastore.get_current_utc_time', autospec=True,
return_value=doc['_updated_on']) as gcut_mock:
ds.update({"_id": doc["_id"]}, doc)
items = list(self.ds.db.cwr.find())
self.assertEqual(items, [doc])
gcut_mock.assert_called_once_with()
def test_get_multiple(self):
doc = make_doc()
doc2 = make_doc(2)
update_data(self.ds, doc)
update_data(self.ds, doc2)
ds = Datastore()
items = list(ds.get())
self.assertEqual(items[0], doc2)
self.assertEqual(items[1], doc)
def test_get_limit(self):
doc = make_doc(1)
doc2 = make_doc(2)
doc3 = make_doc(3)
update_data(self.ds, doc)
update_data(self.ds, doc2)
update_data(self.ds, doc3)
ds = Datastore()
items = list(ds.get(limit=2))
self.assertEqual(items, [doc3, doc2])
def test_get_skip(self):
doc = make_doc(1)
doc2 = make_doc(2)
doc3 = make_doc(3)
update_data(self.ds, doc)
update_data(self.ds, doc2)
update_data(self.ds, doc3)
ds = Datastore()
items = list(ds.get(skip=1))
self.assertEqual(items, [doc2, doc])
def doc_needs_update(key):
""" Determine if a doc in database needs an update.
:param key: document key
:rtype: boolean
"""
_id = _get_id(key)
ds = Datastore()
result = ds.get_one({'_id': _id})
if result and result.get('etag') == key.etag:
return False
return True
def doc_needs_update(key):
""" Determine if a doc in database needs an update.
:param key: document key
:rtype: boolean
"""
_id = _get_id(key)
ds = Datastore()
result = ds.get_one({'_id': _id})
if result and result.get('etag') == key.etag:
return False
return True
def bundles(page):
ds = Datastore()
limit = PAGE_LIMIT
skip = limit * (abs(page) - 1)
cwr_results, count = ds.distinct(key='bundle_name', limit=limit, skip=skip)
bundle_title = 'List of all bundles'
pagination = Pagination(
page=page, total_count=count, limit_per_page=limit,
request=request)
return render_template(
'bundles.html', cwr_results=cwr_results, bundle_title=bundle_title,
pagination=pagination)
def recent_by_bundle(bundle, page):
ds = Datastore()
filter = {'bundle_name': bundle}
limit = PAGE_LIMIT
skip = limit * (abs(page) - 1)
cwr_results = ds.get(filter=filter, limit=limit, skip=skip)
pagination = Pagination(
page=page, total_count=cwr_results.count(), limit_per_page=limit,
request=request)
bundle_title = 'Recent tests: {}'.format(bundle)
return render_template(
'recent.html', cwr_results=cwr_results, bundle_title=bundle_title,
pagination=pagination)
def recent(page):
ds = Datastore()
limit = PAGE_LIMIT
skip = limit * (abs(page) - 1)
cwr_results = ds.get(limit=limit, skip=skip)
bundle_title = 'Recent tests'
pagination = Pagination(
page=page, total_count=cwr_results.count(), limit_per_page=limit,
request=request)
print cwr_results.count(), limit
return render_template(
'recent.html', cwr_results=cwr_results, bundle_title=bundle_title,
pagination=pagination)
def test_get_test_ids_by_bundle(self):
doc = make_doc()
doc['bundle_name'] = 'foo'
update_data(self.ds, doc)
doc = make_doc(2)
doc['bundle_name'] = 'foo'
update_data(self.ds, doc)
doc = make_doc(3, test_id="44")
update_data(self.ds, doc)
ds = Datastore()
distinct = ds.get_test_ids(bundle='foo')
distinct = list(distinct)
expected = [{'date': '1', '_id': '33'}]
self.assertEqual(distinct, expected)
def test_distinct(self):
doc = make_doc()
doc['bundle_name'] = 'foo'
update_data(self.ds, doc)
doc = make_doc(2)
doc['bundle_name'] = 'foo'
update_data(self.ds, doc)
doc = make_doc(3)
update_data(self.ds, doc)
ds = Datastore()
distinct, count = ds.distinct()
distinct = list(distinct)
expected = [{'count': 2, '_id': 'foo'},
{'count': 1, '_id': 'openstack3'}]
self.assertItemsEqual(distinct, expected)
self.assertEqual(count, 2)
def bundle_view(key=None):
if not key:
return render_template('404.html', e='Bundle not found.'), 404
ds = Datastore()
cwr_result = ds.get_one({'_id': key})
if not cwr_result:
return render_template('404.html', e='Bundle not found.'), 404
bundle = Bundle(cwr_result)
svg_path = bundle.svg_path()
bundle_name = bundle.name
results = bundle.test_result()
history = None
chart_data = bundle.generate_chart_data()
return render_template(
'bundle.html', bundle_name=bundle_name, results=results,
svg_path=svg_path, history=history, chart_data=chart_data)
def test_generate_match_filter(self):
match = Datastore._generate_match_filter()
self.assertEqual(match, {})
match = Datastore._generate_match_filter(bundle='foo')
expected = {"bundle_name": 'foo'}
self.assertEqual(match, expected)
match = Datastore._generate_match_filter(date='bar')
expected = {"date": {"$lte": 'bar'}}
self.assertEqual(match, expected)
match = Datastore._generate_match_filter(bundle='foo', date='bar')
expected = {
"$and": [
{"bundle_name": 'foo'},
{"date": {"$lte": 'bar'}}
]
}
self.assertEqual(match, expected)
def test_get_test_ids_by_date(self):
doc = make_doc()
doc['bundle_name'] = 'foo'
doc['date'] = "1"
update_data(self.ds, doc)
doc = make_doc(2)
doc['bundle_name'] = 'foo'
doc['date'] = "2"
update_data(self.ds, doc)
doc = make_doc(3, test_id="44")
doc['bundle_name'] = 'foo'
doc['date'] = "3"
update_data(self.ds, doc)
ds = Datastore()
test_ids = ds.get_test_ids(bundle='foo', date="2")
test_ids = list(test_ids)
expected = [{'date': '1', '_id': '33'}]
self.assertEqual(test_ids, expected)
def test_update_existing_doc(self):
doc = make_doc()
ds = Datastore()
with patch('cwrstatus.datastore.get_current_utc_time', autospec=True,
return_value=doc['_updated_on']):
ds.update({"_id": doc["_id"]}, doc)
items = list(self.ds.db.cwr.find())
self.assertEqual(items, [doc])
doc['bundle_name'] = 'new bundle'
ds.update({"_id": doc["_id"]}, doc)
items = list(self.ds.db.cwr.find())
self.assertEqual(items, [doc])
def get_recent_test_result(page, limit, bundle=None):
ds = Datastore()
skip = limit * (abs(page) - 1)
test_ids = ds.get_test_ids(bundle=bundle, limit=limit, skip=skip)
cwr_results = get_results_by_test_ids(test_ids)
return cwr_results
def __init__(self, bundle):
self.bundle = bundle
self.name = bundle.get('bundle_name')
self.build_info = bundle.get('build_info')
self.test = bundle.get('test') or {}
self.ds = Datastore()
def __init__(self, bundle):
self.bundle = bundle
self.name = bundle.get('bundle_name')
self.build_info = bundle.get('build_info')
self.test = None
self.ds = Datastore()
class Bundle:
def __init__(self, bundle):
self.bundle = bundle
self.name = bundle.get('bundle_name')
self.build_info = bundle.get('build_info')
self.test = None
self.ds = Datastore()
def get_past_tests(self):
filter = {"$and": [
{"bundle_name": self.name},
{"date": {'$lt': self.bundle['date']}},
]}
return self.ds.get(filter=filter)
def get_past_benchmarks(self, provider_name, past_results):
values = []
for result in past_results:
result = result.get('test') or {}
if result.get('results'):
for test_result in result['results']:
if (test_result.get('benchmarks') and
provider_name == test_result['provider_name']):
try:
values.append(
test_result['benchmarks'][0].values()[0]
["value"])
except (KeyError, AttributeError):
raise Exception(
'Non standardized benchmark format.')
# Latest data on right side of the graph
if values:
values.reverse()
return values
def add_test_result(self, result):
if result.get('test') and result.get('test').get('results'):
for x in result['test']['results']:
x['build_id'] = result["_id"]
if not self.test and result.get('test'):
self.test = result.get('test')
elif result.get('test') and result.get('test').get('results'):
self.test['results'].extend(result['test']['results'])
def test_result(self):
return self.test
@staticmethod
def calculate_avg_benchmark(datasets):
for dataset in datasets:
data = [float(x) for x in dataset['data'] if x]
if not data:
dataset['label'] = '{} '.format(dataset['label'])
continue
avg = sum(data) / float(len(data))
differences = [x - avg for x in data]
sq_diff = [d ** 2 for d in differences]
ssd = sum(sq_diff)
variance = ssd / len(data)
standard_deviation = sqrt(variance)
dataset['label'] = '{} ({:.2f} avg {:.2f} sd)'.format(
dataset['label'], avg, standard_deviation)
def generate_chart_data(self):
ds = Datastore()
test_ids = list(ds.get_test_ids(
bundle=self.name, date=self.bundle.get('date')))
if test_ids:
test_ids.reverse()
provider_names = self.get_provider_names(test_ids)
datasets = self._create_initial_datasets(provider_names)
title = None
units = None
direction = None
for test_id in test_ids:
tests = ds.get({'test_id': test_id['_id']})
title, units, direction = self._generate_datasets(
datasets, tests, provider_names)
if not title:
return None
self.calculate_avg_benchmark(datasets)
title = "{} Benchmark Chart (Units: {} Direction: {})".format(
title.title(), units, direction)
chart_data = {
'labels': [x['_id'][-5:] for x in test_ids],
'datasets': datasets,
'title': title,
}
return json.dumps(chart_data)
def _generate_datasets(self, datasets, tests, provider_names):
title = None
units = None
direction = None
for provider_name in provider_names:
data = self._get_dataset(datasets, provider_name)
data['data'].append(None)
for test in tests:
test = test.get('test') or {}
for result in test.get('results', []):
provider_name = result.get('provider_name')
data = self._get_dataset(datasets, provider_name)
benchmarks = result.get('benchmarks')
if not benchmarks:
continue
data['data'][-1] = benchmarks[0].values()[0]['value']
title = benchmarks[0].keys()[0]
units = benchmarks[0][title].get('units', '')
direction = benchmarks[0][title].get('direction', '')
return title, units, direction
@staticmethod
def _create_initial_datasets(provider_names):
border_colors = ['#4B98D9', '#56CE65', '#FFA342', '#AA54AD', '#DC654B',
'#E66BB3']
datasets = []
for provider_name, color in zip(provider_names, border_colors):
data = {
'label': provider_name,
'fill': False,
'borderColor': color,
'borderWidth': 2,
'backgroundColor': color,
'lineTension': 0.1,
'data': [],
}
datasets.append(data)
return datasets
@staticmethod
def _get_dataset(datasets, provider_name):
for data in datasets:
if provider_name == data['label']:
return data
return None
def svg_path(self):
svg_path = self.bundle.get('svg_path')
if not svg_path:
return 'No Image'
return 'http://data.vapour.ws/cwr/{}'.format(svg_path)
@staticmethod
def get_provider_names(test_ids):
ds = Datastore()
provider_names = []
for result in Bundle.iter_results_by_test_ids(test_ids, ds):
provider_name = result.get('provider_name')
if not provider_name:
continue
if provider_name not in provider_names:
provider_names.append(provider_name)
return sorted(provider_names)
@staticmethod
def iter_results_by_test_ids(test_ids, ds):
for test_id in test_ids:
tests = ds.get({'test_id': test_id['_id']})
for test in tests:
test = test.get('test') or {}
for result in test.get('results', []):
yield result
class Bundle:
def __init__(self, bundle):
self.bundle = bundle
self.name = bundle.get('bundle_name')
self.build_info = bundle.get('build_info')
self.test = bundle.get('test') or {}
self.ds = Datastore()
def get_past_tests(self):
filter = {
"$and": [
{
"bundle_name": self.name
},
{
"date": {
'$lt': self.bundle['date']
}
},
]
}
return self.ds.get(filter=filter)
def get_past_benchmarks(self, provider_name, past_results):
values = []
for result in past_results:
result = result.get('test') or {}
if result.get('results'):
for test_result in result['results']:
if (test_result.get('benchmarks')
and provider_name == test_result['provider_name']):
try:
values.append(test_result['benchmarks']
[0].values()[0]["value"])
except (KeyError, AttributeError):
raise Exception(
'Non standardized benchmark format.')
# Latest data on right side of the graph
if values:
values.reverse()
return values
def test_result(self):
return self.test
def generate_chart_data(self):
"""Generate benchmarks by joining the past and current benchmark data.
:rtype: dict
"""
past_results = list(self.get_past_tests())
title = 'No data'
series = []
yaxis_tilte = ''
for test_result in self.test.get('results'):
data = []
benchmarks = test_result.get('benchmarks')
if benchmarks:
past_ben = self.get_past_benchmarks(
test_result.get('provider_name'), past_results)
data = [benchmarks[0].values()[0]['value']]
if past_ben:
data = past_ben + data
yaxis_tilte = benchmarks[0].values()[0].get('units')
title = benchmarks[0].keys()[0]
series.append({
'name': test_result.get('provider_name'),
'data': map(int, data) if data else []
})
chart = {'title': title, 'yaxis_title': yaxis_tilte, 'series': series}
return json.dumps(chart)
def svg_path(self):
svg_path = self.bundle.get('svg_path')
if not svg_path:
return 'No Image'
return 'http://data.vapour.ws/cwr/{}'.format(svg_path)
def test_get(self):
doc = make_doc()
update_data(self.ds, doc)
ds = Datastore()
items = list(ds.get())
self.assertEqual(items, [doc])
class Bundle:
def __init__(self, bundle):
self.bundle = bundle
self.name = bundle.get('bundle_name')
self.build_info = bundle.get('build_info')
self.test = bundle.get('test') or {}
self.ds = Datastore()
def get_past_tests(self):
filter = {"$and": [
{"bundle_name": self.name},
{"date": {'$lt': self.bundle['date']}},
]}
return self.ds.get(filter=filter)
def get_past_benchmarks(self, provider_name, past_results):
values = []
for result in past_results:
result = result.get('test') or {}
if result.get('results'):
for test_result in result['results']:
if (test_result.get('benchmarks') and
provider_name == test_result['provider_name']):
try:
values.append(
test_result['benchmarks'][0].values()[0]
["value"])
except (KeyError, AttributeError):
raise Exception(
'Non standardized benchmark format.')
# Latest data on right side of the graph
if values:
values.reverse()
return values
def test_result(self):
return self.test
def generate_chart_data(self):
"""Generate benchmarks by joining the past and current benchmark data.
:rtype: dict
"""
past_results = list(self.get_past_tests())
title = 'No data'
series = []
yaxis_tilte = ''
for test_result in self.test.get('results'):
data = []
benchmarks = test_result.get('benchmarks')
if benchmarks:
past_ben = self.get_past_benchmarks(
test_result.get('provider_name'), past_results)
data = [benchmarks[0].values()[0]['value']]
if past_ben:
data = past_ben + data
yaxis_tilte = benchmarks[0].values()[0].get('units')
title = benchmarks[0].keys()[0]
series.append(
{
'name': test_result.get('provider_name'),
'data': map(int, data) if data else []
}
)
chart = {
'title': title,
'yaxis_title': yaxis_tilte,
'series': series
}
return json.dumps(chart)
def svg_path(self):
svg_path = self.bundle.get('svg_path')
if not svg_path:
return 'No Image'
return 'http://data.vapour.ws/cwr/{}'.format(svg_path)