def get_by_params(cls, es_index=es_index, **params):
es_query = cls._get_es_query_from_instance_data(params)
filter_path = cls._get_es_filters()
es_data = ES().search(index=es_index,
q=es_query,
filter_path=filter_path,
size=10000)
if not es_data:
return []
es_data = es_data.get('hits', {}).get('hits', {})
if not es_data:
return []
return [cls(es_data=es_test_data) for es_test_data in es_data]
def get_prior_tests(self,
filter_path=None) -> typing.List['TestResultClass']:
output = []
es_query = self.get_same_tests_query()
es_result = ES().search(index=self._es_data['_index'],
q=es_query,
filter_path=filter_path,
size=10000) # pylint: disable=unexpected-keyword-arg
es_result = es_result.get('hits', {}).get('hits',
None) if es_result else None
if not es_result:
return output
for es_data in es_result: # pylint: disable=not-an-iterable
test = TestResultClass(es_data)
output.append(test)
return output
def get_by_params(cls, es_index=es_index, **params):
es_query = cls._get_es_query_from_instance_data(params)
filter_path = cls._get_es_filters()
try:
es_data = ES().search(index=es_index,
q=es_query,
filter_path=filter_path,
size=10000)
except Exception as exc: # pylint: disable=broad-except
LOGGER.warning("Unable to find ES data: %s", exc)
es_data = None
if not es_data:
return []
es_data = es_data.get('hits', {}).get('hits', {})
if not es_data:
return []
return [cls(es_data=es_test_data) for es_test_data in es_data]
def get_prior_tests(self,
filter_path=None) -> typing.List['TestResultClass']:
output = []
try:
es_query = self.get_same_tests_query()
es_result = ES().search( # pylint: disable=unexpected-keyword-arg; pylint doesn't understand Elasticsearch code
index=self._es_data['_index'],
q=es_query,
size=10000,
filter_path=filter_path,
)
es_result = es_result.get('hits', {}).get(
'hits', None) if es_result else None
except Exception as exc: # pylint: disable=broad-except
LOGGER.warning("Unable to find ES data: %s", exc)
es_result = None
if not es_result:
return output
for es_data in es_result: # pylint: disable=not-an-iterable
test = TestResultClass(es_data)
output.append(test)
return output
class BaseResultsAnalyzer(): # pylint: disable=too-many-instance-attributes
def __init__(
self,
es_index,
es_doc_type,
send_email=False,
email_recipients=(), # pylint: disable=too-many-arguments
email_template_fp="",
query_limit=1000,
logger=None):
self._es = ES()
self._conf = self._es._conf # pylint: disable=protected-access
self._es_index = es_index
self._es_doc_type = es_doc_type
self._limit = query_limit
self._send_email = send_email
self._email_recipients = email_recipients
self._email_template_fp = email_template_fp
self.log = logger if logger else LOGGER
def get_all(self):
"""
Get all the test results in json format
"""
return self._es.search(index=self._es_index, size=self._limit) # pylint: disable=unexpected-keyword-arg
def get_test_by_id(self, test_id):
"""
Get test results by test id
:param test_id: test id created by performance test
:return: test results in json format
"""
if not self._es.exists(
index=self._es_index, doc_type=self._es_doc_type, id=test_id):
self.log.error('Test results not found: {}'.format(test_id))
return None
return self._es.get(index=self._es_index,
doc_type=self._es_doc_type,
id=test_id)
def _test_version(self, test_doc):
if test_doc['_source'].get('versions'):
for value in ('scylla-server', 'scylla-enterprise-server'):
key = test_doc['_source']['versions'].get(value)
if key:
return key
self.log.error('Scylla version is not found for test %s',
test_doc['_id'])
return None
def render_to_html(self, results, html_file_path=""):
"""
Render analysis results to html template
:param results: results dictionary
:param html_file_path: Boolean, whether to save html file on disk
:return: html string
"""
self.log.info("Rendering results to html using '%s' template...",
self._email_template_fp)
loader = jinja2.FileSystemLoader(
os.path.dirname(os.path.abspath(__file__)))
print(os.path.dirname(os.path.abspath(__file__)))
env = jinja2.Environment(loader=loader,
autoescape=True,
extensions=['jinja2.ext.loopcontrols'])
template = env.get_template(self._email_template_fp)
html = template.render(results)
self.log.info("Results has been rendered to html")
if html_file_path:
with open(html_file_path, "w") as html_file:
html_file.write(html)
self.log.info("HTML report saved to '%s'.", html_file_path)
return html
def send_email(self, subject, content, html=True, files=()):
if self._send_email and self._email_recipients:
self.log.debug('Send email to {}'.format(self._email_recipients))
email = Email()
email.send(subject,
content,
html=html,
recipients=self._email_recipients,
files=files)
else:
self.log.warning(
"Won't send email (send_email: %s, recipients: %s)",
self._send_email, self._email_recipients)
def gen_kibana_dashboard_url(self, dashboard_path=""):
return "%s/%s" % (self._conf.get('kibana_url'), dashboard_path)