def load_mapping(self, file_path):
try:
self._mapping = PerfRepoMapping(file_path.resolve())
except:
logging.error("Failed to load PerfRepo mapping file '%s'" %\
file_path.abs_path())
self._mapping = None
def load_mapping(self, file_path):
try:
self._mapping = PerfRepoMapping(file_path.resolve())
except:
logging.error("Failed to load PerfRepo mapping file '%s'" %\
file_path.abs_path())
self._mapping = None
class PerfRepoAPI(object):
def __init__(self):
self._rest_api = None
self._mapping = None
def load_mapping(self, file_path):
try:
self._mapping = PerfRepoMapping(file_path.resolve())
except:
logging.error("Failed to load PerfRepo mapping file '%s'" %\
file_path.abs_path())
self._mapping = None
def get_mapping(self):
return self._mapping
def connected(self):
if self._rest_api is not None and self._rest_api.connected() and\
self._mapping is not None:
return True
else:
return False
def connect(self, url, username, password, max_retries=0):
if PerfRepoRESTAPI is not None:
self._rest_api = PerfRepoRESTAPI(url, username, password)
self._rest_api.set_retries(max_retries)
if not self._rest_api.connected():
self._rest_api = None
else:
self._rest_api = None
def new_result(self, mapping_key, name, hash_ignore=[]):
if not self.connected():
return Noop()
mapping_id = self._mapping.get_id(mapping_key)
if mapping_id is None:
logging.debug("Test key '%s' has no mapping defined!" % mapping_key)
return Noop()
logging.debug("Test key '%s' mapped to id '%s'" % (mapping_key,
mapping_id))
try:
test = self._rest_api.test_get_by_id(mapping_id, log=False)
except Exception as e:
test = None
logging.error(str(e))
if test is None:
try:
test = self._rest_api.test_get_by_uid(mapping_id, log=False)
except Exception as e:
test = None
logging.error(str(e))
if test is not None:
test_url = self._rest_api.get_obj_url(test)
logging.debug("Found Test with id='%s' and uid='%s'! %s" % \
(test.get_id(), test.get_uid(), test_url))
else:
logging.debug("No Test with id or uid '%s' found!" % mapping_id)
return Noop()
logging.info("Creating a new result object for PerfRepo")
result = PerfRepoResult(test, name, hash_ignore)
return result
def save_result(self, result, official_result=False):
if isinstance(result, Noop):
return
elif not self.connected():
raise TaskError("Not connected to PerfRepo.")
elif isinstance(result, PerfRepoResult):
if len(result.get_testExecution().get_values()) < 1:
logging.debug("PerfRepoResult with no result data, skipping "\
"send to PerfRepo.")
return
h = result.generate_hash()
if official_result:
logging.debug("Adding hash '%s' as tag to result." % h)
result.add_tag(h)
else:
logging.debug("Unofficial results - skipping hash tag.")
logging.info("Sending TestExecution to PerfRepo.")
try:
self._rest_api.testExecution_create(result.get_testExecution())
except Exception as e:
logging.error(str(e))
return
report_id = self._mapping.get_id(h)
if not report_id and result.get_testExecution().get_id() != None:
logging.debug("No mapping defined for hash '%s'" % h)
logging.debug("If you want to create a new report and set "\
"this result as the baseline run this command:")
cmd = "perfrepo-cli report create"
cmd += " name REPORTNAME"
test = result.get_test()
cmd += " chart CHARTNAME"
cmd += " testid %s" % test.get_id()
series_num = 0
for m in test.get_metrics():
cmd += " series NAME%d" % series_num
cmd += " metric %s" % m.get_id()
cmd += " tags %s" % h
series_num += 1
cmd += " baseline BASELINENAME"
cmd += " execid %s" % result.get_testExecution().get_id()
cmd += " metric %s" % test.get_metrics()[0].get_id()
logging.debug(cmd)
else:
raise TaskError("Parameter result must be an instance "\
"of PerfRepoResult")
def get_baseline(self, report_id):
if report_id is None or not self.connected():
return Noop()
try:
report = self._rest_api.report_get_by_id(report_id, log=False)
except Exception as e:
report = None
logging.error(str(e))
if report is None:
logging.debug("No report with id %s found!" % report_id)
return Noop()
logging.debug("Report found: %s" %\
self._rest_api.get_obj_url(report))
baseline = report.get_baseline()
if baseline is None:
logging.debug("No baseline set for report %s" %\
self._rest_api.get_obj_url(report))
return Noop()
baseline_exec_id = baseline["execId"]
try:
baseline_testExec = self._rest_api.testExecution_get(baseline_exec_id,
log=False)
except Exception as e:
baseline_testExec = None
logging.error(str(e))
if baseline_testExec is not None:
logging.debug("TestExecution of baseline: %s" %\
self._rest_api.get_obj_url(baseline_testExec))
else:
logging.debug("Couldn't get TestExecution of baseline.")
return Noop()
return PerfRepoBaseline(baseline_testExec)
def get_baseline_of_result(self, result):
if not isinstance(result, PerfRepoResult) or not self.connected():
return Noop()
res_hash = result.generate_hash()
logging.debug("Result hash is: '%s'" % res_hash)
report_id = self._mapping.get_id(res_hash)
if report_id is not None:
logging.debug("Hash '%s' maps to report id '%s'" % (res_hash,
report_id))
else:
logging.debug("Hash '%s' has no mapping defined!" % res_hash)
return Noop()
baseline = self.get_baseline(report_id)
if baseline.get_texec() is None:
logging.debug("No baseline set for results with hash %s" % res_hash)
return baseline
def compare_to_baseline(self, result, report_id, metric_name):
if not self.connected():
return False
baseline_testExec = self.get_baseline(report_id)
result_testExec = result.get_testExecution()
return self.compare_testExecutions(result_testExec,
baseline_testExec,
metric_name)
def compare_testExecutions(self, first, second, metric_name):
first_value = first.get_value(metric_name)
first_min = first.get_value(metric_name + "_min")
first_max = first.get_value(metric_name + "_max")
second_value = second.get_value(metric_name)
second_min = second.get_value(metric_name + "_min")
second_max = second.get_value(metric_name + "_max")
comp = second_value.get_comparator()
if comp == "HB":
if second_min.get_result() > first_max.get_result():
return False
return True
elif comp == "LB":
if first_min.get_result() > second_max.get_result():
return False
return True
else:
return False
return False
class PerfRepoAPI(object):
def __init__(self):
self._rest_api = None
self._mapping = None
def load_mapping(self, file_path):
try:
self._mapping = PerfRepoMapping(file_path.resolve())
except:
logging.error("Failed to load PerfRepo mapping file '%s'" %\
file_path.abs_path())
self._mapping = None
def get_mapping(self):
return self._mapping
def connected(self):
if self._rest_api is not None and self._rest_api.connected() and\
self._mapping is not None:
return True
else:
return False
def connect(self, url, username, password, max_retries=0):
if PerfRepoRESTAPI is not None:
self._rest_api = PerfRepoRESTAPI(url, username, password)
self._rest_api.set_retries(max_retries)
if not self._rest_api.connected():
self._rest_api = None
else:
self._rest_api = None
def new_result(self, mapping_key, name, hash_ignore=[]):
if not self.connected():
return Noop()
mapping_id = self._mapping.get_id(mapping_key)
if mapping_id is None:
logging.debug("Test key '%s' has no mapping defined!" % mapping_key)
return Noop()
logging.debug("Test key '%s' mapped to id '%s'" % (mapping_key,
mapping_id))
try:
test = self._rest_api.test_get_by_id(mapping_id, log=False)
except Exception as e:
test = None
logging.error(str(e))
if test is None:
try:
test = self._rest_api.test_get_by_uid(mapping_id, log=False)
except Exception as e:
test = None
logging.error(str(e))
if test is not None:
test_url = self._rest_api.get_obj_url(test)
logging.debug("Found Test with id='%s' and uid='%s'! %s" % \
(test.get_id(), test.get_uid(), test_url))
else:
logging.debug("No Test with id or uid '%s' found!" % mapping_id)
return Noop()
logging.info("Creating a new result object for PerfRepo")
result = PerfRepoResult(test, name, hash_ignore)
return result
def save_result(self, result, official_result=False):
if isinstance(result, Noop):
return
elif not self.connected():
raise TaskError("Not connected to PerfRepo.")
elif isinstance(result, PerfRepoResult):
if len(result.get_testExecution().get_values()) < 1:
logging.debug("PerfRepoResult with no result data, skipping "\
"send to PerfRepo.")
return
h = result.generate_hash()
if official_result:
logging.debug("Adding hash '%s' as tag to result." % h)
result.add_tag(h)
else:
logging.debug("Unofficial results - skipping hash tag.")
logging.info("Sending TestExecution to PerfRepo.")
try:
self._rest_api.testExecution_create(result.get_testExecution())
except Exception as e:
logging.error(str(e))
return
report_id = self._mapping.get_id(h)
if not report_id and result.get_testExecution().get_id() != None:
logging.debug("No mapping defined for hash '%s'" % h)
logging.debug("If you want to create a new report and set "\
"this result as the baseline run this command:")
cmd = "perfrepo-cli report create"
cmd += " name REPORTNAME"
test = result.get_test()
cmd += " chart CHARTNAME"
cmd += " testid %s" % test.get_id()
series_num = 0
for m in test.get_metrics():
cmd += " series NAME%d" % series_num
cmd += " metric %s" % m.get_id()
cmd += " tags %s" % h
series_num += 1
cmd += " baseline BASELINENAME"
cmd += " execid %s" % result.get_testExecution().get_id()
cmd += " metric %s" % test.get_metrics()[0].get_id()
logging.debug(cmd)
else:
raise TaskError("Parameter result must be an instance "\
"of PerfRepoResult")
def get_baseline(self, report_id):
if report_id is None or not self.connected():
return Noop()
try:
report = self._rest_api.report_get_by_id(report_id, log=False)
except Exception as e:
report = None
logging.error(str(e))
if report is None:
logging.debug("No report with id %s found!" % report_id)
return Noop()
logging.debug("Report found: %s" %\
self._rest_api.get_obj_url(report))
baseline = report.get_baseline()
if baseline is None:
logging.debug("No baseline set for report %s" %\
self._rest_api.get_obj_url(report))
return Noop()
baseline_exec_id = baseline["execId"]
try:
baseline_testExec = self._rest_api.testExecution_get(baseline_exec_id,
log=False)
except Exception as e:
baseline_testExec = None
logging.error(str(e))
if baseline_testExec is not None:
logging.debug("TestExecution of baseline: %s" %\
self._rest_api.get_obj_url(baseline_testExec))
else:
logging.debug("Couldn't get TestExecution of baseline.")
return Noop()
return PerfRepoBaseline(baseline_testExec)
def get_baseline_of_result(self, result):
if not isinstance(result, PerfRepoResult) or not self.connected():
return Noop()
res_hash = result.generate_hash()
logging.debug("Result hash is: '%s'" % res_hash)
report_id = self._mapping.get_id(res_hash)
if report_id is not None:
logging.debug("Hash '%s' maps to report id '%s'" % (res_hash,
report_id))
else:
logging.debug("Hash '%s' has no mapping defined!" % res_hash)
return Noop()
baseline = self.get_baseline(report_id)
if baseline.get_texec() is None:
logging.debug("No baseline set for results with hash %s" % res_hash)
return baseline
def compare_to_baseline(self, result, report_id, metric_name):
if not self.connected():
return False
baseline_testExec = self.get_baseline(report_id)
result_testExec = result.get_testExecution()
return self.compare_testExecutions(result_testExec,
baseline_testExec,
metric_name)
def compare_testExecutions(self, first, second, metric_name):
first_value = first.get_value(metric_name)
first_min = first.get_value(metric_name + "_min")
first_max = first.get_value(metric_name + "_max")
second_value = second.get_value(metric_name)
second_min = second.get_value(metric_name + "_min")
second_max = second.get_value(metric_name + "_max")
comp = second_value.get_comparator()
if comp == "HB":
if second_min.get_result() > first_max.get_result():
return False
return True
elif comp == "LB":
if first_min.get_result() > second_max.get_result():
return False
return True
else:
return False
return False