def table_performance_trending_dashboard(table, input_data):
"""Generate the table(s) with algorithm:
table_performance_trending_dashboard
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
header = [
"Test Case", "Trend [Mpps]", "Short-Term Change [%]",
"Long-Term Change [%]", "Regressions [#]", "Progressions [#]"
]
header_str = ",".join(header) + "\n"
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if tst_name.lower() in table["ignore-list"]:
continue
if tbl_dict.get(tst_name, None) is None:
groups = re.search(REGEX_NIC, tst_data["parent"])
if not groups:
continue
nic = groups.group(0)
tbl_dict[tst_name] = {
"name": "{0}-{1}".format(nic, tst_data["name"]),
"data": OrderedDict()
}
try:
tbl_dict[tst_name]["data"][str(build)] = \
tst_data["result"]["receive-rate"]
except (TypeError, KeyError):
pass # No data in output.xml for this test
tbl_lst = list()
for tst_name in tbl_dict.keys():
data_t = tbl_dict[tst_name]["data"]
if len(data_t) < 2:
continue
classification_lst, avgs = classify_anomalies(data_t)
win_size = min(len(data_t), table["window"])
long_win_size = min(len(data_t), table["long-trend-window"])
try:
max_long_avg = max(
[x for x in avgs[-long_win_size:-win_size] if not isnan(x)])
except ValueError:
max_long_avg = nan
last_avg = avgs[-1]
avg_week_ago = avgs[max(-win_size, -len(avgs))]
if isnan(last_avg) or isnan(avg_week_ago) or avg_week_ago == 0.0:
rel_change_last = nan
else:
rel_change_last = round(
((last_avg - avg_week_ago) / avg_week_ago) * 100, 2)
if isnan(max_long_avg) or isnan(last_avg) or max_long_avg == 0.0:
rel_change_long = nan
else:
rel_change_long = round(
((last_avg - max_long_avg) / max_long_avg) * 100, 2)
if classification_lst:
if isnan(rel_change_last) and isnan(rel_change_long):
continue
if (isnan(last_avg) or isnan(rel_change_last)
or isnan(rel_change_long)):
continue
tbl_lst.append([
tbl_dict[tst_name]["name"],
round(last_avg / 1000000, 2), rel_change_last, rel_change_long,
classification_lst[-win_size:].count("regression"),
classification_lst[-win_size:].count("progression")
])
tbl_lst.sort(key=lambda rel: rel[0])
tbl_sorted = list()
for nrr in range(table["window"], -1, -1):
tbl_reg = [item for item in tbl_lst if item[4] == nrr]
for nrp in range(table["window"], -1, -1):
tbl_out = [item for item in tbl_reg if item[5] == nrp]
tbl_out.sort(key=lambda rel: rel[2])
tbl_sorted.extend(tbl_out)
file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(file_name))
with open(file_name, "w") as file_handler:
file_handler.write(header_str)
for test in tbl_sorted:
file_handler.write(",".join([str(item) for item in test]) + '\n')
txt_file_name = "{0}.txt".format(table["output-file"])
logging.info(" Writing file: '{0}'".format(txt_file_name))
convert_csv_to_pretty_txt(file_name, txt_file_name)
def table_nics_comparison(table, input_data):
"""Generate the table(s) with algorithm: table_nics_comparison
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
try:
header = [
"Test case",
]
if table["include-tests"] == "MRR":
hdr_param = "Receive Rate"
else:
hdr_param = "Throughput"
header.extend([
"{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
"{0} Stdev [Mpps]".format(table["reference"]["title"]),
"{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
"{0} Stdev [Mpps]".format(table["compare"]["title"]), "Delta [%]"
])
header_str = ",".join(header) + "\n"
except (AttributeError, KeyError) as err:
logging.error(
"The model is invalid, missing parameter: {0}".format(err))
return
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
replace("-ndrpdr", "").replace("-pdrdisc", "").\
replace("-ndrdisc", "").replace("-pdr", "").\
replace("-ndr", "").\
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
tst_name_mod = re.sub(REGEX_NIC, "", tst_name_mod)
if tbl_dict.get(tst_name_mod, None) is None:
name = "-".join(tst_data["name"].split("-")[:-1])
tbl_dict[tst_name_mod] = {
"name": name,
"ref-data": list(),
"cmp-data": list()
}
try:
if table["include-tests"] == "MRR":
result = tst_data["result"]["receive-rate"].avg
elif table["include-tests"] == "PDR":
result = tst_data["throughput"]["PDR"]["LOWER"]
elif table["include-tests"] == "NDR":
result = tst_data["throughput"]["NDR"]["LOWER"]
else:
result = None
if result:
if table["reference"]["nic"] in tst_data["tags"]:
tbl_dict[tst_name_mod]["ref-data"].append(result)
elif table["compare"]["nic"] in tst_data["tags"]:
tbl_dict[tst_name_mod]["cmp-data"].append(result)
except (TypeError, KeyError) as err:
logging.debug("No data for {0}".format(tst_name))
logging.debug(repr(err))
# No data in output.xml for this test
tbl_lst = list()
for tst_name in tbl_dict.keys():
item = [
tbl_dict[tst_name]["name"],
]
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
if item[-4] is not None and item[-2] is not None and item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
if len(item) == len(header):
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate csv tables:
csv_file = "{0}.csv".format(table["output-file"])
with open(csv_file, "w") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
def table_soak_vs_ndr(table, input_data):
"""Generate the table(s) with algorithm: table_soak_vs_ndr
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the table
try:
header = [
"Test case",
"{0} Throughput [Mpps]".format(table["reference"]["title"]),
"{0} Stdev [Mpps]".format(table["reference"]["title"]),
"{0} Throughput [Mpps]".format(table["compare"]["title"]),
"{0} Stdev [Mpps]".format(table["compare"]["title"]), "Delta [%]",
"Stdev of delta [%]"
]
header_str = ",".join(header) + "\n"
except (AttributeError, KeyError) as err:
logging.error(
"The model is invalid, missing parameter: {0}".format(err))
return
# Create a list of available SOAK test results:
tbl_dict = dict()
for job, builds in table["compare"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if tst_data["type"] == "SOAK":
tst_name_mod = tst_name.replace("-soak", "")
if tbl_dict.get(tst_name_mod, None) is None:
groups = re.search(REGEX_NIC, tst_data["parent"])
nic = groups.group(0) if groups else ""
name = "{0}-{1}".format(
nic, "-".join(tst_data["name"].split("-")[:-1]))
tbl_dict[tst_name_mod] = {
"name": name,
"ref-data": list(),
"cmp-data": list()
}
try:
tbl_dict[tst_name_mod]["cmp-data"].append(
tst_data["throughput"]["LOWER"])
except (KeyError, TypeError):
pass
tests_lst = tbl_dict.keys()
# Add corresponding NDR test results:
for job, builds in table["reference"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdr", "").\
replace("-mrr", "")
if tst_name_mod in tests_lst:
try:
if tst_data["type"] in ("NDRPDR", "MRR", "BMRR"):
if table["include-tests"] == "MRR":
result = tst_data["result"]["receive-rate"].avg
elif table["include-tests"] == "PDR":
result = tst_data["throughput"]["PDR"]["LOWER"]
elif table["include-tests"] == "NDR":
result = tst_data["throughput"]["NDR"]["LOWER"]
else:
result = None
if result is not None:
tbl_dict[tst_name_mod]["ref-data"].append(
result)
except (KeyError, TypeError):
continue
tbl_lst = list()
for tst_name in tbl_dict.keys():
item = [
tbl_dict[tst_name]["name"],
]
data_r = tbl_dict[tst_name]["ref-data"]
if data_r:
data_r_mean = mean(data_r)
item.append(round(data_r_mean / 1000000, 2))
data_r_stdev = stdev(data_r)
item.append(round(data_r_stdev / 1000000, 2))
else:
data_r_mean = None
data_r_stdev = None
item.extend([None, None])
data_c = tbl_dict[tst_name]["cmp-data"]
if data_c:
data_c_mean = mean(data_c)
item.append(round(data_c_mean / 1000000, 2))
data_c_stdev = stdev(data_c)
item.append(round(data_c_stdev / 1000000, 2))
else:
data_c_mean = None
data_c_stdev = None
item.extend([None, None])
if data_r_mean and data_c_mean:
delta, d_stdev = relative_change_stdev(data_r_mean, data_c_mean,
data_r_stdev, data_c_stdev)
item.append(round(delta, 2))
item.append(round(d_stdev, 2))
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate csv tables:
csv_file = "{0}.csv".format(table["output-file"])
with open(csv_file, "w") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
def table_performance_comparison(table, input_data):
"""Generate the table(s) with algorithm: table_performance_comparison
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
try:
header = [
"Test case",
]
if table["include-tests"] == "MRR":
hdr_param = "Receive Rate"
else:
hdr_param = "Throughput"
history = table.get("history", None)
if history:
for item in history:
header.extend([
"{0} {1} [Mpps]".format(item["title"], hdr_param),
"{0} Stdev [Mpps]".format(item["title"])
])
header.extend([
"{0} {1} [Mpps]".format(table["reference"]["title"], hdr_param),
"{0} Stdev [Mpps]".format(table["reference"]["title"]),
"{0} {1} [Mpps]".format(table["compare"]["title"], hdr_param),
"{0} Stdev [Mpps]".format(table["compare"]["title"]), "Delta [%]"
])
header_str = ",".join(header) + "\n"
except (AttributeError, KeyError) as err:
logging.error(
"The model is invalid, missing parameter: {0}".format(err))
return
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdrdisc", "").\
replace("-ndrpdr", "").replace("-pdrdisc", "").\
replace("-ndrdisc", "").replace("-pdr", "").\
replace("-ndr", "").\
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
groups = re.search(REGEX_NIC, tst_data["parent"])
nic = groups.group(0) if groups else ""
name = "{0}-{1}".format(
nic, "-".join(tst_data["name"].split("-")[:-1]))
if "across testbeds" in table["title"].lower() or \
"across topologies" in table["title"].lower():
name = name.\
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
tbl_dict[tst_name_mod] = {
"name": name,
"ref-data": list(),
"cmp-data": list()
}
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
tbl_dict[tst_name_mod]["ref-data"]. \
append(tst_data["result"]["receive-rate"].avg)
elif table["include-tests"] == "PDR":
if tst_data["type"] == "PDR":
tbl_dict[tst_name_mod]["ref-data"]. \
append(tst_data["throughput"]["value"])
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["ref-data"].append(
tst_data["throughput"]["PDR"]["LOWER"])
elif table["include-tests"] == "NDR":
if tst_data["type"] == "NDR":
tbl_dict[tst_name_mod]["ref-data"]. \
append(tst_data["throughput"]["value"])
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["ref-data"].append(
tst_data["throughput"]["NDR"]["LOWER"])
else:
continue
except TypeError:
pass # No data in output.xml for this test
for job, builds in table["compare"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
replace("-ndrpdr", "").replace("-pdrdisc", ""). \
replace("-ndrdisc", "").replace("-pdr", ""). \
replace("-ndr", "").\
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
tbl_dict[tst_name_mod]["cmp-data"]. \
append(tst_data["result"]["receive-rate"].avg)
elif table["include-tests"] == "PDR":
if tst_data["type"] == "PDR":
tbl_dict[tst_name_mod]["cmp-data"]. \
append(tst_data["throughput"]["value"])
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["cmp-data"].append(
tst_data["throughput"]["PDR"]["LOWER"])
elif table["include-tests"] == "NDR":
if tst_data["type"] == "NDR":
tbl_dict[tst_name_mod]["cmp-data"]. \
append(tst_data["throughput"]["value"])
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["cmp-data"].append(
tst_data["throughput"]["NDR"]["LOWER"])
else:
continue
except KeyError:
pass
except TypeError:
tbl_dict.pop(tst_name_mod, None)
if history:
for item in history:
for job, builds in item["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(
build)].iteritems():
tst_name_mod = tst_name.replace("-ndrpdrdisc", ""). \
replace("-ndrpdr", "").replace("-pdrdisc", ""). \
replace("-ndrdisc", "").replace("-pdr", ""). \
replace("-ndr", "").\
replace("1t1c", "1c").replace("2t1c", "1c").\
replace("2t2c", "2c").replace("4t2c", "2c").\
replace("4t4c", "4c").replace("8t4c", "4c")
if "across topologies" in table["title"].lower():
tst_name_mod = tst_name_mod.replace("2n1l-", "")
if tbl_dict.get(tst_name_mod, None) is None:
continue
if tbl_dict[tst_name_mod].get("history", None) is None:
tbl_dict[tst_name_mod]["history"] = OrderedDict()
if tbl_dict[tst_name_mod]["history"].get(
item["title"], None) is None:
tbl_dict[tst_name_mod]["history"][item["title"]] = \
list()
try:
# TODO: Re-work when NDRPDRDISC tests are not used
if table["include-tests"] == "MRR":
tbl_dict[tst_name_mod]["history"][
item["title"]].append(
tst_data["result"]["receive-rate"].avg)
elif table["include-tests"] == "PDR":
if tst_data["type"] == "PDR":
tbl_dict[tst_name_mod]["history"][
item["title"]].\
append(tst_data["throughput"]["value"])
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["history"][
item["title"]].append(
tst_data["throughput"]["PDR"]
["LOWER"])
elif table["include-tests"] == "NDR":
if tst_data["type"] == "NDR":
tbl_dict[tst_name_mod]["history"][
item["title"]].\
append(tst_data["throughput"]["value"])
elif tst_data["type"] == "NDRPDR":
tbl_dict[tst_name_mod]["history"][
item["title"]].append(
tst_data["throughput"]["NDR"]
["LOWER"])
else:
continue
except (TypeError, KeyError):
pass
tbl_lst = list()
for tst_name in tbl_dict.keys():
item = [
tbl_dict[tst_name]["name"],
]
if history:
if tbl_dict[tst_name].get("history", None) is not None:
for hist_data in tbl_dict[tst_name]["history"].values():
if hist_data:
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
item.extend([None, None])
else:
item.extend([None, None])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
if item[-4] is not None and item[-2] is not None and item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
if len(item) == len(header):
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate csv tables:
csv_file = "{0}.csv".format(table["output-file"])
with open(csv_file, "w") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
file_handler.write(",".join([str(item) for item in test]) + "\n")
convert_csv_to_pretty_txt(csv_file, "{0}.txt".format(table["output-file"]))
def table_failed_tests(table, input_data):
"""Generate the table(s) with algorithm: table_failed_tests
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
header = [
"Test Case", "Failures [#]", "Last Failure [Time]",
"Last Failure [VPP-Build-Id]", "Last Failure [CSIT-Job-Build-Id]"
]
# Generate the data for the table according to the model in the table
# specification
now = dt.utcnow()
timeperiod = timedelta(int(table.get("window", 7)))
tbl_dict = dict()
for job, builds in table["data"].items():
for build in builds:
build = str(build)
for tst_name, tst_data in data[job][build].iteritems():
if tst_name.lower() in table["ignore-list"]:
continue
if tbl_dict.get(tst_name, None) is None:
groups = re.search(REGEX_NIC, tst_data["parent"])
if not groups:
continue
nic = groups.group(0)
tbl_dict[tst_name] = {
"name": "{0}-{1}".format(nic, tst_data["name"]),
"data": OrderedDict()
}
try:
generated = input_data.metadata(job, build).\
get("generated", "")
if not generated:
continue
then = dt.strptime(generated, "%Y%m%d %H:%M")
if (now - then) <= timeperiod:
tbl_dict[tst_name]["data"][build] = (
tst_data["status"], generated,
input_data.metadata(job, build).get("version",
""), build)
except (TypeError, KeyError) as err:
logging.warning("tst_name: {} - err: {}".format(
tst_name, repr(err)))
max_fails = 0
tbl_lst = list()
for tst_data in tbl_dict.values():
fails_nr = 0
for val in tst_data["data"].values():
if val[0] == "FAIL":
fails_nr += 1
fails_last_date = val[1]
fails_last_vpp = val[2]
fails_last_csit = val[3]
if fails_nr:
max_fails = fails_nr if fails_nr > max_fails else max_fails
tbl_lst.append([
tst_data["name"], fails_nr, fails_last_date, fails_last_vpp,
"mrr-daily-build-{0}".format(fails_last_csit)
])
tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
tbl_sorted = list()
for nrf in range(max_fails, -1, -1):
tbl_fails = [item for item in tbl_lst if item[1] == nrf]
tbl_sorted.extend(tbl_fails)
file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(file_name))
with open(file_name, "w") as file_handler:
file_handler.write(",".join(header) + "\n")
for test in tbl_sorted:
file_handler.write(",".join([str(item) for item in test]) + '\n')
txt_file_name = "{0}.txt".format(table["output-file"])
logging.info(" Writing file: '{0}'".format(txt_file_name))
convert_csv_to_pretty_txt(file_name, txt_file_name)
def table_failed_tests(table, input_data):
"""Generate the table(s) with algorithm: table_failed_tests
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
header = [
"Test Case", "Failures [#]", "Last Failure [Time]",
"Last Failure [VPP-Build-Id]", "Last Failure [CSIT-Job-Build-Id]"
]
# Generate the data for the table according to the model in the table
# specification
tbl_dict = dict()
for job, builds in table["data"].items():
for build in builds:
build = str(build)
for tst_name, tst_data in data[job][build].iteritems():
if tst_name.lower() in table["ignore-list"]:
continue
if tbl_dict.get(tst_name, None) is None:
name = "{0}-{1}".format(
tst_data["parent"].split("-")[0],
"-".join(tst_data["name"].split("-")[1:]))
tbl_dict[tst_name] = {"name": name, "data": OrderedDict()}
try:
tbl_dict[tst_name]["data"][build] = (tst_data["status"],
input_data.metadata(
job, build).get(
"generated",
""),
input_data.metadata(
job, build).get(
"version",
""), build)
except (TypeError, KeyError):
pass # No data in output.xml for this test
tbl_lst = list()
for tst_data in tbl_dict.values():
win_size = min(len(tst_data["data"]), table["window"])
fails_nr = 0
for val in tst_data["data"].values()[-win_size:]:
if val[0] == "FAIL":
fails_nr += 1
fails_last_date = val[1]
fails_last_vpp = val[2]
fails_last_csit = val[3]
if fails_nr:
tbl_lst.append([
tst_data["name"], fails_nr, fails_last_date, fails_last_vpp,
"mrr-daily-build-{0}".format(fails_last_csit)
])
tbl_lst.sort(key=lambda rel: rel[2], reverse=True)
tbl_sorted = list()
for nrf in range(table["window"], -1, -1):
tbl_fails = [item for item in tbl_lst if item[1] == nrf]
tbl_sorted.extend(tbl_fails)
file_name = "{0}{1}".format(table["output-file"], table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(file_name))
with open(file_name, "w") as file_handler:
file_handler.write(",".join(header) + "\n")
for test in tbl_sorted:
file_handler.write(",".join([str(item) for item in test]) + '\n')
txt_file_name = "{0}.txt".format(table["output-file"])
logging.info(" Writing file: '{0}'".format(txt_file_name))
convert_csv_to_pretty_txt(file_name, txt_file_name)
def table_performance_comparison_mrr(table, input_data):
"""Generate the table(s) with algorithm: table_performance_comparison_mrr
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
try:
header = [
"Test case",
"{0} Throughput [Mpps]".format(table["reference"]["title"]),
"{0} stdev [Mpps]".format(table["reference"]["title"]),
"{0} Throughput [Mpps]".format(table["compare"]["title"]),
"{0} stdev [Mpps]".format(table["compare"]["title"]), "Change [%]"
]
header_str = ",".join(header) + "\n"
except (AttributeError, KeyError) as err:
logging.error(
"The model is invalid, missing parameter: {0}".format(err))
return
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if tbl_dict.get(tst_name, None) is None:
name = "{0}-{1}".format(
tst_data["parent"].split("-")[0],
"-".join(tst_data["name"].split("-")[1:]))
tbl_dict[tst_name] = {
"name": name,
"ref-data": list(),
"cmp-data": list()
}
try:
tbl_dict[tst_name]["ref-data"].\
append(tst_data["result"]["receive-rate"].avg)
except TypeError:
pass # No data in output.xml for this test
for job, builds in table["compare"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
try:
tbl_dict[tst_name]["cmp-data"].\
append(tst_data["result"]["receive-rate"].avg)
except KeyError:
pass
except TypeError:
tbl_dict.pop(tst_name, None)
tbl_lst = list()
for tst_name in tbl_dict.keys():
item = [
tbl_dict[tst_name]["name"],
]
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
if item[1] is not None and item[3] is not None and item[1] != 0:
item.append(int(relative_change(float(item[1]), float(item[3]))))
if len(item) == 6:
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate tables:
# All tests in csv:
tbl_names = [
"{0}-1t1c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-2t2c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-4t4c-full{1}".format(table["output-file"],
table["output-file-ext"])
]
for file_name in tbl_names:
logging.info(" Writing file: '{0}'".format(file_name))
with open(file_name, "w") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
if file_name.split("-")[-2] in test[0]: # cores
test[0] = "-".join(test[0].split("-")[:-1])
file_handler.write(",".join([str(item)
for item in test]) + "\n")
# All tests in txt:
tbl_names_txt = [
"{0}-1t1c-full.txt".format(table["output-file"]),
"{0}-2t2c-full.txt".format(table["output-file"]),
"{0}-4t4c-full.txt".format(table["output-file"])
]
for i, txt_name in enumerate(tbl_names_txt):
logging.info(" Writing file: '{0}'".format(txt_name))
convert_csv_to_pretty_txt(tbl_names[i], txt_name)
def table_performance_comparison(table, input_data):
"""Generate the table(s) with algorithm: table_performance_comparison
specified in the specification file.
:param table: Table to generate.
:param input_data: Data to process.
:type table: pandas.Series
:type input_data: InputData
"""
logging.info(" Generating the table {0} ...".format(table.get(
"title", "")))
# Transform the data
logging.info(" Creating the data set for the {0} '{1}'.".format(
table.get("type", ""), table.get("title", "")))
data = input_data.filter_data(table, continue_on_error=True)
# Prepare the header of the tables
try:
header = [
"Test case",
]
history = table.get("history", None)
if history:
for item in history:
header.extend([
"{0} Throughput [Mpps]".format(item["title"]),
"{0} Stdev [Mpps]".format(item["title"])
])
header.extend([
"{0} Throughput [Mpps]".format(table["reference"]["title"]),
"{0} Stdev [Mpps]".format(table["reference"]["title"]),
"{0} Throughput [Mpps]".format(table["compare"]["title"]),
"{0} Stdev [Mpps]".format(table["compare"]["title"]), "Change [%]"
])
header_str = ",".join(header) + "\n"
except (AttributeError, KeyError) as err:
logging.error(
"The model is invalid, missing parameter: {0}".format(err))
return
# Prepare data to the table:
tbl_dict = dict()
for job, builds in table["reference"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
if tbl_dict.get(tst_name, None) is None:
name = "{0}-{1}".format(
tst_data["parent"].split("-")[0],
"-".join(tst_data["name"].split("-")[1:]))
tbl_dict[tst_name] = {
"name": name,
"ref-data": list(),
"cmp-data": list()
}
try:
tbl_dict[tst_name]["ref-data"].\
append(tst_data["throughput"]["value"])
except TypeError:
pass # No data in output.xml for this test
for job, builds in table["compare"]["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(build)].iteritems():
try:
tbl_dict[tst_name]["cmp-data"].\
append(tst_data["throughput"]["value"])
except KeyError:
pass
except TypeError:
tbl_dict.pop(tst_name, None)
if history:
for item in history:
for job, builds in item["data"].items():
for build in builds:
for tst_name, tst_data in data[job][str(
build)].iteritems():
if tbl_dict.get(tst_name, None) is None:
continue
if tbl_dict[tst_name].get("history", None) is None:
tbl_dict[tst_name]["history"] = OrderedDict()
if tbl_dict[tst_name]["history"].get(
item["title"], None) is None:
tbl_dict[tst_name]["history"][item["title"]] = \
list()
try:
tbl_dict[tst_name]["history"][item["title"]].\
append(tst_data["throughput"]["value"])
except (TypeError, KeyError):
pass
tbl_lst = list()
for tst_name in tbl_dict.keys():
item = [
tbl_dict[tst_name]["name"],
]
if history:
if tbl_dict[tst_name].get("history", None) is not None:
for hist_data in tbl_dict[tst_name]["history"].values():
if hist_data:
item.append(round(mean(hist_data) / 1000000, 2))
item.append(round(stdev(hist_data) / 1000000, 2))
else:
item.extend([None, None])
else:
item.extend([None, None])
data_t = tbl_dict[tst_name]["ref-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
data_t = tbl_dict[tst_name]["cmp-data"]
if data_t:
item.append(round(mean(data_t) / 1000000, 2))
item.append(round(stdev(data_t) / 1000000, 2))
else:
item.extend([None, None])
if item[-4] is not None and item[-2] is not None and item[-4] != 0:
item.append(int(relative_change(float(item[-4]), float(item[-2]))))
if len(item) == len(header):
tbl_lst.append(item)
# Sort the table according to the relative change
tbl_lst.sort(key=lambda rel: rel[-1], reverse=True)
# Generate tables:
# All tests in csv:
tbl_names = [
"{0}-ndr-1t1c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-ndr-2t2c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-ndr-4t4c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-pdr-1t1c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-pdr-2t2c-full{1}".format(table["output-file"],
table["output-file-ext"]),
"{0}-pdr-4t4c-full{1}".format(table["output-file"],
table["output-file-ext"])
]
for file_name in tbl_names:
logging.info(" Writing file: '{0}'".format(file_name))
with open(file_name, "w") as file_handler:
file_handler.write(header_str)
for test in tbl_lst:
if (file_name.split("-")[-3] in test[0] and # NDR vs PDR
file_name.split("-")[-2] in test[0]): # cores
test[0] = "-".join(test[0].split("-")[:-1])
file_handler.write(",".join([str(item)
for item in test]) + "\n")
# All tests in txt:
tbl_names_txt = [
"{0}-ndr-1t1c-full.txt".format(table["output-file"]),
"{0}-ndr-2t2c-full.txt".format(table["output-file"]),
"{0}-ndr-4t4c-full.txt".format(table["output-file"]),
"{0}-pdr-1t1c-full.txt".format(table["output-file"]),
"{0}-pdr-2t2c-full.txt".format(table["output-file"]),
"{0}-pdr-4t4c-full.txt".format(table["output-file"])
]
for i, txt_name in enumerate(tbl_names_txt):
logging.info(" Writing file: '{0}'".format(txt_name))
convert_csv_to_pretty_txt(tbl_names[i], txt_name)
# Selected tests in csv:
input_file = "{0}-ndr-1t1c-full{1}".format(table["output-file"],
table["output-file-ext"])
with open(input_file, "r") as in_file:
lines = list()
for line in in_file:
lines.append(line)
output_file = "{0}-ndr-1t1c-top{1}".format(table["output-file"],
table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(output_file))
with open(output_file, "w") as out_file:
out_file.write(header_str)
for i, line in enumerate(lines[1:]):
if i == table["nr-of-tests-shown"]:
break
out_file.write(line)
output_file = "{0}-ndr-1t1c-bottom{1}".format(table["output-file"],
table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(output_file))
with open(output_file, "w") as out_file:
out_file.write(header_str)
for i, line in enumerate(lines[-1:0:-1]):
if i == table["nr-of-tests-shown"]:
break
out_file.write(line)
input_file = "{0}-pdr-1t1c-full{1}".format(table["output-file"],
table["output-file-ext"])
with open(input_file, "r") as in_file:
lines = list()
for line in in_file:
lines.append(line)
output_file = "{0}-pdr-1t1c-top{1}".format(table["output-file"],
table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(output_file))
with open(output_file, "w") as out_file:
out_file.write(header_str)
for i, line in enumerate(lines[1:]):
if i == table["nr-of-tests-shown"]:
break
out_file.write(line)
output_file = "{0}-pdr-1t1c-bottom{1}".format(table["output-file"],
table["output-file-ext"])
logging.info(" Writing file: '{0}'".format(output_file))
with open(output_file, "w") as out_file:
out_file.write(header_str)
for i, line in enumerate(lines[-1:0:-1]):
if i == table["nr-of-tests-shown"]:
break
out_file.write(line)
