def test_file_handler(self):
"""Test for the file handler."""
test_raw_data = str(self.parent_path / 'test_results.jsonl')
test_rule_file = str(self.parent_path / 'test_rules.yaml')
test_baseline_file = str(self.parent_path / 'test_baseline.json')
test_raw_data_fake = str(self.parent_path / 'test_results_fake.jsonl')
test_rule_file_fake = str(self.parent_path / 'test_rules_fake.yaml')
test_aseline_file_fake = str(self.parent_path / 'test_baseline_fake.json')
# Test - read_raw_data
raw_data_df = file_handler.read_raw_data(test_raw_data)
assert (not raw_data_df.empty)
raw_data_df = file_handler.read_raw_data(test_raw_data_fake)
assert (raw_data_df.empty)
# Test - read rules
rules = file_handler.read_rules(test_rule_file_fake)
assert (not rules)
rules = file_handler.read_rules(test_rule_file)
assert (rules)
# Test - read baseline
baseline = file_handler.read_baseline(test_aseline_file_fake)
assert (not baseline)
baseline = file_handler.read_baseline(test_baseline_file)
assert (baseline)
# Test - generate_md_table
data_df = pd.DataFrame([[1, 2], [3, 4]])
lines = file_handler.generate_md_table(data_df, header=['A', 'B'])
expected_lines = ['| A | B |\n', '| --- | --- |\n', '| 1 | 2 |\n', '| 3 | 4 |\n']
assert (lines == expected_lines)
def _preprocess(self, raw_data_file, rule_file):
"""Preprocess/preparation operations for the rules.
Args:
raw_data_file (str): the path of raw data file
rule_file (str): the path of rule file
Returns:
dict: dict of rules
"""
# read raw data from file
self._raw_data_df = file_handler.read_raw_data(raw_data_file)
# re-organize metrics by benchmark names
self._benchmark_metrics_dict = self._get_metrics_by_benchmarks(list(self._raw_data_df.columns))
# check raw data whether empty
if len(self._raw_data_df) == 0:
logger.error('RuleBase: empty raw data')
return None
# read rules
rules = file_handler.read_rules(rule_file)
return rules
def test_no_matched_rule(self):
"""Test for support no matching rules."""
# Positive case
rules = {
'superbench': {
'rules': {
'fake': {
'categories': 'FAKE',
'statistics': ['mean', 'max'],
'metrics': ['abb/fake:\\d+'],
'aggregate': True
}
}
}
}
rs1 = ResultSummary()
rs1._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
rs1._benchmark_metrics_dict = rs1._get_metrics_by_benchmarks(
list(rs1._raw_data_df))
assert (rs1._parse_rules(rules))
summary = rs1._generate_summary(round=2)
assert (len(summary) == 1)
assert (summary['FAKE'] == [['FAKE', '', 'mean', ''],
['FAKE', '', 'max', '']])
def test_rule_base(self):
"""Test for rule-based functions."""
# Test - read_raw_data and get_metrics_from_raw_data
# Positive case
test_raw_data = str(self.parent_path / 'test_results.jsonl')
test_rule_file = str(self.parent_path / 'test_rules.yaml')
rulebase1 = RuleBase()
rulebase1._raw_data_df = file_handler.read_raw_data(test_raw_data)
rulebase1._benchmark_metrics_dict = rulebase1._get_metrics_by_benchmarks(
list(rulebase1._raw_data_df))
assert (len(rulebase1._raw_data_df) == 3)
# Negative case
test_rule_file_fake = str(self.parent_path / 'test_rules_fake.yaml')
test_raw_data_fake = str(self.parent_path / 'test_results_fake.jsonl')
rulebase2 = RuleBase()
rulebase2._raw_data_df = file_handler.read_raw_data(test_raw_data_fake)
rulebase2._benchmark_metrics_dict = rulebase2._get_metrics_by_benchmarks(
list(rulebase2._raw_data_df))
assert (len(rulebase2._raw_data_df) == 0)
assert (len(rulebase2._benchmark_metrics_dict) == 0)
metric_list = [
'gpu_temperature', 'gpu_power_limit', 'gemm-flops/FP64',
'bert_models/pytorch-bert-base/steptime_train_float32'
]
self.assertDictEqual(
rulebase2._get_metrics_by_benchmarks(metric_list), {
'gemm-flops': {'gemm-flops/FP64'},
'bert_models':
{'bert_models/pytorch-bert-base/steptime_train_float32'}
})
# Test - _preprocess
rules = rulebase1._preprocess(test_raw_data_fake, test_rule_file)
assert (not rules)
rules = rulebase1._preprocess(test_raw_data, test_rule_file_fake)
assert (not rules)
rules = rulebase1._preprocess(test_raw_data, test_rule_file)
assert (rules)
# Test - _check_and_format_rules
# Negative case
false_rule = {
'criteria': 'lambda x:x>0',
'function': 'variance',
'metrics': ['kernel-launch/event_overhead:\\d+']
}
metric = 'kernel-launch/event_overhead:0'
self.assertRaises(Exception, rulebase1._check_and_format_rules,
false_rule, metric)
# Positive case
true_rule = {
'categories': 'KernelLaunch',
'criteria': 'lambda x:x<-0.05',
'function': 'variance',
'metrics': 'kernel-launch/event_overhead:\\d+'
}
true_rule = rulebase1._check_and_format_rules(true_rule, metric)
assert (true_rule)
assert (true_rule['metrics'] == ['kernel-launch/event_overhead:\\d+'])
# Test - _get_metrics
rules = rules['superbench']['rules']
for rule in ['rule0', 'rule1']:
rulebase1._sb_rules[rule] = {}
rulebase1._sb_rules[rule]['metrics'] = {}
rulebase1._get_metrics(rule, rules)
assert (len(rulebase1._sb_rules[rule]['metrics']) == 16)
def test_result_summary(self):
"""Test result summary class."""
rs1 = ResultSummary()
rs1._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
rs1._benchmark_metrics_dict = rs1._get_metrics_by_benchmarks(
list(rs1._raw_data_df))
# Test - _check_rules
# Negative case
false_rules = [{
'categories': 'KernelLaunch',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'categories': 'KernelLaunch',
'statistics': 'abb',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'categories': 'KernelLaunch',
'statistics': 'mean',
'metrics': ['kernel-launch/event_overhead:\\d+'],
'aggregate': 'abb'
}]
metric = 'kernel-launch/event_overhead:0'
for rules in false_rules:
self.assertRaises(Exception, rs1._check_rules, rules, metric)
# Positive case
true_rules = [
{
'categories': 'KernelLaunch',
'statistics': 'mean',
'metrics': ['kernel-launch/event_overhead:\\d+'],
'aggregate': True
},
{
'categories': 'KernelLaunch',
'statistics': ['mean', 'p50'],
'metrics': ['kernel-launch/event_overhead:\\d+']
},
{
'categories': 'KernelLaunch',
'statistics': 'mean',
'metrics': ['kernel-launch/event_overhead:\\d+'],
'aggregate': 'kernel-launch/event_overhead(:\\d+)'
},
]
for rules in true_rules:
assert (rs1._check_rules(rules, metric))
# Test - _parse_rules
# Negative case
rs2 = ResultSummary()
fake_rules = file_handler.read_rules(self.test_rule_file_fake)
assert (rs2._parse_rules(fake_rules) is False)
rs2._raw_data_df = file_handler.read_raw_data(self.test_raw_data)
rs2._benchmark_metrics_dict = rs2._get_metrics_by_benchmarks(
list(rs2._raw_data_df))
p = Path(self.test_rule_file)
with p.open() as f:
rules = yaml.load(f, Loader=yaml.SafeLoader)
rules['superbench']['rules']['fake'] = false_rules[0]
with open(self.test_rule_file_fake, 'w') as f:
yaml.dump(rules, f)
assert (rs1._parse_rules(fake_rules) is False)
# Positive case
rules = file_handler.read_rules(self.test_rule_file)
assert (rs1._parse_rules(rules))
# Test - _generate_summary
summary = rs1._generate_summary(round=2)
assert (len(summary) == 3)
# Test - _merge_summary
expected_summary_merge = [
['KernelLaunch', 'kernel-launch/event_overhead', 'mean', 0.0097],
['KernelLaunch', 'kernel-launch/event_overhead', 'p90', 0.006],
['KernelLaunch', 'kernel-launch/event_overhead', 'min', 0.0055],
['KernelLaunch', 'kernel-launch/event_overhead', 'max', 0.1],
['KernelLaunch', 'kernel-launch/wall_overhead', 'mean', 0.01],
['KernelLaunch', 'kernel-launch/wall_overhead', 'p90', 0.011],
['KernelLaunch', 'kernel-launch/wall_overhead', 'min', 0.01],
['KernelLaunch', 'kernel-launch/wall_overhead', 'max', 0.011],
['NCCL', 'nccl-bw/allreduce_8388608_busbw:0', 'mean', 89.51],
['RDMA', 'ib-loopback/IB_write_8388608_Avg_*:0', 'mean', 23925.84]
]
expected_summary_merge_df = pd.DataFrame(expected_summary_merge)
summary_merge_df = rs1._merge_summary(summary)
pd.testing.assert_frame_equal(expected_summary_merge_df,
summary_merge_df)
def test_data_diagnosis(self):
"""Test for rule-based data diagnosis."""
# Test - read_raw_data and get_metrics_from_raw_data
# Positive case
test_raw_data = str(self.parent_path / 'test_results.jsonl')
test_rule_file = str(self.parent_path / 'test_rules.yaml')
test_baseline_file = str(self.parent_path / 'test_baseline.json')
diag1 = DataDiagnosis()
diag1._raw_data_df = file_handler.read_raw_data(test_raw_data)
diag1._benchmark_metrics_dict = diag1._get_metrics_by_benchmarks(
list(diag1._raw_data_df))
assert (len(diag1._raw_data_df) == 3)
# Negative case
test_raw_data_fake = str(self.parent_path / 'test_results_fake.jsonl')
test_rule_file_fake = str(self.parent_path / 'test_rules_fake.yaml')
diag2 = DataDiagnosis()
diag2._raw_data_df = file_handler.read_raw_data(test_raw_data_fake)
diag2._benchmark_metrics_dict = diag2._get_metrics_by_benchmarks(
list(diag2._raw_data_df))
assert (len(diag2._raw_data_df) == 0)
assert (len(diag2._benchmark_metrics_dict) == 0)
metric_list = [
'gpu_temperature', 'gpu_power_limit', 'gemm-flops/FP64',
'bert_models/pytorch-bert-base/steptime_train_float32'
]
self.assertDictEqual(
diag2._get_metrics_by_benchmarks(metric_list), {
'gemm-flops': {'gemm-flops/FP64'},
'bert_models':
{'bert_models/pytorch-bert-base/steptime_train_float32'}
})
# Test - read rules
rules = file_handler.read_rules(test_rule_file_fake)
assert (not rules)
rules = file_handler.read_rules(test_rule_file)
assert (rules)
# Test - _check_and_format_rules
# Negative case
false_rules = [{
'criteria': 'lambda x:x>0',
'categories': 'KernelLaunch',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'criteria': 'lambda x:x>0',
'function': 'variance',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'categories': 'KernelLaunch',
'function': 'variance',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'criteria': 'lambda x:x>0',
'function': 'abb',
'categories': 'KernelLaunch',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'criteria': 'lambda x:x>0',
'function': 'abb',
'categories': 'KernelLaunch',
}, {
'criteria': 'x>5',
'function': 'abb',
'categories': 'KernelLaunch',
'metrics': ['kernel-launch/event_overhead:\\d+']
}]
metric = 'kernel-launch/event_overhead:0'
for rules in false_rules:
self.assertRaises(Exception, diag1._check_and_format_rules, rules,
metric)
# Positive case
true_rules = [{
'categories': 'KernelLaunch',
'criteria': 'lambda x:x>0.05',
'function': 'variance',
'metrics': ['kernel-launch/event_overhead:\\d+']
}, {
'categories': 'KernelLaunch',
'criteria': 'lambda x:x<-0.05',
'function': 'variance',
'metrics': 'kernel-launch/event_overhead:\\d+'
}, {
'categories': 'KernelLaunch',
'criteria': 'lambda x:x>0',
'function': 'value',
'metrics': ['kernel-launch/event_overhead:\\d+']
}]
for rules in true_rules:
assert (diag1._check_and_format_rules(rules, metric))
# Test - _get_baseline_of_metric
baseline = file_handler.read_baseline(test_baseline_file)
assert (diag1._get_baseline_of_metric(
baseline, 'kernel-launch/event_overhead:0') == 0.00596)
assert (diag1._get_baseline_of_metric(
baseline, 'kernel-launch/return_code') == 0)
assert (diag1._get_baseline_of_metric(baseline, 'mem-bw/H2D:0') == -1)
# Test - _parse_rules_and_baseline
# Negative case
fake_rules = file_handler.read_rules(test_rule_file_fake)
baseline = file_handler.read_baseline(test_baseline_file)
assert (diag2._parse_rules_and_baseline(fake_rules, baseline) is False)
diag2 = DataDiagnosis()
diag2._raw_data_df = file_handler.read_raw_data(test_raw_data)
diag2._benchmark_metrics_dict = diag2._get_metrics_by_benchmarks(
list(diag2._raw_data_df))
p = Path(test_rule_file)
with p.open() as f:
rules = yaml.load(f, Loader=yaml.SafeLoader)
rules['superbench']['rules']['fake'] = false_rules[0]
with open(test_rule_file_fake, 'w') as f:
yaml.dump(rules, f)
assert (diag1._parse_rules_and_baseline(fake_rules, baseline) is False)
# Positive case
rules = file_handler.read_rules(test_rule_file)
assert (diag1._parse_rules_and_baseline(rules, baseline))
# Test - _run_diagnosis_rules_for_single_node
(details_row, summary_data_row
) = diag1._run_diagnosis_rules_for_single_node('sb-validation-01')
assert (details_row)
(details_row, summary_data_row
) = diag1._run_diagnosis_rules_for_single_node('sb-validation-02')
assert (not details_row)
# Test - _run_diagnosis_rules
baseline = file_handler.read_baseline(test_baseline_file)
data_not_accept_df, label_df = diag1.run_diagnosis_rules(
rules, baseline)
assert (len(label_df) == 3)
assert (label_df.loc['sb-validation-01']['label'] == 1)
assert (label_df.loc['sb-validation-02']['label'] == 0)
assert (label_df.loc['sb-validation-03']['label'] == 1)
node = 'sb-validation-01'
row = data_not_accept_df.loc[node]
assert (len(row) == 36)
assert (row['Category'] == 'KernelLaunch')
assert (
row['Defective Details'] ==
'kernel-launch/event_overhead:0(B/L: 0.0060 VAL: 0.1000 VAR: 1577.85% Rule:lambda x:x>0.05)'
)
node = 'sb-validation-03'
row = data_not_accept_df.loc[node]
assert (len(row) == 36)
assert ('FailedTest' in row['Category'])
assert ('mem-bw/return_code(VAL: 1.0000 Rule:lambda x:x>0)'
in row['Defective Details'])
assert ('mem-bw/H2D_Mem_BW:0_miss' in row['Defective Details'])
assert (len(data_not_accept_df) == 2)
# Test - output in excel
diag1.output_diagnosis_in_excel(diag1._raw_data_df, data_not_accept_df,
self.output_excel_file,
diag1._sb_rules)
excel_file = pd.ExcelFile(self.output_excel_file, engine='openpyxl')
data_sheet_name = 'Raw Data'
raw_data_df = excel_file.parse(data_sheet_name)
assert (len(raw_data_df) == 3)
data_sheet_name = 'Not Accept'
data_not_accept_read_from_excel = excel_file.parse(data_sheet_name)
assert (len(data_not_accept_read_from_excel) == 2)
assert ('Category' in data_not_accept_read_from_excel)
assert ('Defective Details' in data_not_accept_read_from_excel)
# Test - output in jsonl
diag1.output_diagnosis_in_jsonl(data_not_accept_df,
self.output_json_file)
assert (Path(self.output_json_file).is_file())
with Path(self.output_json_file).open() as f:
data_not_accept_read_from_json = f.readlines()
assert (len(data_not_accept_read_from_json) == 2)
for line in data_not_accept_read_from_json:
json.loads(line)
assert ('Category' in line)
assert ('Defective Details' in line)
assert ('Index' in line)
# Test - generate_md_lines
lines = diag1.generate_md_lines(data_not_accept_df, diag1._sb_rules, 2)
assert (lines)
expected_md_file = str(self.parent_path /
'../data/diagnosis_summary.md')
with open(expected_md_file, 'r') as f:
expect_result = f.readlines()
assert (lines == expect_result)
# Test - output_all_nodes_results
# case 1: 1 accept, 2 not accept
data_df = diag1.output_all_nodes_results(diag1._raw_data_df,
data_not_accept_df)
assert (len(data_df) == 3)
assert (not data_df.loc['sb-validation-01']['Accept'])
assert (data_df.loc['sb-validation-02']['Accept'])
assert (not data_df.loc['sb-validation-03']['Accept'])
assert ('Category' in data_df)
assert ('Issue_Details' in data_df)
# case 1: 3 accept, 0 not accept
data_df_all_accept = diag1.output_all_nodes_results(
diag1._raw_data_df, pd.DataFrame())
assert (len(data_df_all_accept) == 3)
assert (data_df_all_accept.loc['sb-validation-01']['Accept'])
assert (data_df_all_accept.loc['sb-validation-02']['Accept'])
assert (data_df_all_accept.loc['sb-validation-03']['Accept'])
# Test - output in json
diag1.output_diagnosis_in_json(data_df, self.output_all_json_file)
assert (Path(self.output_all_json_file).is_file())
expected_result_file = str(self.parent_path /
'../data/diagnosis_summary.json')
with Path(self.output_all_json_file).open() as f:
data_not_accept_read_from_json = f.read()
with Path(expected_result_file).open() as f:
expect_result = f.read()
assert (data_not_accept_read_from_json == expect_result)