def __parse_and_check_args(cls, name, class_def, parameters):
"""Parse and check the predefine parameters.
If ignore_invalid is True, and 'required' arguments are not set when register the benchmark,
the arguments should be provided by user in config and skip the arguments checking.
Args:
name (str): internal name of benchmark.
class_def (Benchmark): class object of benchmark.
parameters (str): predefined parameters of benchmark.
"""
benchmark = class_def(name, parameters)
benchmark.add_parser_arguments()
ret, args, unknown = benchmark.parse_args(ignore_invalid=True)
if not ret or len(unknown) >= 1:
logger.log_and_raise(
TypeError,
'Registered benchmark has invalid arguments - benchmark: {}, parameters: {}'.format(name, parameters)
)
elif args is not None:
cls.benchmarks[name]['predefine_param'] = vars(args)
logger.debug('Benchmark registration - benchmark: {}, predefine_parameters: {}'.format(name, vars(args)))
else:
cls.benchmarks[name]['predefine_param'] = dict()
logger.info(
'Benchmark registration - benchmark: {}, missing required parameters or invalid parameters, '
'skip the arguments checking.'.format(name)
)
def _check_raw_data_df(raw_data_df):
"""Check whether raw_data_df is empty or None.
Args:
raw_data_df (DataFrame): raw data df
"""
if raw_data_df is None or raw_data_df.empty:
logger.log_and_raise(exception=Exception,
msg='empty data in summary op')
def check_criterion_with_a_value(rule):
"""Check if the criterion is valid with a numeric variable and return bool type.
Args:
rule (dict): rule including function, criteria, metrics with their baseline values and categories
"""
# parse criteria and check if valid
if not isinstance(eval(rule['criteria'])(0), bool):
logger.log_and_raise(exception=Exception,
msg='invalid criteria format')
def percentile(raw_data_df, val):
"""Pencentile$(val) of raw_data_df.
Args:
raw_data_df (DataFrame): raw data df
val (numbers.Number): the pencentile value, 1-99
Returns:
Series: mean of raw_data_df
"""
SummaryOp._check_raw_data_df(raw_data_df)
if not isinstance(val, numbers.Number) or val < 1 or val > 99:
logger.log_and_raise(exception=Exception,
msg='val in pencentile should be 1-99')
return raw_data_df.quantile(val / 100)
def variance(data_row, rule, summary_data_row, details, categories):
"""Rule op function of variance.
Each metric in the rule will calculate the variance (val - baseline / baseline),
and use criteria in the rule to determine whether metric's variance meet the criteria,
if any metric meet the criteria, the rule is not passed.
Args:
data_row (pd.Series): raw data of the metrics
rule (dict): rule including function, criteria, metrics with their baseline values and categories
summary_data_row (pd.Series): results of the metrics processed after the function
details (list): details about violated rules and related data
categories (set): categories of violated rules
Returns:
number: the number of the metrics that violate the rule if the rule is not passed, otherwise 0
"""
violated_metric_num = 0
RuleOp.check_criterion_with_a_value(rule)
# every metric should pass the rule
for metric in rule['metrics']:
# metric not in raw_data or the value is none, miss test
if not RuleOp.miss_test(metric, rule, data_row, details,
categories):
violate_metric = False
# check if metric pass the rule
val = data_row[metric]
baseline = rule['metrics'][metric]
if baseline == 0:
logger.log_and_raise(
exception=Exception,
msg='invalid baseline 0 in variance rule')
var = (val - baseline) / baseline
summary_data_row[metric] = var
violate_metric = eval(rule['criteria'])(var)
# add issued details and categories
if violate_metric:
violated_metric_num += 1
info = '(B/L: {:.4f} VAL: {:.4f} VAR: {:.2f}% Rule:{})'.format(
baseline, val, var * 100, rule['criteria'])
if 'store' not in rule or not rule['store']:
RuleOp.add_categories_and_details(
metric + info, rule['categories'], details,
categories)
else:
RuleOp.add_categories_and_details(
metric + info, None, details, categories)
return violated_metric_num
def _check_and_format_rules(self, rule, name):
"""Check the rule of the metric whether the format is valid.
Args:
rule (dict): the rule
name (str): the rule name
Returns:
dict: the rule for the metric
"""
# check if rule is supported
if 'categories' not in rule:
logger.log_and_raise(exception=Exception, msg='{} lack of category'.format(name))
if 'metrics' in rule:
if isinstance(rule['metrics'], str):
rule['metrics'] = [rule['metrics']]
return rule
def _check_rules(self, rule, name):
"""Check whether the formart of the rule is valid.
Args:
rule (dict): the rule
name (str): the rule name
Returns:
dict: the rule for the metric
"""
# check if rule is supported
super()._check_and_format_rules(rule, name)
if 'metrics' not in rule:
logger.log_and_raise(exception=Exception,
msg='{} lack of metrics'.format(name))
if 'statistics' not in rule:
logger.log_and_raise(exception=Exception,
msg='{} lack of function'.format(name))
# convert single statistic str to list
if not isinstance(rule['statistics'], list):
rule['statistics'] = [rule['statistics']]
# check statistics format, should be SummaryType or p\d\d?
for function in rule['statistics']:
try:
if not (re.fullmatch(r'p\d\d?', function)
or isinstance(SummaryType(function), SummaryType)):
logger.log_and_raise(
exception=Exception,
msg='{} has invalid statistics name {}'.format(
name, function))
except Exception:
logger.log_and_raise(
exception=Exception,
msg='{} has invalid statistics name {}'.format(
name, function))
# check aggregate format, should be None or bool or pattern in regex with () group
if 'aggregate' in rule and not isinstance(
rule['aggregate'], bool) and not re.search(
r'\(.*\)', rule['aggregate']):
logger.log_and_raise(
exception=Exception,
msg='{} aggregate must be bool type'.format(name))
return rule
def multi_rules(rule, details, categories, violation):
"""Rule op function of multi_rules.
The criteria in this rule will use the combined results of multiple previous rules and their metrics
which has been stored in advance to determine whether this rule is passed.
Args:
rule (dict): rule including function, criteria, metrics with their baseline values and categories
details (list): details about violated rules and related data
categories (set): categories of violated rules
violation (dict): the number of the metrics that violate the rules
Returns:
number: 0 if the rule is passed, otherwise 1
"""
violated = eval(rule['criteria'])(violation)
if not isinstance(violated, bool):
logger.log_and_raise(exception=Exception,
msg='invalid upper criteria format')
if violated:
info = '{}:{}'.format(rule['name'], rule['criteria'])
RuleOp.add_categories_and_details(info, rule['categories'],
details, categories)
return 1 if violated else 0
def register_benchmark(cls, name, class_def, parameters='', platform=None):
"""Register new benchmark, key is the benchmark name.
Args:
name (str): internal name of benchmark.
class_def (Benchmark): class object of benchmark.
parameters (str): predefined parameters of benchmark.
platform (Platform): Platform types like CUDA, ROCM.
"""
if not name or not isinstance(name, str):
logger.log_and_raise(
TypeError,
'Name of registered benchmark is not string - benchmark: {}, type: {}'.format(name, type(name))
)
if not issubclass(class_def, Benchmark):
logger.log_and_raise(
TypeError,
'Registered class is not subclass of Benchmark - benchmark: {}, type: {}'.format(name, type(class_def))
)
if name not in cls.benchmarks:
cls.benchmarks[name] = dict()
if platform:
if platform not in Platform:
platform_list = list(map(str, Platform))
logger.log_and_raise(
TypeError, 'Unknown platform - benchmark: {}, supportted platforms: {}, but got: {}'.format(
name, platform_list, platform
)
)
if platform in cls.benchmarks[name]:
logger.warning('Duplicate registration - benchmark: {}, platform: {}'.format(name, platform))
cls.benchmarks[name][platform] = (class_def, parameters)
else:
# If not specified the tag, means the benchmark works for all platforms.
for p in Platform:
if p in cls.benchmarks[name]:
logger.warning('Duplicate registration - benchmark: {}, platform: {}'.format(name, p))
cls.benchmarks[name][p] = (class_def, parameters)
cls.__parse_and_check_args(name, class_def, parameters)
def _check_and_format_rules(self, rule, name):
"""Check the rule of the metric whether the formart is valid.
Args:
rule (dict): the rule
name (str): the rule name
Returns:
dict: the rule for the metric
"""
# check if rule is supported
super()._check_and_format_rules(rule, name)
if 'function' not in rule:
logger.log_and_raise(exception=Exception, msg='{} lack of function'.format(name))
if not isinstance(DiagnosisRuleType(rule['function']), DiagnosisRuleType):
logger.log_and_raise(exception=Exception, msg='{} invalid function name'.format(name))
# check rule format
if 'criteria' not in rule:
logger.log_and_raise(exception=Exception, msg='{} lack of criteria'.format(name))
if not isinstance(eval(rule['criteria']), Callable):
logger.log_and_raise(exception=Exception, msg='invalid criteria format')
if rule['function'] != 'multi_rules':
if 'metrics' not in rule:
logger.log_and_raise(exception=Exception, msg='{} lack of metrics'.format(name))
if 'store' in rule and not isinstance(rule['store'], bool):
logger.log_and_raise(exception=Exception, msg='{} store must be bool type'.format(name))
return rule