def validate_and_normalize_profiler_path(summary_dir, summary_base_dir):
"""
Validate and normalize profiler path.
Args:
summary_dir (str): The relative path of summary directory.
summary_base_dir (str): The summary base directory.
Returns:
str, normalized path of profiler directory.
"""
if not summary_dir:
raise ProfilerParamValueErrorException('The file dir does not exist.')
try:
unquote_path = unquote(summary_dir, errors='strict')
except UnicodeDecodeError:
raise ProfilerParamValueErrorException('Unquote error with strict mode')
profiler_dir = os.path.join(summary_base_dir, unquote_path, 'profiler')
try:
profiler_dir = validate_and_normalize_path(profiler_dir, 'profiler')
except ValidationError:
log.error('profiler dir <%s> is invalid', profiler_dir)
raise ProfilerParamValueErrorException('Profiler dir is invalid.')
return profiler_dir
def validate_and_normalize_profiler_path(summary_dir, summary_base_dir):
"""
Validate and normalize profiler path.
Args:
summary_dir (str): The relative path of summary directory.
summary_base_dir (str): The summary base directory.
Returns:
str, normalized path of profiler directory.
"""
profiler_directory_pattern = r'^profiler.*'
if not summary_dir:
raise ProfilerParamValueErrorException('The file dir does not exist.')
try:
unquote_path = unquote(summary_dir, errors='strict')
except UnicodeDecodeError:
raise ProfilerParamValueErrorException(
'Unquote error with strict mode')
train_job_dir = os.path.join(summary_base_dir, unquote_path)
try:
train_job_dir_abs = validate_and_normalize_path(
train_job_dir, 'train_job_dir')
except ValidationError:
log.error('train_job dir <%s> is invalid', train_job_dir)
raise ProfilerParamValueErrorException('train_job dir is invalid.')
if not os.path.exists(train_job_dir_abs):
raise TrainJobNotExistError(error_detail=train_job_dir_abs)
try:
profiler_name_list = []
for dir_name in os.listdir(train_job_dir_abs):
search_res = re.search(profiler_directory_pattern, dir_name)
if search_res:
profiler_name_list.append(search_res[0])
profiler_name_list.sort()
profiler_name_newest = profiler_name_list[-1]
profiler_dir = os.path.join(summary_base_dir, unquote_path,
profiler_name_newest)
except ValidationError:
log.error('no valid profiler dir under <%s>', train_job_dir_abs)
raise ProfilerDirNotFoundException('Profiler dir not found.')
try:
profiler_dir = validate_and_normalize_path(profiler_dir, 'profiler')
except ValidationError:
log.error('profiler dir <%s> is invalid', profiler_dir)
raise ProfilerParamValueErrorException('Profiler dir is invalid.')
return profiler_dir
def _get_proc_details(self, proc_name, step_id=None, time_type='realtime'):
"""
Get step trace info for selected step and save the result.
Args:
proc_name (str): The selected field name.
step_id (int): The selected step_id. If not given, it means all steps is required.
If the value is 0, it means average info for all steps except the first is
required. Default: None.
time_type (str): The value type. `systime` keeps the original value.
`realtime` transforms the value in millisecond. Default: `realtime`.
"""
if proc_name is None:
log.error('`proc_name` is required for query.')
raise ProfilerParamValueErrorException(
'`proc_name` is required for query.')
if step_id is None:
rows_info = self._data[:-1]
else:
rows_info = [self._data[step_id - 1]]
proc_info = [
get_field_value(row_info, proc_name, self.__column__, time_type)
for row_info in rows_info
]
self._result['info'] = {proc_name: proc_info}
def _validate_str_param(proc_name, accept_param, error_name=''):
"""Validate proc_name."""
if proc_name is None or isinstance(proc_name,
str) and proc_name in accept_param:
return
log.error("Invalid param %s in request. Acceptable value is %s.",
error_name, accept_param)
raise ProfilerParamValueErrorException(f"Invalid {error_name}.")
def validate_ui_proc(proc_name):
"""
Validate proc name in restful request.
Args:
proc_name (str): The proc name to query. Acceptable value is in
[`iteration_interval`, `fp_and_bp`, `tail`].
Raises:
ProfilerParamValueErrorException: If the proc_name is invalid.
"""
accept_names = ['iteration_interval', 'fp_and_bp', 'tail']
if proc_name not in accept_names:
log.error("Invalid proc_name. The proc_name for restful api is in %s", accept_names)
raise ProfilerParamValueErrorException(f'proc_name should be in {accept_names}.')
def check_op_name(self, op_name, is_prefix=True):
"""
Check whether the operator name exists.
Args:
op_name (str): The operator name or operator name prefix.
is_prefix (bool): `True` if the op_name is prefix, else `False`.
Default: True.
Returns:
bool, `True` if the operator name does exist in framework file, else
`False`.
"""
if not op_name:
raise ProfilerParamValueErrorException('The op_name should exist.')
for full_op_name in self._task_id_full_op_name_dict.values():
if full_op_name:
if is_prefix and full_op_name.startswith(op_name):
return True
if not is_prefix and op_name == full_op_name:
return True
return False
def _get_threshold(self, options):
"""
Get the threshold of the minddata pipeline queue usage rate.
Args:
options (dict): The options for analysis.
Returns:
list[float], the threshold of the minddata pipeline queue usage rate.
Raises:
ProfilerParamValueErrorException: If the threshold is invalid.
"""
options = get_options(options)
pipeline_options = options.get(self.__proposer_type)
threshold = None
if pipeline_options:
threshold = options.get('threshold')
if threshold is None:
threshold = [0.8, 0.2]
if not isinstance(threshold, list) or len(threshold) != 2:
raise ProfilerParamValueErrorException('The threshold is invalid.')
return threshold