def validate_path(summary_path):
"""
Verify the summary path is valid or not.
Args:
summary_path (str): The summary path which is a dir.
Raises:
LineageParamValueError: If the input param value is invalid.
LineageDirNotExistError: If the summary path is invalid.
"""
try:
summary_path = safe_normalize_path(summary_path,
"summary_path",
None,
check_absolute_path=True)
except ValidationError:
log.error("The summary path is invalid.")
raise LineageParamValueError("The summary path is invalid.")
if not os.path.isdir(summary_path):
log.error("The summary path does not exist or is not a dir.")
raise LineageDirNotExistError(
"The summary path does not exist or is not a dir.")
return summary_path
def _organize_from_disk(self):
"""Organize lineage objs from disk."""
if self._summary_base_dir is None:
return
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=self._summary_base_dir)
no_lineage_count = 0
for item in relative_dirs:
relative_dir = item.get('relative_path')
update_time = item.get('update_time')
abs_summary_dir = os.path.realpath(
os.path.join(self._summary_base_dir, relative_dir))
try:
lineage_parser = LineageParser(abs_summary_dir, update_time)
super_lineage_obj = lineage_parser.super_lineage_obj
if super_lineage_obj is not None:
self._super_lineage_objs.update(
{abs_summary_dir: super_lineage_obj})
except LineageFileNotFoundError:
no_lineage_count += 1
if no_lineage_count == len(relative_dirs):
logger.error(
'There is no summary log file under summary_base_dir.')
raise LineageFileNotFoundError(
'There is no summary log file under summary_base_dir.')
def general_get_summary_lineage(data_manager=None,
summary_dir=None,
keys=None):
"""
Get summary lineage from data_manager or parsing from summaries.
One of data_manager or summary_dir needs to be specified. Support getting
super_lineage_obj from data_manager or parsing summaries by summary_dir.
Args:
data_manager (DataManager): Data manager defined as
mindinsight.datavisual.data_transform.data_manager.DataManager
summary_dir (str): The summary directory. It contains summary logs for
one training.
keys (list[str]): The filter keys of lineage information. The acceptable
keys are `metric`, `user_defined`, `hyper_parameters`, `algorithm`,
`train_dataset`, `model`, `valid_dataset` and `dataset_graph`.
If it is `None`, all information will be returned. Default: None.
Returns:
dict, the lineage information for one training.
Raises:
LineageParamSummaryPathError: If summary path is invalid.
LineageQuerySummaryDataError: If querying summary data fails.
LineageFileNotFoundError: If the summary log file is not found.
"""
default_result = {}
if data_manager is None and summary_dir is None:
raise LineageParamTypeError(
"One of data_manager or summary_dir needs to be specified.")
if data_manager is not None and summary_dir is None:
raise LineageParamTypeError(
"If data_manager is specified, the summary_dir needs to be "
"specified as relative path.")
if keys is not None:
validate_filter_key(keys)
if data_manager is None:
normalize_summary_dir(summary_dir)
super_lineage_obj = LineageParser(summary_dir).super_lineage_obj
else:
validate_train_id(summary_dir)
super_lineage_obj = LineageOrganizer(
data_manager=data_manager).get_super_lineage_obj(summary_dir)
if super_lineage_obj is None:
return default_result
try:
result = Querier({
summary_dir: super_lineage_obj
}).get_summary_lineage(summary_dir, keys)
except (LineageQuerierParamException, LineageParamTypeError) as error:
log.error(str(error))
log.exception(error)
raise LineageQuerySummaryDataError("Get summary lineage failed.")
return result[0]
def make_directory(path):
"""Make directory."""
if path is None or not isinstance(path, str) or not path.strip():
log.error("Invalid input path: %r.", path)
raise LineageParamTypeError("Invalid path type")
# convert relative path to abs path
path = os.path.realpath(path)
log.debug("The abs path is %r", path)
# check path exist and its write permissions]
if os.path.exists(path):
real_path = path
else:
# All exceptions need to be caught because create directory maybe have some limit(permissions)
log.debug("The directory(%s) doesn't exist, will create it", path)
try:
os.makedirs(path, exist_ok=True)
real_path = path
except PermissionError as err:
log.error("No write permission on the directory(%r), error = %r",
path, err)
raise LineageParamTypeError(
"No write permission on the directory.")
return real_path
def __init__(self,
summary_record,
raise_exception=False,
user_defined_info=None):
super(TrainLineage, self).__init__()
try:
validate_raise_exception(raise_exception)
self.raise_exception = raise_exception
if isinstance(summary_record, str):
# make directory if not exist
self.lineage_log_dir = make_directory(summary_record)
else:
summary_log_path = summary_record.full_file_name
validate_file_path(summary_log_path)
self.lineage_log_dir = os.path.dirname(summary_log_path)
self.lineage_summary = LineageSummary(self.lineage_log_dir)
self.initial_learning_rate = None
self.user_defined_info = user_defined_info
if user_defined_info:
validate_user_defined_info(user_defined_info)
except MindInsightException as err:
log.error(err)
if raise_exception:
raise
def validate_int_params(int_param, param_name):
"""
Verify the parameter which type is integer valid or not.
Args:
int_param (int): parameter that is integer,
including epoch, dataset_batch_size, step_num
param_name (str): the name of parameter,
including epoch, dataset_batch_size, step_num
Raises:
MindInsightException: If the parameters are invalid.
"""
if not isinstance(int_param,
int) or int_param <= 0 or int_param > pow(2, 63) - 1:
if param_name == 'step_num':
log.error(
'Invalid step_num. The step number should be a positive integer.'
)
raise MindInsightException(
error=LineageErrors.PARAM_STEP_NUM_ERROR,
message=LineageErrorMsg.PARAM_STEP_NUM_ERROR.value)
if param_name == 'dataset_batch_size':
log.error('Invalid dataset_batch_size. '
'The batch size should be a positive integer.')
raise MindInsightException(
error=LineageErrors.PARAM_BATCH_SIZE_ERROR,
message=LineageErrorMsg.PARAM_BATCH_SIZE_ERROR.value)
def _package_parameter(key, value, message):
"""
Package parameters in operation.
Args:
key (str): Operation name.
value (Union[str, bool, int, float, list, None]): Operation args.
message (OperationParameter): Operation proto message.
"""
if isinstance(value, str):
message.mapStr[key] = value
elif isinstance(value, bool):
message.mapBool[key] = value
elif isinstance(value, int):
message.mapInt[key] = value
elif isinstance(value, float):
message.mapDouble[key] = value
elif isinstance(value, list) and key != "operations":
if value:
replace_value_list = list(
map(lambda x: "" if x is None else x, value))
message.mapStrList[key].strValue.extend(replace_value_list)
elif value is None:
message.mapStr[key] = "None"
else:
error_msg = "Parameter {} is not supported " \
"in event package.".format(key)
log.error(error_msg)
raise LineageParamTypeError(error_msg)
def _package_user_defined_info(user_defined_dict, user_defined_message):
"""
Setting attribute in user defined proto message.
Args:
user_defined_dict (dict): User define info dict.
user_defined_message (LineageEvent): Proto message of user defined info.
Raises:
LineageParamValueError: When the value is out of range.
LineageParamTypeError: When given a type not support yet.
"""
for key, value in user_defined_dict.items():
if not isinstance(key, str):
error_msg = f"Invalid key type in user defined info. The {key}'s type" \
f"'{type(key).__name__}' is not supported. It should be str."
log.error(error_msg)
if isinstance(value, int):
attr_name = "map_int32"
elif isinstance(value, float):
attr_name = "map_double"
elif isinstance(value, str):
attr_name = "map_str"
else:
attr_name = "attr_name"
add_user_defined_info = user_defined_message.user_info.add()
try:
getattr(add_user_defined_info, attr_name)[key] = value
except AttributeError:
error_msg = f"Invalid value type in user defined info. The {value}'s type" \
f"'{type(value).__name__}' is not supported. It should be float, int or str."
log.error(error_msg)
def normalize_summary_dir(summary_dir):
"""Normalize summary dir."""
try:
summary_dir = validate_path(summary_dir)
except (LineageParamValueError, LineageDirNotExistError) as error:
log.error(str(error))
log.exception(error)
raise LineageParamSummaryPathError(str(error.message))
return summary_dir
def _check_crc(source_str, crc_str):
"""
Check the integrity of source string.
Args:
source_str (bytes): Source string in bytes.
crc_str (bytes): CRC string of source string in bytes.
Raises:
LineageVerificationException: Raise when verification failed.
"""
if not crc32.CheckValueAgainstData(crc_str, source_str, len(source_str)):
log.error("The CRC verification failed.")
raise LineageVerificationException("The CRC verification failed.")
def end(self, run_context):
"""
Collect lineage information when the training job ends.
Args:
run_context (RunContext): It contains all lineage information,
see mindspore.train.callback.RunContext.
Raises:
MindInsightException: If validating parameter fails.
LineageLogError: If recording lineage information fails.
"""
if self.user_defined_info:
self.lineage_summary.record_user_defined_info(
self.user_defined_info)
if not isinstance(run_context, RunContext):
error_msg = f'Invalid EvalLineage run_context.'
log.error(error_msg)
raise LineageParamRunContextError(error_msg)
run_context_args = run_context.original_args()
validate_eval_run_context(EvalParameter, run_context_args)
valid_dataset = run_context_args.get('valid_dataset')
eval_lineage = dict()
metrics = run_context_args.get('metrics')
eval_lineage[Metadata.metrics] = json.dumps(metrics)
eval_lineage[Metadata.step_num] = run_context_args.get('cur_step_num')
log.info('Analyzing dataset object...')
eval_lineage = AnalyzeObject.analyze_dataset(valid_dataset,
eval_lineage, 'valid')
log.info('Logging evaluation job lineage...')
try:
self.lineage_summary.record_evaluation_lineage(eval_lineage)
except IOError as error:
error_msg = f'End error in EvalLineage: {error}'
log.error(error_msg)
log.error('Fail to log the lineage of the evaluation job.')
raise LineageLogError(error_msg)
except Exception as error:
error_msg = f'End error in EvalLineage: {error}'
log.error(error_msg)
log.error('Fail to log the lineage of the evaluation job.')
raise LineageLogError(error_msg)
log.info('The lineage of the evaluation job has logged successfully.')
def _check_crc(source_str, crc_str):
"""
Check the integrity of source string.
Args:
source_str (bytes): Source string in bytes.
crc_str (bytes): CRC string of source string in bytes.
Raises:
LineageVerificationException: Raise when verification failed.
"""
if crc32.GetValueFromStr(crc_str) != \
crc32.GetMaskCrc32cValue(source_str, len(source_str)):
log.error("The CRC verification failed.")
raise LineageVerificationException("The CRC verification failed.")
def __init__(self, summary_record, raise_exception=False):
super(EvalLineage, self).__init__()
try:
validate_raise_exception(raise_exception)
self.raise_exception = raise_exception
validate_summary_record(summary_record)
self.summary_record = summary_record
summary_log_path = summary_record.full_file_name
validate_file_path(summary_log_path)
self.lineage_log_path = summary_log_path + '_lineage'
except MindInsightException as err:
log.error(err)
if raise_exception:
raise
def get_file_size(file_path):
"""
Get the file size.
Args:
file_path (str): The file path.
Returns:
int, the file size.
"""
try:
return os.path.getsize(file_path)
except (OSError, IOError) as error:
error_msg = f"Error when get model file size: {error}"
log.error(error_msg)
raise LineageGetModelFileError(error_msg)
def validate_raise_exception(raise_exception):
"""
Validate raise_exception.
Args:
raise_exception (bool): decide raise exception or not,
if True, raise exception; else, catch exception and continue.
Raises:
MindInsightException: If the parameters are invalid.
"""
if not isinstance(raise_exception, bool):
log.error("Invalid raise_exception. It should be True or False.")
raise MindInsightException(
error=LineageErrors.PARAM_RAISE_EXCEPTION_ERROR,
message=LineageErrorMsg.PARAM_RAISE_EXCEPTION_ERROR.value)
def validate_search_model_condition(schema, data):
"""
Validate search model condition.
Args:
schema (Schema): Data schema.
data (dict): Data to check schema.
Raises:
MindInsightException: If the parameters are invalid.
"""
error = schema().validate(data)
for error_key in error.keys():
if error_key in SEARCH_MODEL_ERROR_MAPPING.keys():
error_code = SEARCH_MODEL_ERROR_MAPPING.get(error_key)
error_msg = SEARCH_MODEL_ERROR_MSG_MAPPING.get(error_key)
log.error(error_msg)
raise MindInsightException(error=error_code, message=error_msg)
def validate_summary_record(summary_record):
"""
Validate summary_record.
Args:
summary_record (SummaryRecord): SummaryRecord is used to record
the summary value, and summary_record is an instance of SummaryRecord,
see mindspore.train.summary.SummaryRecord
Raises:
MindInsightException: If the parameters are invalid.
"""
if not isinstance(summary_record, SummaryRecord):
log.error("Invalid summary_record. It should be an instance "
"of mindspore.train.summary.SummaryRecord.")
raise MindInsightException(
error=LineageErrors.PARAM_SUMMARY_RECORD_ERROR,
message=LineageErrorMsg.PARAM_SUMMARY_RECORD_ERROR.value)
def validate_eval_run_context(schema, data):
"""
Validate mindspore evaluation job run_context data according to schema.
Args:
schema (Schema): data schema.
data (dict): data to check schema.
Raises:
MindInsightException: If the parameters are invalid.
"""
errors = schema().validate(data)
for error_key, error_msg in errors.items():
if error_key in EVAL_RUN_CONTEXT_ERROR_MAPPING.keys():
error_code = EVAL_RUN_CONTEXT_ERROR_MAPPING.get(error_key)
if EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key):
error_msg = EVAL_RUN_CONTEXT_ERROR_MSG_MAPPING.get(error_key)
log.error(error_msg)
raise MindInsightException(error=error_code, message=error_msg)
def validate_file_path(file_path, allow_empty=False):
"""
Verify that the file_path is valid.
Args:
file_path (str): Input file path.
allow_empty (bool): Whether file_path can be empty.
Raises:
MindInsightException: If the parameters are invalid.
"""
try:
if allow_empty and not file_path:
return file_path
return safe_normalize_path(file_path, raise_key='dataset_path', safe_prefixes=None)
except ValidationError as error:
log.error(str(error))
raise MindInsightException(error=LineageErrors.PARAM_FILE_PATH_ERROR,
message=str(error))
def filter_summary_lineage(data_manager=None, summary_base_dir=None, search_condition=None):
"""
Filter summary lineage from data_manager or parsing from summaries.
One of data_manager or summary_base_dir needs to be specified. Support getting
super_lineage_obj from data_manager or parsing summaries by summary_base_dir.
Args:
data_manager (DataManager): Data manager defined as
mindinsight.datavisual.data_transform.data_manager.DataManager
summary_base_dir (str): The summary base directory. It contains summary
directories generated by training.
search_condition (dict): The search condition.
"""
if data_manager is None and summary_base_dir is None:
raise LineageParamTypeError("One of data_manager or summary_base_dir needs to be specified.")
if data_manager is None:
summary_base_dir = validate_and_normalize_path(summary_base_dir, 'summary_base_dir')
else:
summary_base_dir = data_manager.summary_base_dir
search_condition = {} if search_condition is None else search_condition
try:
validate_condition(search_condition)
validate_search_model_condition(SearchModelConditionParameter, search_condition)
except MindInsightException as error:
log.error(str(error))
log.exception(error)
raise LineageSearchConditionParamError(str(error.message))
try:
lineage_objects = LineageOrganizer(data_manager, summary_base_dir).super_lineage_objs
result = Querier(lineage_objects).filter_summary_lineage(condition=search_condition)
except LineageSummaryParseException:
result = {'object': [], 'count': 0}
except (LineageQuerierParamException, LineageParamTypeError) as error:
log.error(str(error))
log.exception(error)
raise LineageQuerySummaryDataError("Filter summary lineage failed.")
return result
def begin(self, run_context):
"""
Initialize the training progress when the training job begins.
Args:
run_context (RunContext): It contains all lineage information,
see mindspore.train.callback.RunContext.
Raises:
MindInsightException: If validating parameter fails.
"""
log.info('Initialize training lineage collection...')
if self.user_defined_info:
self.lineage_summary.record_user_defined_info(
self.user_defined_info)
if not isinstance(run_context, RunContext):
error_msg = f'Invalid TrainLineage run_context.'
log.error(error_msg)
raise LineageParamRunContextError(error_msg)
run_context_args = run_context.original_args()
if not self.initial_learning_rate:
optimizer = run_context_args.get('optimizer')
if optimizer and not isinstance(optimizer, Optimizer):
log.error(
"The parameter optimizer is invalid. It should be an instance of "
"mindspore.nn.optim.optimizer.Optimizer.")
raise MindInsightException(
error=LineageErrors.PARAM_OPTIMIZER_ERROR,
message=LineageErrorMsg.PARAM_OPTIMIZER_ERROR.value)
if optimizer:
log.info('Obtaining initial learning rate...')
self.initial_learning_rate = AnalyzeObject.analyze_optimizer(
optimizer)
log.debug('initial_learning_rate: %s',
self.initial_learning_rate)
else:
network = run_context_args.get('train_network')
optimizer = AnalyzeObject.get_optimizer_by_network(network)
self.initial_learning_rate = AnalyzeObject.analyze_optimizer(
optimizer)
log.debug('initial_learning_rate: %s',
self.initial_learning_rate)
# get train dataset graph
train_dataset = run_context_args.get('train_dataset')
dataset_graph_dict = ds.serialize(train_dataset)
dataset_graph_json_str = json.dumps(dataset_graph_dict, indent=2)
dataset_graph_dict = json.loads(dataset_graph_json_str)
log.info('Logging dataset graph...')
try:
self.lineage_summary.record_dataset_graph(
dataset_graph=dataset_graph_dict)
except Exception as error:
error_msg = f'Dataset graph log error in TrainLineage begin: {error}'
log.error(error_msg)
raise LineageLogError(error_msg)
log.info('Dataset graph logged successfully.')
def validate_filter_key(keys):
"""
Verify the keys of filtering is valid or not.
Args:
keys (list): The keys to get the relative lineage info.
Raises:
LineageParamTypeError: If keys is not list.
LineageParamValueError: If the value of keys is invalid.
"""
filter_keys = [
'metric', 'hyper_parameters', 'algorithm', 'train_dataset', 'model',
'valid_dataset', 'dataset_graph'
]
if not isinstance(keys, list):
log.error("Keys must be list.")
raise LineageParamTypeError("Keys must be list.")
for element in keys:
if not isinstance(element, str):
log.error("Element of keys must be str.")
raise LineageParamTypeError("Element of keys must be str.")
if not set(keys).issubset(filter_keys):
err_msg = "Keys must be in {}.".format(filter_keys)
log.error(err_msg)
raise LineageParamValueError(err_msg)
def validate_network(network):
"""
Verify if the network is valid.
Args:
network (Cell): See mindspore.nn.Cell.
Raises:
LineageParamMissingError: If the network is None.
MindInsightException: If the network is invalid.
"""
if not network:
error_msg = "The input network for TrainLineage should not be None."
log.error(error_msg)
raise LineageParamMissingError(error_msg)
if not isinstance(network, Cell):
log.error("Invalid network. Network should be an instance"
"of mindspore.nn.Cell.")
raise MindInsightException(
error=LineageErrors.PARAM_TRAIN_NETWORK_ERROR,
message=LineageErrorMsg.PARAM_TRAIN_NETWORK_ERROR.value)
def get_summary_infos(cls, file_path):
"""
Get lineage summary information from summary log file.
Args:
file_path (str): The file path of summary log.
Returns:
LineageInfo, the lineage summary information.
Raises:
LineageSummaryAnalyzeException: If failed to get lineage information.
"""
analyzer = cls(file_path)
try:
lineage_info = analyzer.get_latest_info()
except (MindInsightException, IOError) as err:
log.error("Failed to get lineage information.")
log.exception(err)
raise LineageSummaryAnalyzeException()
return lineage_info
def load(self):
"""Find and load summaries."""
# get sorted lineage files
lineage_files = SummaryPathParser.get_lineage_summaries(
self._summary_dir, is_sorted=True)
if not lineage_files:
logger.info('There is no summary log file under summary_dir %s.',
self._summary_dir)
raise LineageFileNotFoundError(
'There is no summary log file under summary_dir.')
self._init_if_files_deleted(lineage_files)
index = 0
if self._latest_filename is not None:
index = lineage_files.index(self._latest_filename)
for filename in lineage_files[index:]:
if filename != self._latest_filename:
self._latest_filename = filename
self._latest_file_size = 0
file_path = os.path.join(self._summary_dir, filename)
new_size = FileHandler(file_path).size
if new_size == self._latest_file_size:
continue
self._latest_file_size = new_size
try:
self._parse_summary_log()
except (LineageSummaryAnalyzeException,
LineageEventNotExistException,
LineageEventFieldNotExistException) as error:
logger.error("Parse file failed, file_path is %s. Detail: %s",
file_path, str(error))
except MindInsightException as error:
logger.exception(error)
logger.error("Parse file failed, file_path is %s.", file_path)
def get_summary_lineage(summary_dir, keys=None):
"""
Get the lineage information according to summary directory and keys.
The function queries lineage information of single train process
corresponding to the given summary directory. Users can query the
information according to `keys`.
Args:
summary_dir (str): The summary directory. It contains summary logs for
one training.
keys (list[str]): The filter keys of lineage information. The acceptable
keys are `metric`, `hyper_parameters`, `algorithm`, `train_dataset`,
`model`, `valid_dataset` and `dataset_graph`. If it is `None`, all
information will be returned. Default: None.
Returns:
dict, the lineage information for one training.
Raises:
LineageParamSummaryPathError: If summary path is invalid.
LineageQuerySummaryDataError: If querying summary data fails.
LineageFileNotFoundError: If the summary log file is not found.
Examples:
>>> summary_dir = "/path/to/summary"
>>> summary_lineage_info = get_summary_lineage(summary_dir)
>>> hyper_parameters = get_summary_lineage(summary_dir, keys=["hyper_parameters"])
"""
try:
summary_dir = validate_path(summary_dir)
except MindInsightException as error:
log.error(str(error))
log.exception(error)
raise LineageParamSummaryPathError(str(error.message))
if keys is not None:
validate_filter_key(keys)
summary_path = SummaryPathParser.get_latest_lineage_summary(summary_dir)
if summary_path is None:
log.error('There is no summary log file under summary_dir.')
raise LineageFileNotFoundError(
'There is no summary log file under summary_dir.')
try:
result = Querier(summary_path).get_summary_lineage(summary_dir,
filter_keys=keys)
except LineageSummaryParseException:
return {}
except (LineageQuerierParamException, LineageParamTypeError) as error:
log.error(str(error))
log.exception(error)
raise LineageQuerySummaryDataError("Get summary lineage failed.")
return result[0]
def validate_user_defined_info(user_defined_info):
"""
Validate user defined info， delete the item if its key is in lineage.
Args:
user_defined_info (dict): The user defined info.
Raises:
LineageParamTypeError: If the type of parameters is invalid.
LineageParamValueError: If user defined keys have been defined in lineage.
"""
if not isinstance(user_defined_info, dict):
log.error("Invalid user defined info. It should be a dict.")
raise LineageParamTypeError(
"Invalid user defined info. It should be dict.")
for key, value in user_defined_info.items():
if not isinstance(key, str):
error_msg = "Dict key type {} is not supported in user defined info." \
"Only str is permitted now.".format(type(key))
log.error(error_msg)
raise LineageParamTypeError(error_msg)
if not isinstance(value, (int, str, float)):
error_msg = "Dict value type {} is not supported in user defined info." \
"Only str, int and float are permitted now.".format(type(value))
log.error(error_msg)
raise LineageParamTypeError(error_msg)
field_map = set(FIELD_MAPPING.keys())
user_defined_keys = set(user_defined_info.keys())
insertion = list(field_map & user_defined_keys)
if insertion:
for key in insertion:
user_defined_info.pop(key)
raise LineageParamValueError(
"There are some keys have defined in lineage. "
"Duplicated key(s): %s. " % insertion)
def filter_summary_lineage(summary_base_dir, search_condition=None):
"""
Filter the lineage information under summary base directory according to search condition.
Users can filter and sort all lineage information according to the search
condition. The supported filter fields include `summary_dir`, `network`,
etc. The filter conditions include `eq`, `lt`, `gt`, `le`, `ge` and `in`.
At the same time, the combined use of these fields and conditions is
supported. If you want to sort based on filter fields, the field of
`sorted_name` and `sorted_type` should be specified.
Users can use `lineage_type` to decide what kind of lineage information to
query. If the `lineage_type` is `dataset`, the query result is only the
lineage information related to data augmentation. If the `lineage_type` is
`model` or `None`, the query result is all lineage information.
Users can paginate query result based on `offset` and `limit`. The `offset`
refers to page number. The `limit` refers to the number in one page.
Args:
summary_base_dir (str): The summary base directory. It contains summary
directories generated by training.
search_condition (dict): The search condition. When filtering and
sorting, in addition to the following supported fields, fields
prefixed with `metric_` are also supported. The fields prefixed with
`metric_` are related to the `metrics` parameter in the training
script. For example, if the key of `metrics` parameter is
`accuracy`, the field should be `metric_accuracy`. Default: None.
- summary_dir (dict): The filter condition of summary directory.
- loss_function (dict): The filter condition of loss function.
- train_dataset_path (dict): The filter condition of train dataset path.
- train_dataset_count (dict): The filter condition of train dataset count.
- test_dataset_path (dict): The filter condition of test dataset path.
- test_dataset_count (dict): The filter condition of test dataset count.
- network (dict): The filter condition of network.
- optimizer (dict): The filter condition of optimizer.
- learning_rate (dict): The filter condition of learning rate.
- epoch (dict): The filter condition of epoch.
- batch_size (dict): The filter condition of batch size.
- loss (dict): The filter condition of loss.
- model_size (dict): The filter condition of model size.
- dataset_mark (dict): The filter condition of dataset mark.
- offset (int): Page number, the value range is [0, 100000].
- limit (int): The number in one page, the value range is [1, 100].
- sorted_name (str): Specify which field to sort by.
- sorted_type (str): Specify sort order. It can be `ascending` or
`descending`.
- lineage_type (str): It decides what kind of lineage information to
query. It can be `dataset` or `model`. If it is `dataset`,
the query result is only the lineage information related to data
augmentation. If it is `model` or `None`, the query result is all
lineage information.
Returns:
dict, all lineage information under summary base directory according to
search condition.
Raises:
LineageSearchConditionParamError: If search_condition param is invalid.
LineageParamSummaryPathError: If summary path is invalid.
LineageFileNotFoundError: If the summary log file is not found.
LineageQuerySummaryDataError: If querying summary log file data fails.
Examples:
>>> summary_base_dir = "/path/to/summary_base"
>>> search_condition = {
>>> 'summary_dir': {
>>> 'in': [
>>> os.path.join(summary_base_dir, 'summary_1'),
>>> os.path.join(summary_base_dir, 'summary_2'),
>>> os.path.join(summary_base_dir, 'summary_3')
>>> ]
>>> },
>>> 'loss': {
>>> 'gt': 2.0
>>> },
>>> 'batch_size': {
>>> 'ge': 128,
>>> 'le': 256
>>> },
>>> 'metric_accuracy': {
>>> 'lt': 0.1
>>> },
>>> 'sorted_name': 'summary_dir',
>>> 'sorted_type': 'descending',
>>> 'limit': 3,
>>> 'offset': 0,
>>> 'lineage_type': 'model'
>>> }
>>> summary_lineage = filter_summary_lineage(summary_base_dir)
>>> summary_lineage_filter = filter_summary_lineage(summary_base_dir, search_condition)
"""
try:
summary_base_dir = validate_path(summary_base_dir)
except (LineageParamValueError, LineageDirNotExistError) as error:
log.error(str(error))
log.exception(error)
raise LineageParamSummaryPathError(str(error.message))
search_condition = {} if search_condition is None else search_condition
try:
validate_condition(search_condition)
validate_search_model_condition(SearchModelConditionParameter,
search_condition)
except MindInsightException as error:
log.error(str(error))
log.exception(error)
raise LineageSearchConditionParamError(str(error.message))
try:
search_condition = _convert_relative_path_to_abspath(
summary_base_dir, search_condition)
except (LineageParamValueError, LineageDirNotExistError) as error:
log.error(str(error))
log.exception(error)
raise LineageParamSummaryPathError(str(error.message))
summary_path = SummaryPathParser.get_latest_lineage_summaries(
summary_base_dir)
if not summary_path:
log.error('There is no summary log file under summary_base_dir.')
raise LineageFileNotFoundError(
'There is no summary log file under summary_base_dir.')
try:
result = Querier(summary_path).filter_summary_lineage(
condition=search_condition)
except LineageSummaryParseException:
result = {'object': [], 'count': 0}
except (LineageQuerierParamException, LineageParamTypeError) as error:
log.error(str(error))
log.exception(error)
raise LineageQuerySummaryDataError("Filter summary lineage failed.")
return result
LineageErrors
from mindinsight.lineagemgr.common.exceptions.exceptions import \
LineageParamTypeError, LineageParamValueError
from mindinsight.lineagemgr.common.log import logger
from mindinsight.lineagemgr.common.utils import enum_to_list
from mindinsight.lineagemgr.querier.querier import LineageType
from mindinsight.lineagemgr.querier.query_model import FIELD_MAPPING
from mindinsight.utils.exceptions import MindInsightException
try:
from mindspore.dataset.engine import Dataset
from mindspore.nn import Cell, Optimizer
from mindspore.common.tensor import Tensor
from mindspore.train.callback import _ListCallback
except (ImportError, ModuleNotFoundError):
logger.error('MindSpore Not Found!')
class RunContextArgs(Schema):
"""Define the parameter schema for RunContext."""
optimizer = fields.Function(allow_none=True)
loss_fn = fields.Function(allow_none=True)
net_outputs = fields.Function(allow_none=True)
train_network = fields.Function(allow_none=True)
train_dataset = fields.Function(allow_none=True)
epoch_num = fields.Int(allow_none=True, validate=Range(min=1))
batch_num = fields.Int(allow_none=True, validate=Range(min=0))
cur_step_num = fields.Int(allow_none=True, validate=Range(min=0))
parallel_mode = fields.Str(allow_none=True)
device_number = fields.Int(allow_none=True, validate=Range(min=1))
list_callback = fields.Function(allow_none=True)
def validate_condition(search_condition):
"""
Verify the param in search_condition is valid or not.
Args:
search_condition (dict): The search condition.
Raises:
LineageParamTypeError: If the type of the param in search_condition is invalid.
LineageParamValueError: If the value of the param in search_condition is invalid.
"""
if not isinstance(search_condition, dict):
log.error("Invalid search_condition type, it should be dict.")
raise LineageParamTypeError("Invalid search_condition type, "
"it should be dict.")
if "limit" in search_condition:
if isinstance(search_condition.get("limit"), bool) \
or not isinstance(search_condition.get("limit"), int):
log.error("The limit must be int.")
raise LineageParamTypeError("The limit must be int.")
if "offset" in search_condition:
if isinstance(search_condition.get("offset"), bool) \
or not isinstance(search_condition.get("offset"), int):
log.error("The offset must be int.")
raise LineageParamTypeError("The offset must be int.")
if "sorted_name" in search_condition:
sorted_name = search_condition.get("sorted_name")
err_msg = "The sorted_name must be in {} or start with " \
"`metric/` or `user_defined/`.".format(list(FIELD_MAPPING.keys()))
if not isinstance(sorted_name, str):
log.error(err_msg)
raise LineageParamValueError(err_msg)
if not (sorted_name in FIELD_MAPPING or
(sorted_name.startswith('metric/')
and len(sorted_name) > len('metric/')) or
(sorted_name.startswith('user_defined/')
and len(sorted_name) > len('user_defined/'))
or sorted_name in ['tag']):
log.error(err_msg)
raise LineageParamValueError(err_msg)
sorted_type_param = ['ascending', 'descending', None]
if "sorted_type" in search_condition:
if "sorted_name" not in search_condition:
log.error("The sorted_name have to exist when sorted_type exists.")
raise LineageParamValueError(
"The sorted_name have to exist when sorted_type exists.")
if search_condition.get("sorted_type") not in sorted_type_param:
err_msg = "The sorted_type must be ascending or descending."
log.error(err_msg)
raise LineageParamValueError(err_msg)
