def test_filter_summary_lineage(self):
"""Test the interface of filter_summary_lineage."""
expect_result = {
'customized': event_data.CUSTOMIZED__1,
'object': [
LINEAGE_FILTRATION_EXCEPT_RUN,
LINEAGE_FILTRATION_RUN1,
LINEAGE_FILTRATION_RUN2
],
'count': 3
}
search_condition = {
'sorted_name': 'summary_dir'
}
res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(res.get('object')):
expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
assert expect_result == res
expect_result = {
'customized': {},
'object': [],
'count': 0
}
res = filter_summary_lineage(self.dir_with_empty_lineage)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(res.get('object')):
expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
assert expect_result == res
def test_filter_summary_lineage_exception_6(self):
"""Test the abnormal execution of the filter_summary_lineage interface."""
# gt > lt
search_condition1 = {
'metric/accuracy': {
'gt': 1,
'lt': 0.5
}
}
expect_result = {
'customized': {},
'object': [],
'count': 0
}
partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1)
assert expect_result == partial_res1
# the (offset + 1) * limit > count
search_condition2 = {
'loss': {
'lt': 1
},
'limit': 1,
'offset': 4
}
expect_result = {
'customized': {},
'object': [],
'count': 2
}
partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2)
assert expect_result == partial_res2
def test_filter_summary_lineage(self, validate_path_mock,
convert_path_mock, latest_summary_mock,
qurier_mock, organizer_mock):
"""Test the function of filter_summary_lineage."""
convert_path_mock.return_value = {
'summary_dir': {
'in': ['/path/to/summary_base_dir']
},
'loss': {
'gt': 2.0
}
}
organizer_mock = MagicMock()
organizer_mock.super_lineage_objs = None
validate_path_mock.return_value = True
latest_summary_mock.return_value = [
'/path/to/summary_base_dir/summary_dir'
]
mock_querier = MagicMock()
qurier_mock.return_value = mock_querier
mock_querier.filter_summary_lineage.return_value = [{'loss': 3.0}]
summary_base_dir = '/path/to/summary_base_dir'
result = filter_summary_lineage(summary_base_dir)
self.assertEqual(result, [{'loss': 3.0}])
def test_filter_summary_lineage_with_condition_2(self):
"""Test the interface of filter_summary_lineage with condition."""
search_condition = {
'summary_dir': {
'in': [
'./run1',
'./run2'
]
},
'metric/accuracy': {
'lt': 3.0,
'gt': 0.5
},
'sorted_name': 'metric/accuracy',
'sorted_type': 'descending',
'limit': 3,
'offset': 0
}
expect_result = {
'customized': event_data.CUSTOMIZED__1,
'object': [
LINEAGE_FILTRATION_RUN2,
LINEAGE_FILTRATION_RUN1
],
'count': 2
}
partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(partial_res.get('object')):
expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
assert expect_result == partial_res
def test_filter_summary_lineage_with_condition_3(self):
"""Test the interface of filter_summary_lineage with condition."""
search_condition1 = {
'batch_size': {
'ge': 30
},
'sorted_name': 'metric/accuracy',
}
expect_result = {
'customized': event_data.CUSTOMIZED__1,
'object': [
LINEAGE_FILTRATION_EXCEPT_RUN,
LINEAGE_FILTRATION_RUN1,
LINEAGE_FILTRATION_RUN2
],
'count': 3
}
partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(partial_res1.get('object')):
expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
assert expect_result == partial_res1
search_condition2 = {
'batch_size': {
'lt': 30
},
'lineage_type': {
'eq': 'model'
},
}
expect_result = {
'customized': {},
'object': [],
'count': 0
}
partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(partial_res2.get('object')):
expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark')
assert expect_result == partial_res2
def test_failed_to_querier(self, mock_query, mock_parse, *args):
"""Test filter_summary_lineage with invalid invalid param."""
mock_query.side_effect = LineageSummaryParseException()
mock_parse.return_value = ['/path/to/summary/file']
args[0].return_value = None
res = filter_summary_lineage('/path/to/summary')
assert res == {'object': [], 'count': 0}
mock_query.side_effect = LineageQuerierParamException(['keys'], 'key')
self.assertRaisesRegex(LineageQuerySummaryDataError,
'Filter summary lineage failed.',
filter_summary_lineage, '/path/to/summary/dir')
def test_train_begin_with_user_defined_key_in_lineage(self):
"""Test TrainLineage with nested user defined info."""
expected_res = {"info": "info1", "version": "v1"}
user_defined_info = {
"info": "info1",
"version": "v1",
"network": "LeNet"
}
train_callback = TrainLineage(self.summary_record, False,
user_defined_info)
train_callback.begin(RunContext(self.run_context))
assert train_callback.initial_learning_rate == 0.12
lineage_log_path = train_callback.lineage_summary.lineage_log_path
assert os.path.isfile(lineage_log_path) is True
res = filter_summary_lineage(os.path.dirname(lineage_log_path))
assert expected_res == res['object'][0]['model_lineage'][
'user_defined']
def _get_lineage_info(lineage_type, search_condition):
"""
Get lineage info for dataset or model.
Args:
lineage_type (str): Lineage type, 'dataset' or 'model'.
search_condition (dict): Search condition.
Returns:
dict, lineage info.
Raises:
MindInsightException: If method fails to be called.
"""
if 'lineage_type' in search_condition:
raise ParamValueError(
"Lineage type does not need to be assigned in a specific interface."
)
if lineage_type == 'dataset':
search_condition.update({'lineage_type': 'dataset'})
summary_base_dir = str(settings.SUMMARY_BASE_DIR)
try:
lineage_info = filter_summary_lineage(summary_base_dir,
search_condition)
lineages = lineage_info['object']
summary_base_dir = os.path.realpath(summary_base_dir)
length = len(summary_base_dir)
for lineage in lineages:
summary_dir = lineage['summary_dir']
summary_dir = os.path.realpath(summary_dir)
if summary_base_dir == summary_dir:
relative_dir = './'
else:
relative_dir = os.path.join(os.curdir,
summary_dir[length + 1:])
lineage['summary_dir'] = relative_dir
except MindInsightException as exception:
raise MindInsightException(exception.error,
exception.message,
http_code=400)
return lineage_info
def test_filter_summary_lineage_with_lineage_type(self):
"""Test the interface of filter_summary_lineage with lineage_type."""
summary_dir = os.path.join(BASE_SUMMARY_DIR, 'except_run')
search_condition = {
'summary_dir': {
'in': [summary_dir]
},
'lineage_type': 'dataset'
}
expect_result = {
'object': [
{
'summary_dir': summary_dir,
'dataset_graph': DATASET_GRAPH
}
],
'count': 1
}
res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
assert expect_result == res
def test_filter_summary_lineage_with_condition_3(self):
"""Test the interface of filter_summary_lineage with condition."""
search_condition1 = {
'batch_size': {
'ge': 30
},
'sorted_name': 'metric_accuracy',
'lineage_type': None
}
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 1024,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 10,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 64,
'metric': {},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
],
'count': 2
}
partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(partial_res1.get('object')):
expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark')
assert expect_result == partial_res1
search_condition2 = {
'batch_size': {
'lt': 30
},
'lineage_type': 'model'
}
expect_result = {
'object': [],
'count': 0
}
partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(partial_res2.get('object')):
expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark')
assert expect_result == partial_res2
def test_filter_summary_lineage_with_condition_2(self):
"""Test the interface of filter_summary_lineage with condition."""
search_condition = {
'summary_dir': {
'in': [
'./run1',
'./run2'
]
},
'metric_accuracy': {
'lt': 3.0,
'gt': 0.5
},
'sorted_name': 'metric_accuracy',
'sorted_type': 'descending',
'limit': 3,
'offset': 0
}
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'loss_function': None,
'train_dataset_path': None,
'train_dataset_count': None,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': None,
'optimizer': None,
'learning_rate': None,
'epoch': None,
'batch_size': None,
'loss': None,
'model_size': None,
'metric': {
'accuracy': 2.7800000000000002
},
'dataset_graph': {},
'dataset_mark': 3
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
}
],
'count': 2
}
partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(partial_res.get('object')):
expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark')
assert expect_result == partial_res
def test_filter_summary_lineage(self):
"""Test the interface of filter_summary_lineage."""
expect_result = {
'object': [
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 1024,
'test_dataset_path': None,
'test_dataset_count': None,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 10,
'batch_size': 32,
'loss': 0.029999999329447746,
'model_size': 64,
'metric': {},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'train_dataset_path': None,
'train_dataset_count': 731,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': 'ResNet',
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'epoch': 14,
'batch_size': 32,
'loss': None,
'model_size': 64,
'metric': {
'accuracy': 0.78
},
'dataset_graph': DATASET_GRAPH,
'dataset_mark': 2
},
{
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'loss_function': None,
'train_dataset_path': None,
'train_dataset_count': None,
'test_dataset_path': None,
'test_dataset_count': 10240,
'network': None,
'optimizer': None,
'learning_rate': None,
'epoch': None,
'batch_size': None,
'loss': None,
'model_size': None,
'metric': {
'accuracy': 2.7800000000000002
},
'dataset_graph': {},
'dataset_mark': 3
}
],
'count': 3
}
search_condition = {
'sorted_name': 'summary_dir'
}
res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(res.get('object')):
expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark')
assert expect_result == res
expect_result = {
'object': [],
'count': 0
}
res = filter_summary_lineage(self.dir_with_empty_lineage)
expect_objects = expect_result.get('object')
for idx, res_object in enumerate(res.get('object')):
expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark')
assert expect_result == res
