def test_train_multi_eval(self, *args):
"""Callback for train once and eval twice."""
args[0].return_value = 10
summary_dir = os.path.join(BASE_SUMMARY_DIR, 'train_multi_eval')
make_directory(summary_dir)
args[1].return_value = os.path.join(
summary_dir, 'train_out.events.summary.1590107366.ubuntu_lineage')
train_callback = TrainLineage(summary_dir, True)
train_callback.begin(RunContext(self.run_context))
train_callback.end(RunContext(self.run_context))
args[1].return_value = os.path.join(
summary_dir, 'eval_out.events.summary.1590107367.ubuntu_lineage')
eval_callback = EvalLineage(summary_dir, True)
eval_run_context = self.run_context
eval_run_context['valid_dataset'] = self.run_context['train_dataset']
eval_run_context['metrics'] = {'accuracy': 0.79}
eval_callback.end(RunContext(eval_run_context))
res = get_summary_lineage(summary_dir)
assert res.get('metric', {}).get('accuracy') == 0.79
args[1].return_value = os.path.join(
summary_dir, 'eval_out.events.summary.1590107368.ubuntu_lineage')
eval_callback = EvalLineage(summary_dir, True)
eval_run_context = self.run_context
eval_run_context['valid_dataset'] = self.run_context['train_dataset']
eval_run_context['metrics'] = {'accuracy': 0.80}
eval_callback.end(RunContext(eval_run_context))
res = get_summary_lineage(summary_dir)
assert res.get('metric', {}).get('accuracy') == 0.80
if os.path.exists(summary_dir):
shutil.rmtree(summary_dir)
def test_raise_exception_non_lineage_file(self):
"""Test exception when lineage summary file cannot be found."""
summary_dir = os.path.join(BASE_SUMMARY_DIR, 'run4')
if os.path.exists(summary_dir):
shutil.rmtree(summary_dir)
summary_record = SummaryRecord(summary_dir, file_suffix='_MS_lineage')
full_file_name = summary_record.full_file_name
assert full_file_name.endswith('_lineage')
assert os.path.isfile(full_file_name)
with self.assertRaisesRegex(LineageFileNotFoundError,
'no summary log file'):
get_summary_lineage(summary_dir)
def test_training_end(self, *args):
"""Test the end function in TrainLineage."""
args[0].return_value = 64
train_callback = TrainLineage(SUMMARY_DIR, True, self.user_defined_info)
train_callback.initial_learning_rate = 0.12
train_callback.end(RunContext(self.run_context))
res = get_summary_lineage(SUMMARY_DIR)
assert res.get('hyper_parameters', {}).get('epoch') == 10
run_context = self.run_context
run_context['epoch_num'] = 14
train_callback.end(RunContext(run_context))
res = get_summary_lineage(SUMMARY_DIR)
assert res.get('hyper_parameters', {}).get('epoch') == 14
def test_train_with_customized_network(self, *args):
"""Test train with customized network."""
args[0].return_value = 64
train_callback = TrainLineage(self.summary_record, True)
run_context_customized = self.run_context
del run_context_customized['optimizer']
del run_context_customized['net_outputs']
del run_context_customized['loss_fn']
net = WithLossCell(self.net, self.loss_fn)
net_cap = net
net_cap._cells = {'_backbone': self.net, '_loss_fn': self.loss_fn}
net = TrainOneStep(net, self.optimizer)
net._cells = {
'optimizer': self.optimizer,
'network': net_cap,
'backbone': self.net
}
run_context_customized['train_network'] = net
train_callback.begin(RunContext(run_context_customized))
train_callback.end(RunContext(run_context_customized))
res = get_summary_lineage(SUMMARY_DIR)
assert res.get('hyper_parameters', {}).get('loss_function') \
== 'SoftmaxCrossEntropyWithLogits'
assert res.get('algorithm', {}).get('network') == 'ResNet'
assert res.get('hyper_parameters', {}).get('optimizer') == 'Momentum'
def test_train_eval(self, *args):
"""Callback for train once and eval once."""
args[0].return_value = 10
summary_dir = os.path.join(BASE_SUMMARY_DIR, 'train_eval')
make_directory(summary_dir)
args[1].return_value = os.path.join(
summary_dir,
f'train_out.events.summary.{str(int(time.time()))}.ubuntu_lineage')
train_callback = TrainLineage(summary_dir)
train_callback.begin(RunContext(self.run_context))
train_callback.end(RunContext(self.run_context))
args[1].return_value = os.path.join(
summary_dir,
f'eval_out.events.summary.{str(int(time.time())+1)}.ubuntu_lineage'
)
eval_callback = EvalLineage(summary_dir)
eval_run_context = self.run_context
eval_run_context['metrics'] = {'accuracy': 0.78}
eval_run_context['valid_dataset'] = self.run_context['train_dataset']
eval_run_context['step_num'] = 32
eval_callback.end(RunContext(eval_run_context))
res = get_summary_lineage(summary_dir)
assert res.get('hyper_parameters', {}).get('loss_function') \
== 'SoftmaxCrossEntropyWithLogits'
assert res.get('algorithm', {}).get('network') == 'ResNet'
if os.path.exists(summary_dir):
shutil.rmtree(summary_dir)
def test_get_summary_lineage_failed3(self, mock_summary, mock_valid,
mock_parser, mock_file_handler):
"""Test get_summary_lineage failed."""
mock_summary.return_value = ['/path/to/summary/file']
mock_valid.return_value = '/path/to/summary_dir'
mock_parser.return_value = None
mock_file_handler = MagicMock()
mock_file_handler.size = 1
result = get_summary_lineage('/path/to/summary_dir')
assert {} == result
def test_get_summary_lineage(self):
"""Test the interface of get_summary_lineage."""
total_res = get_summary_lineage(SUMMARY_DIR)
partial_res1 = get_summary_lineage(SUMMARY_DIR, ['hyper_parameters'])
partial_res2 = get_summary_lineage(SUMMARY_DIR,
['metric', 'algorithm'])
expect_total_res = LINEAGE_INFO_RUN1
expect_partial_res1 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'hyper_parameters': {
'optimizer': 'Momentum',
'learning_rate': 0.12,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'epoch': 14,
'parallel_mode': 'stand_alone',
'device_num': 2,
'batch_size': 32
}
}
expect_partial_res2 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'metric': {
'accuracy': 0.78
},
'algorithm': {
'network': 'ResNet'
}
}
assert_equal_lineages(expect_total_res, total_res,
self.assertDictEqual)
assert_equal_lineages(expect_partial_res1, partial_res1,
self.assertDictEqual)
assert_equal_lineages(expect_partial_res2, partial_res2,
self.assertDictEqual)
# the lineage summary file is empty
result = get_summary_lineage(self.dir_with_empty_lineage)
assert {} == result
# keys is empty list
expect_result = {'summary_dir': SUMMARY_DIR}
result = get_summary_lineage(SUMMARY_DIR, [])
assert expect_result == result
def test_get_summary_lineage_failed3(self, mock_summary, mock_querier,
mock_valid):
"""Test get_summary_lineage failed."""
mock_summary.return_value = '/path/to/summary/file'
mock_querier.return_value.get_summary_lineage.side_effect = \
LineageSummaryParseException()
mock_valid.return_value = '/path/to/summary_dir'
res = get_summary_lineage('/path/to/summary_dir')
assert res == {}
mock_querier.side_effect = LineageQuerierParamException(['keys'],
'key')
self.assertRaisesRegex(LineageQuerySummaryDataError,
'Get summary lineage failed',
get_summary_lineage, '/path/to/summary_dir')
def test_get_summary_lineage_success(self, isdir_mock, parser_mock,
qurier_mock):
"""Test the function of get_summary_lineage."""
isdir_mock.return_value = True
parser_mock.return_value = MagicMock()
mock_querier = MagicMock()
qurier_mock.return_value = mock_querier
mock_querier.get_summary_lineage.return_value = [{
'algorithm': {
'network': 'ResNet'
}
}]
summary_dir = '/path/to/summary_dir'
result = get_summary_lineage(summary_dir, keys=['algorithm'])
self.assertEqual(result, {'algorithm': {'network': 'ResNet'}})
def get_dataset_graph():
"""
Get dataset graph.
Returns:
str, the dataset graph information.
Raises:
MindInsightException: If method fails to be called.
ParamValueError: If summary_dir is invalid.
Examples:
>>> GET http://xxxx/v1/mindinsight/datasets/dataset_graph?train_id=xxx
"""
summary_base_dir = str(settings.SUMMARY_BASE_DIR)
summary_dir = get_train_id(request)
if summary_dir.startswith('/'):
validate_path(summary_dir)
elif summary_dir.startswith('./'):
summary_dir = os.path.join(summary_base_dir, summary_dir[2:])
summary_dir = validate_path(summary_dir)
else:
raise ParamValueError("Summary dir should be absolute path or "
"relative path that relate to summary base dir.")
try:
dataset_graph = get_summary_lineage(summary_dir=summary_dir,
keys=['dataset_graph'])
except MindInsightException as exception:
raise MindInsightException(exception.error,
exception.message,
http_code=400)
if dataset_graph:
summary_dir_result = dataset_graph.get('summary_dir')
base_dir_len = len(summary_base_dir)
if summary_base_dir == summary_dir_result:
relative_dir = './'
else:
relative_dir = os.path.join(os.curdir,
summary_dir[base_dir_len + 1:])
dataset_graph['summary_dir'] = relative_dir
return jsonify(dataset_graph)
def test_eval_only(self):
"""Test record evaluation event only."""
summary_dir = os.path.join(BASE_SUMMARY_DIR, 'eval_only_dir')
summary_record = SummaryRecord(summary_dir)
eval_run_context = self.run_context
eval_run_context['metrics'] = {'accuracy': 0.58}
eval_run_context['valid_dataset'] = self.run_context['train_dataset']
eval_run_context['step_num'] = 32
eval_only_callback = EvalLineage(summary_record)
eval_only_callback.end(RunContext(eval_run_context))
res = get_summary_lineage(summary_dir, ['metric', 'dataset_graph'])
expect_res = {
'summary_dir': summary_dir,
'dataset_graph': {},
'metric': {
'accuracy': 0.58
}
}
assert res == expect_res
shutil.rmtree(summary_dir)
def test_get_summary_lineage(self):
"""Test the interface of get_summary_lineage."""
total_res = get_summary_lineage(SUMMARY_DIR)
partial_res1 = get_summary_lineage(SUMMARY_DIR, ['hyper_parameters'])
partial_res2 = get_summary_lineage(SUMMARY_DIR, ['metric', 'algorithm'])
expect_total_res = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'metric': {
'accuracy': 0.78
},
'hyper_parameters': {
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'epoch': 14,
'parallel_mode': 'stand_alone',
'device_num': 2,
'batch_size': 32
},
'algorithm': {
'network': 'ResNet'
},
'train_dataset': {
'train_dataset_size': 731
},
'valid_dataset': {
'valid_dataset_size': 10240
},
'model': {
'path': '{"ckpt": "'
+ BASE_SUMMARY_DIR + '/run1/CKPtest_model.ckpt"}',
'size': 64
},
'dataset_graph': DATASET_GRAPH
}
expect_partial_res1 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'hyper_parameters': {
'optimizer': 'Momentum',
'learning_rate': 0.11999999731779099,
'loss_function': 'SoftmaxCrossEntropyWithLogits',
'epoch': 14,
'parallel_mode': 'stand_alone',
'device_num': 2,
'batch_size': 32
}
}
expect_partial_res2 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'),
'metric': {
'accuracy': 0.78
},
'algorithm': {
'network': 'ResNet'
}
}
assert expect_total_res == total_res
assert expect_partial_res1 == partial_res1
assert expect_partial_res2 == partial_res2
# the lineage summary file is empty
result = get_summary_lineage(self.dir_with_empty_lineage)
assert {} == result
# keys is empty list
expect_result = {
'summary_dir': SUMMARY_DIR
}
result = get_summary_lineage(SUMMARY_DIR, [])
assert expect_result == result
