def test_filter_summary_lineage(self): """Test the interface of filter_summary_lineage.""" expect_result = { 'customized': event_data.CUSTOMIZED__1, 'object': [ LINEAGE_FILTRATION_EXCEPT_RUN, LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2 ], 'count': 3 } search_condition = { 'sorted_name': 'summary_dir' } res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(res.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') assert expect_result == res expect_result = { 'customized': {}, 'object': [], 'count': 0 } res = filter_summary_lineage(self.dir_with_empty_lineage) expect_objects = expect_result.get('object') for idx, res_object in enumerate(res.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') assert expect_result == res
def test_filter_summary_lineage_exception_6(self): """Test the abnormal execution of the filter_summary_lineage interface.""" # gt > lt search_condition1 = { 'metric/accuracy': { 'gt': 1, 'lt': 0.5 } } expect_result = { 'customized': {}, 'object': [], 'count': 0 } partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1) assert expect_result == partial_res1 # the (offset + 1) * limit > count search_condition2 = { 'loss': { 'lt': 1 }, 'limit': 1, 'offset': 4 } expect_result = { 'customized': {}, 'object': [], 'count': 2 } partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2) assert expect_result == partial_res2
def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock, latest_summary_mock, qurier_mock, organizer_mock): """Test the function of filter_summary_lineage.""" convert_path_mock.return_value = { 'summary_dir': { 'in': ['/path/to/summary_base_dir'] }, 'loss': { 'gt': 2.0 } } organizer_mock = MagicMock() organizer_mock.super_lineage_objs = None validate_path_mock.return_value = True latest_summary_mock.return_value = [ '/path/to/summary_base_dir/summary_dir' ] mock_querier = MagicMock() qurier_mock.return_value = mock_querier mock_querier.filter_summary_lineage.return_value = [{'loss': 3.0}] summary_base_dir = '/path/to/summary_base_dir' result = filter_summary_lineage(summary_base_dir) self.assertEqual(result, [{'loss': 3.0}])
def test_filter_summary_lineage_with_condition_2(self): """Test the interface of filter_summary_lineage with condition.""" search_condition = { 'summary_dir': { 'in': [ './run1', './run2' ] }, 'metric/accuracy': { 'lt': 3.0, 'gt': 0.5 }, 'sorted_name': 'metric/accuracy', 'sorted_type': 'descending', 'limit': 3, 'offset': 0 } expect_result = { 'customized': event_data.CUSTOMIZED__1, 'object': [ LINEAGE_FILTRATION_RUN2, LINEAGE_FILTRATION_RUN1 ], 'count': 2 } partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') assert expect_result == partial_res
def test_filter_summary_lineage_with_condition_3(self): """Test the interface of filter_summary_lineage with condition.""" search_condition1 = { 'batch_size': { 'ge': 30 }, 'sorted_name': 'metric/accuracy', } expect_result = { 'customized': event_data.CUSTOMIZED__1, 'object': [ LINEAGE_FILTRATION_EXCEPT_RUN, LINEAGE_FILTRATION_RUN1, LINEAGE_FILTRATION_RUN2 ], 'count': 3 } partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res1.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') assert expect_result == partial_res1 search_condition2 = { 'batch_size': { 'lt': 30 }, 'lineage_type': { 'eq': 'model' }, } expect_result = { 'customized': {}, 'object': [], 'count': 0 } partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res2.get('object')): expect_objects[idx]['model_lineage']['dataset_mark'] = res_object['model_lineage'].get('dataset_mark') assert expect_result == partial_res2
def test_failed_to_querier(self, mock_query, mock_parse, *args): """Test filter_summary_lineage with invalid invalid param.""" mock_query.side_effect = LineageSummaryParseException() mock_parse.return_value = ['/path/to/summary/file'] args[0].return_value = None res = filter_summary_lineage('/path/to/summary') assert res == {'object': [], 'count': 0} mock_query.side_effect = LineageQuerierParamException(['keys'], 'key') self.assertRaisesRegex(LineageQuerySummaryDataError, 'Filter summary lineage failed.', filter_summary_lineage, '/path/to/summary/dir')
def test_train_begin_with_user_defined_key_in_lineage(self): """Test TrainLineage with nested user defined info.""" expected_res = {"info": "info1", "version": "v1"} user_defined_info = { "info": "info1", "version": "v1", "network": "LeNet" } train_callback = TrainLineage(self.summary_record, False, user_defined_info) train_callback.begin(RunContext(self.run_context)) assert train_callback.initial_learning_rate == 0.12 lineage_log_path = train_callback.lineage_summary.lineage_log_path assert os.path.isfile(lineage_log_path) is True res = filter_summary_lineage(os.path.dirname(lineage_log_path)) assert expected_res == res['object'][0]['model_lineage'][ 'user_defined']
def _get_lineage_info(lineage_type, search_condition): """ Get lineage info for dataset or model. Args: lineage_type (str): Lineage type, 'dataset' or 'model'. search_condition (dict): Search condition. Returns: dict, lineage info. Raises: MindInsightException: If method fails to be called. """ if 'lineage_type' in search_condition: raise ParamValueError( "Lineage type does not need to be assigned in a specific interface." ) if lineage_type == 'dataset': search_condition.update({'lineage_type': 'dataset'}) summary_base_dir = str(settings.SUMMARY_BASE_DIR) try: lineage_info = filter_summary_lineage(summary_base_dir, search_condition) lineages = lineage_info['object'] summary_base_dir = os.path.realpath(summary_base_dir) length = len(summary_base_dir) for lineage in lineages: summary_dir = lineage['summary_dir'] summary_dir = os.path.realpath(summary_dir) if summary_base_dir == summary_dir: relative_dir = './' else: relative_dir = os.path.join(os.curdir, summary_dir[length + 1:]) lineage['summary_dir'] = relative_dir except MindInsightException as exception: raise MindInsightException(exception.error, exception.message, http_code=400) return lineage_info
def test_filter_summary_lineage_with_lineage_type(self): """Test the interface of filter_summary_lineage with lineage_type.""" summary_dir = os.path.join(BASE_SUMMARY_DIR, 'except_run') search_condition = { 'summary_dir': { 'in': [summary_dir] }, 'lineage_type': 'dataset' } expect_result = { 'object': [ { 'summary_dir': summary_dir, 'dataset_graph': DATASET_GRAPH } ], 'count': 1 } res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) assert expect_result == res
def test_filter_summary_lineage_with_condition_3(self): """Test the interface of filter_summary_lineage with condition.""" search_condition1 = { 'batch_size': { 'ge': 30 }, 'sorted_name': 'metric_accuracy', 'lineage_type': None } expect_result = { 'object': [ { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'), 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 1024, 'test_dataset_path': None, 'test_dataset_count': None, 'network': 'ResNet', 'optimizer': 'Momentum', 'learning_rate': 0.11999999731779099, 'epoch': 10, 'batch_size': 32, 'loss': 0.029999999329447746, 'model_size': 64, 'metric': {}, 'dataset_graph': DATASET_GRAPH, 'dataset_mark': 2 }, { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 731, 'test_dataset_path': None, 'test_dataset_count': 10240, 'network': 'ResNet', 'optimizer': 'Momentum', 'learning_rate': 0.11999999731779099, 'epoch': 14, 'batch_size': 32, 'loss': None, 'model_size': 64, 'metric': { 'accuracy': 0.78 }, 'dataset_graph': DATASET_GRAPH, 'dataset_mark': 2 } ], 'count': 2 } partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res1.get('object')): expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') assert expect_result == partial_res1 search_condition2 = { 'batch_size': { 'lt': 30 }, 'lineage_type': 'model' } expect_result = { 'object': [], 'count': 0 } partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res2.get('object')): expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') assert expect_result == partial_res2
def test_filter_summary_lineage_with_condition_2(self): """Test the interface of filter_summary_lineage with condition.""" search_condition = { 'summary_dir': { 'in': [ './run1', './run2' ] }, 'metric_accuracy': { 'lt': 3.0, 'gt': 0.5 }, 'sorted_name': 'metric_accuracy', 'sorted_type': 'descending', 'limit': 3, 'offset': 0 } expect_result = { 'object': [ { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), 'loss_function': None, 'train_dataset_path': None, 'train_dataset_count': None, 'test_dataset_path': None, 'test_dataset_count': 10240, 'network': None, 'optimizer': None, 'learning_rate': None, 'epoch': None, 'batch_size': None, 'loss': None, 'model_size': None, 'metric': { 'accuracy': 2.7800000000000002 }, 'dataset_graph': {}, 'dataset_mark': 3 }, { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 731, 'test_dataset_path': None, 'test_dataset_count': 10240, 'network': 'ResNet', 'optimizer': 'Momentum', 'learning_rate': 0.11999999731779099, 'epoch': 14, 'batch_size': 32, 'loss': None, 'model_size': 64, 'metric': { 'accuracy': 0.78 }, 'dataset_graph': DATASET_GRAPH, 'dataset_mark': 2 } ], 'count': 2 } partial_res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(partial_res.get('object')): expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') assert expect_result == partial_res
def test_filter_summary_lineage(self): """Test the interface of filter_summary_lineage.""" expect_result = { 'object': [ { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'except_run'), 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 1024, 'test_dataset_path': None, 'test_dataset_count': None, 'network': 'ResNet', 'optimizer': 'Momentum', 'learning_rate': 0.11999999731779099, 'epoch': 10, 'batch_size': 32, 'loss': 0.029999999329447746, 'model_size': 64, 'metric': {}, 'dataset_graph': DATASET_GRAPH, 'dataset_mark': 2 }, { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 731, 'test_dataset_path': None, 'test_dataset_count': 10240, 'network': 'ResNet', 'optimizer': 'Momentum', 'learning_rate': 0.11999999731779099, 'epoch': 14, 'batch_size': 32, 'loss': None, 'model_size': 64, 'metric': { 'accuracy': 0.78 }, 'dataset_graph': DATASET_GRAPH, 'dataset_mark': 2 }, { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), 'loss_function': None, 'train_dataset_path': None, 'train_dataset_count': None, 'test_dataset_path': None, 'test_dataset_count': 10240, 'network': None, 'optimizer': None, 'learning_rate': None, 'epoch': None, 'batch_size': None, 'loss': None, 'model_size': None, 'metric': { 'accuracy': 2.7800000000000002 }, 'dataset_graph': {}, 'dataset_mark': 3 } ], 'count': 3 } search_condition = { 'sorted_name': 'summary_dir' } res = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition) expect_objects = expect_result.get('object') for idx, res_object in enumerate(res.get('object')): expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') assert expect_result == res expect_result = { 'object': [], 'count': 0 } res = filter_summary_lineage(self.dir_with_empty_lineage) expect_objects = expect_result.get('object') for idx, res_object in enumerate(res.get('object')): expect_objects[idx]['dataset_mark'] = res_object.get('dataset_mark') assert expect_result == res