def test_get_project_structure(self, datatype_group_factory, dummy_datatype_index_factory, project_factory, user_factory): """ Tests project structure is as expected and contains all datatypes """ SELF_DTS_NUMBER = 3 user = user_factory() project = project_factory(user) dt_group = datatype_group_factory(project=project) link_ids, expected_links = [], [] # Prepare link towards a simple DT dt_to_link = dummy_datatype_index_factory(state="RAW_DATA") link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree link_gr = dt_group dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id) link_ids.extend([dt_to_link.id for dt_to_link in dts]) link_ids.append(link_gr.id) expected_links.append(link_gr.gid) # Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree link_gr = dt_group dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0] link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Actually create the links from Prj2 into Prj1 FlowService().create_link(link_ids, project.id) # Retrieve the raw data used to compose the tree (for easy parsing) dts_in_tree = dao.get_data_in_project(project.id) dts_in_tree = [dt.gid for dt in dts_in_tree] # Retrieve the tree json (for trivial validations only, as we can not decode) node_json = self.project_service.get_project_structure( project, None, DataTypeMetaData.KEY_STATE, DataTypeMetaData.KEY_SUBJECT, None) assert len(expected_links) + SELF_DTS_NUMBER + 2 == len( dts_in_tree), "invalid number of nodes in tree" assert not link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected." assert dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!" assert dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!" project_dts = dao.get_datatypes_in_project(project.id) for dt in project_dts: if dt.fk_datatype_group is not None: assert not dt.gid in node_json, "DTs part of a group should not be" assert not dt.gid in dts_in_tree, "DTs part of a group should not be" else: assert dt.gid in node_json, "Simple DTs and DT_Groups should be" assert dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be" for link_gid in expected_links: assert link_gid in node_json, "Expected Link not present" assert link_gid in dts_in_tree, "Expected Link not present"
def test_get_project_structure(self): """ Tests project structure is as expected and contains all datatypes """ SELF_DTS_NUMBER = 3 dt_factory_1 = datatypes_factory.DatatypesFactory() self._create_datatypes(dt_factory_1, SELF_DTS_NUMBER) dt_group = dt_factory_1.create_datatype_group() link_ids, expected_links = [], [] # Prepare link towards a simple DT dt_factory_2 = datatypes_factory.DatatypesFactory() dt_to_link = dt_factory_2.create_simple_datatype() link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree link_gr = dt_factory_2.create_datatype_group() dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id) link_ids.extend([dt_to_link.id for dt_to_link in dts]) link_ids.append(link_gr.id) expected_links.append(link_gr.gid) # Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree link_gr = dt_factory_2.create_datatype_group() dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0] link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Actually create the links from Prj2 into Prj1 FlowService().create_link(link_ids, dt_factory_1.project.id) # Retrieve the raw data used to compose the tree (for easy parsing) dts_in_tree = dao.get_data_in_project(dt_factory_1.project.id) dts_in_tree = [dt.gid for dt in dts_in_tree] # Retrieve the tree json (for trivial validations only, as we can not decode) node_json = self.project_service.get_project_structure(dt_factory_1.project, None, DataTypeMetaData.KEY_STATE, DataTypeMetaData.KEY_SUBJECT, None) self.assertEqual(len(expected_links) + SELF_DTS_NUMBER + 2, len(dts_in_tree), "invalid number of nodes in tree") self.assertFalse(link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected.") self.assertTrue(dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!") self.assertTrue(dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!") project_dts = dao.get_datatypes_in_project(dt_factory_1.project.id) for dt in project_dts: if dt.fk_datatype_group is not None: self.assertFalse(dt.gid in node_json, "DTs part of a group should not be") self.assertFalse(dt.gid in dts_in_tree, "DTs part of a group should not be") else: self.assertTrue(dt.gid in node_json, "Simple DTs and DT_Groups should be") self.assertTrue(dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be") for link_gid in expected_links: self.assertTrue(link_gid in node_json, "Expected Link not present") self.assertTrue(link_gid in dts_in_tree, "Expected Link not present")
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" adapter_instance = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 1, "Expected one operation group" assert all_operations[0][2] == 2, "Expected 2 operations in group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapter3") group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: "param_5", "param_5": [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def update_metadata(self, submit_data): """ Update DataType/ DataTypeGroup metadata THROW StructureException when input data is invalid. """ new_data = dict() for key in DataTypeOverlayDetails().meta_attributes_list: if key in submit_data: new_data[key] = submit_data[key] if new_data[CommonDetails.CODE_OPERATION_TAG] == '': new_data[CommonDetails.CODE_OPERATION_TAG] = None try: if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''): # We need to edit a group all_data_in_group = dao.get_datatype_in_group(operation_group_id= new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if len(all_data_in_group) < 1: raise StructureException("Inconsistent group, can not be updated!") datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0] all_data_in_group.append(datatype_group) for datatype in all_data_in_group: new_data[CommonDetails.CODE_GID] = datatype.gid self._edit_data(datatype, new_data, True) else: # Get the required DataType and operation from DB to store changes that will be done in XML. gid = new_data[CommonDetails.CODE_GID] datatype = dao.get_datatype_by_gid(gid) self._edit_data(datatype, new_data) except Exception as excep: self.logger.exception(excep) raise StructureException(str(excep))
def create_group(test_user=None, test_project=None, subject="John Doe"): """ Create a group of 2 operations, each with at least one resultant DataType. """ if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) adapter_inst = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') adapter_inst.generic_attributes.subject = subject view_model = adapter_inst.get_view_model()() args = {RANGE_PARAMETER_1: 'param_5', 'param_5': json.dumps({constants.ATT_MINVALUE: 1, constants.ATT_MAXVALUE: 2.1, constants.ATT_STEP: 1})} algo = adapter_inst.stored_adapter algo_category = dao.get_category_by_id(algo.fk_category) # Prepare Operations group. Execute them synchronously service = OperationService() operations = service.prepare_operations(test_user.id, test_project, algo, algo_category, view_model=view_model, **args)[0] service.launch_operation(operations[0].id, False, adapter_inst) service.launch_operation(operations[1].id, False, adapter_inst) resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group) return resulted_dts, operations[0].fk_operation_group
def update_metadata(self, submit_data): """ Update DataType/ DataTypeGroup metadata THROW StructureException when input data is invalid. """ new_data = dict() for key in DataTypeOverlayDetails().meta_attributes_list: if key in submit_data: new_data[key] = submit_data[key] if new_data[CommonDetails.CODE_OPERATION_TAG] == '': new_data[CommonDetails.CODE_OPERATION_TAG] = None try: if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''): # We need to edit a group all_data_in_group = dao.get_datatype_in_group(operation_group_id= new_data[CommonDetails.CODE_OPERATION_GROUP_ID]) if len(all_data_in_group) < 1: raise StructureException("Inconsistent group, can not be updated!") datatype_group = dao.get_generic_entity(model.DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0] all_data_in_group.append(datatype_group) for datatype in all_data_in_group: new_data[CommonDetails.CODE_GID] = datatype.gid self._edit_data(datatype, new_data, True) else: # Get the required DataType and operation from DB to store changes that will be done in XML. gid = new_data[CommonDetails.CODE_GID] datatype = dao.get_datatype_by_gid(gid) self._edit_data(datatype, new_data) except Exception, excep: self.logger.exception(excep) raise StructureException(str(excep))
def create_group(test_user=None, test_project=None, subject="John Doe"): """ Create a group of 2 operations, each with at least one resultant DataType. """ if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) ### Retrieve Adapter instance algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') algo_category = dao.get_category_by_id(algo_group.fk_category) algo = dao.get_algorithm_by_group(algo_group.id) adapter_inst = TestFactory.create_adapter(algo_group=algo_group, test_project=test_project) adapter_inst.meta_data = {DataTypeMetaData.KEY_SUBJECT: subject} args = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ### Prepare Operations group. Execute them synchronously service = OperationService() operations = service.prepare_operations(test_user.id, test_project.id, algo, algo_category, {}, **args)[0] service.launch_operation(operations[0].id, False, adapter_inst) service.launch_operation(operations[1].id, False, adapter_inst) resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group) return resulted_dts, operations[0].fk_operation_group
def test_datatypes_groups(self, test_adapter_factory): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ # TODO: re-write this to use groups correctly all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" algo = test_adapter_factory(TestAdapter3) adapter_instance = ABCAdapter.build_adapter(algo) data = {model_burst.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations FlowService().fire_operation(adapter_instance, self.test_user, self.test_project.id) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 1, "Expected one operation group" assert all_operations[0][2] == 2, "Expected 2 operations in group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def create_group(test_user=None, test_project=None, subject="John Doe"): """ Create a group of 2 operations, each with at least one resultant DataType. """ if test_user is None: test_user = TestFactory.create_user() if test_project is None: test_project = TestFactory.create_project(test_user) ### Retrieve Adapter instance algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') algo_category = dao.get_category_by_id(algo_group.fk_category) algo = dao.get_algorithm_by_group(algo_group.id) adapter_inst = TestFactory.create_adapter(algo_group=algo_group, test_project=test_project) adapter_inst.meta_data = {DataTypeMetaData.KEY_SUBJECT: subject} args = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ### Prepare Operations group. Execute them synchronously service = OperationService() operations = service.prepare_operations(test_user.id, test_project.id, algo, algo_category, {}, **args)[0] service.launch_operation(operations[0].id, False, adapter_inst) service.launch_operation(operations[1].id, False, adapter_inst) resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group) return resulted_dts, operations[0].fk_operation_group
def test_get_project_structure(self, datatype_group_factory, dummy_datatype_index_factory, project_factory, user_factory): """ Tests project structure is as expected and contains all datatypes and created links """ user = user_factory() project1 = project_factory(user, name="TestPS1") project2 = project_factory(user, name="TestPS2") dt_group = datatype_group_factory(project=project1) dt_simple = dummy_datatype_index_factory(state="RAW_DATA", project=project1) # Create 3 DTs directly in Project 2 dummy_datatype_index_factory(state="RAW_DATA", project=project2) dummy_datatype_index_factory(state="RAW_DATA", project=project2) dummy_datatype_index_factory(state="RAW_DATA", project=project2) # Create Links from Project 1 into Project 2 link_ids, expected_links = [], [] link_ids.append(dt_simple.id) expected_links.append(dt_simple.gid) # Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree dts = dao.get_datatype_in_group(datatype_group_id=dt_group.id) link_ids.extend([dt_to_link.id for dt_to_link in dts]) link_ids.append(dt_group.id) expected_links.append(dt_group.gid) # Actually create the links from Prj1 into Prj2 AlgorithmService().create_link(link_ids, project2.id) # Retrieve the raw data used to compose the tree (for easy parsing) dts_in_tree = dao.get_data_in_project(project2.id) dts_in_tree = [dt.gid for dt in dts_in_tree] # Retrieve the tree json (for trivial validations only, as we can not decode) node_json = self.project_service.get_project_structure( project2, None, DataTypeMetaData.KEY_STATE, DataTypeMetaData.KEY_SUBJECT, None) assert len(expected_links) + 3 == len( dts_in_tree), "invalid number of nodes in tree" assert dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!" assert dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!" project_dts = dao.get_datatypes_in_project(project2.id) for dt in project_dts: if dt.fk_datatype_group is not None: assert not dt.gid in node_json, "DTs part of a group should not be" assert not dt.gid in dts_in_tree, "DTs part of a group should not be" else: assert dt.gid in node_json, "Simple DTs and DT_Groups should be" assert dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be" for link_gid in expected_links: assert link_gid in node_json, "Expected Link not present" assert link_gid in dts_in_tree, "Expected Link not present"
def test_datatypes_groups(self, test_adapter_factory, datatype_group_factory): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" dt_group = datatype_group_factory(project=self.test_project) model = TestModel() test_adapter_factory() adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter1", "TestAdapter1") operations = dao.get_operations_in_group(dt_group.id) for op in operations: model.gid = uuid.uuid4() op_path = StorageInterface().get_project_folder( self.test_project.name, str(op.id)) op.view_model_gid = model.gid.hex op.algorithm = adapter.stored_adapter h5.store_view_model(model, op_path) dao.store_entity(op) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 2, "Expected two operation groups" assert all_operations[0][2] == 6, "Expected 6 operations in one group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[1][0]) self.operation_service.stop_operation(all_operations[1][1]) # Make sure operations are executed self.operation_service.launch_operation(all_operations[1][0], False) self.operation_service.launch_operation(all_operations[1][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) assert len( resulted_datatypes) >= 2, "Expected at least 2, but: " + str( len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def _remove_operation_group(self, operation_group_id, project_id, skip_validation, operations_set, links): metrics_groups = dao.get_generic_entity(DataTypeGroup, operation_group_id, 'fk_operation_group') if len(metrics_groups) > 0: metric_datatype_group_id = metrics_groups[0].id self._remove_datatype_group_dts(project_id, metric_datatype_group_id, skip_validation, operations_set, links) datatypes = dao.get_datatype_in_group(metric_datatype_group_id) # If this condition is false, then there is a link for the datatype group and we don't need to delete it if len(datatypes) == 0: dao.remove_entity(DataTypeGroup, metric_datatype_group_id) return dao.remove_entity(OperationGroup, operation_group_id) else: return True
def test_update_meta_data_group(self, test_adapter_factory, datatype_group_factory): """ Test the new update metaData for a group of dataTypes. """ test_adapter_factory(adapter_class=DummyAdapter3) group, _ = datatype_group_factory() op_group_id = group.fk_operation_group new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject", DataTypeOverlayDetails.DATA_STATE: "updated_state", DataTypeOverlayDetails.CODE_OPERATION_GROUP_ID: op_group_id, DataTypeOverlayDetails.CODE_OPERATION_TAG: 'newGroupName'} self.project_service.update_metadata(new_meta_data) datatypes = dao.get_datatype_in_group(op_group_id) for datatype in datatypes: new_datatype = dao.get_datatype_by_id(datatype.id) assert op_group_id == new_datatype.parent_operation.fk_operation_group new_group = dao.get_generic_entity(model_operation.OperationGroup, op_group_id)[0] assert new_group.name == "newGroupName" self.__check_meta_data(new_meta_data, new_datatype)
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" adapter_instance = TestFactory.create_adapter( 'tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 1, "Expected one operation group" assert all_operations[0][2] == 2, "Expected 2 operations in group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) assert len( resulted_datatypes) >= 2, "Expected at least 2, but: " + str( len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same DTgroup. """ return dao.get_datatype_in_group(datatype_group_id=group)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False, existing_dt_links=None): """ Method used for removing a dataType. If the given dataType is a DatatypeGroup or a dataType from a DataTypeGroup than this method will remove the entire group. The operation(s) used for creating the dataType(s) will also be removed. """ datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is None: self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid) return if datatype.parent_operation.fk_launched_in != int(project_id): self.logger.warning("Datatype with GUID [%s] has been moved to another project and does " "not need to be deleted anymore." % datatype_gid) return is_datatype_group = False datatype_group = None new_dt_links = [] # Datatype Groups were already handled when the first DatatypeMeasureIndex has been found if dao.is_datatype_group(datatype_gid): is_datatype_group = True datatype_group = datatype # Found the first DatatypeMeasureIndex from a group elif datatype.fk_datatype_group is not None: is_datatype_group = True # We load it this way to make sure we have the 'fk_operation_group' in every case datatype_group_gid = dao.get_datatype_by_id(datatype.fk_datatype_group).gid datatype_group = h5.load_entity_by_gid(datatype_group_gid) operations_set = [datatype.fk_from_operation] correct = True if is_datatype_group: operations_set = [datatype_group.fk_from_operation] self.logger.debug("Removing datatype group %s" % datatype_group) datatypes = self.get_all_datatypes_from_data(datatype_group) first_datatype = datatypes[0] if hasattr(first_datatype, 'fk_source_gid'): ts = h5.load_entity_by_gid(first_datatype.fk_source_gid) ts_group = dao.get_datatypegroup_by_op_group_id(ts.parent_operation.fk_operation_group) dm_group = datatype_group else: dt_measure_index = get_class_by_name("{}.{}".format(DATATYPE_MEASURE_INDEX_MODULE, DATATYPE_MEASURE_INDEX_CLASS)) dm_group = dao.get_datatype_measure_group_from_ts_from_pse(first_datatype.gid, dt_measure_index) ts_group = datatype_group links = [] if ts_group: links.extend(dao.get_links_for_datatype(ts_group.id)) correct = correct and self._remove_operation_group(ts_group.fk_operation_group, project_id, skip_validation, operations_set, links) if dm_group: links.extend(dao.get_links_for_datatype(dm_group.id)) correct = correct and self._remove_operation_group(dm_group.fk_operation_group, project_id, skip_validation, operations_set, links) if len(links) > 0: # We want to get the links for the first TSIndex directly # This code works for all cases datatypes = dao.get_datatype_in_group(ts_group.id) ts = datatypes[0] new_dt_links = self._add_links_for_datatype_references(ts, links[0].fk_to_project, links[0].id, existing_dt_links) else: self.logger.debug("Removing datatype %s" % datatype) links = dao.get_links_for_datatype(datatype.id) if len(links) > 0: new_dt_links = self._add_links_for_datatype_references(datatype, links[0].fk_to_project, links[0].id, existing_dt_links) self._remove_project_node_files(project_id, datatype.gid, links, skip_validation) # Remove Operation entity in case no other DataType needs them. project = dao.get_project_by_id(project_id) for operation_id in operations_set: dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation") if len(dependent_dt) > 0: # Do not remove Operation in case DataType still exist referring it. continue correct = correct and dao.remove_entity(Operation, operation_id) # Make sure Operation folder is removed self.storage_interface.remove_operation_data(project.name, operation_id) self.storage_interface.push_folder_to_sync(project.name) if not correct: raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid)) return new_dt_links
def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same DTgroup. """ return dao.get_datatype_in_group(datatype_group_id=group)
def get_datatype_in_group(group): """ Return all dataTypes that are the result of the same group of operations. """ return dao.get_datatype_in_group(group)
def test_get_project_structure(self): """ Tests project structure is as expected and contains all datatypes """ SELF_DTS_NUMBER = 3 dt_factory_1 = datatypes_factory.DatatypesFactory() self._create_datatypes(dt_factory_1, SELF_DTS_NUMBER) dt_group = dt_factory_1.create_datatype_group() link_ids, expected_links = [], [] # Prepare link towards a simple DT dt_factory_2 = datatypes_factory.DatatypesFactory() dt_to_link = dt_factory_2.create_simple_datatype() link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree link_gr = dt_factory_2.create_datatype_group() dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id) link_ids.extend([dt_to_link.id for dt_to_link in dts]) link_ids.append(link_gr.id) expected_links.append(link_gr.gid) # Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree link_gr = dt_factory_2.create_datatype_group() dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0] link_ids.append(dt_to_link.id) expected_links.append(dt_to_link.gid) # Actually create the links from Prj2 into Prj1 FlowService().create_link(link_ids, dt_factory_1.project.id) # Retrieve the raw data used to compose the tree (for easy parsing) dts_in_tree = dao.get_data_in_project(dt_factory_1.project.id) dts_in_tree = [dt.gid for dt in dts_in_tree] # Retrieve the tree json (for trivial validations only, as we can not decode) node_json = self.project_service.get_project_structure( dt_factory_1.project, None, DataTypeMetaData.KEY_STATE, DataTypeMetaData.KEY_SUBJECT, None) self.assertEqual( len(expected_links) + SELF_DTS_NUMBER + 2, len(dts_in_tree), "invalid number of nodes in tree") self.assertFalse( link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected.") self.assertTrue(dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!") self.assertTrue(dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!") project_dts = dao.get_datatypes_in_project(dt_factory_1.project.id) for dt in project_dts: if dt.fk_datatype_group is not None: self.assertFalse(dt.gid in node_json, "DTs part of a group should not be") self.assertFalse(dt.gid in dts_in_tree, "DTs part of a group should not be") else: self.assertTrue(dt.gid in node_json, "Simple DTs and DT_Groups should be") self.assertTrue(dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be") for link_gid in expected_links: self.assertTrue(link_gid in node_json, "Expected Link not present") self.assertTrue(link_gid in dts_in_tree, "Expected Link not present")