def test_get_project_structure(self, datatype_group_factory,
dummy_datatype_index_factory,
project_factory, user_factory):
"""
Tests project structure is as expected and contains all datatypes
"""
SELF_DTS_NUMBER = 3
user = user_factory()
project = project_factory(user)
dt_group = datatype_group_factory(project=project)
link_ids, expected_links = [], []
# Prepare link towards a simple DT
dt_to_link = dummy_datatype_index_factory(state="RAW_DATA")
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
link_gr = dt_group
dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(link_gr.id)
expected_links.append(link_gr.gid)
# Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree
link_gr = dt_group
dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0]
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Actually create the links from Prj2 into Prj1
FlowService().create_link(link_ids, project.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(project.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(
project, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
assert len(expected_links) + SELF_DTS_NUMBER + 2 == len(
dts_in_tree), "invalid number of nodes in tree"
assert not link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected."
assert dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!"
assert dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!"
project_dts = dao.get_datatypes_in_project(project.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
assert not dt.gid in node_json, "DTs part of a group should not be"
assert not dt.gid in dts_in_tree, "DTs part of a group should not be"
else:
assert dt.gid in node_json, "Simple DTs and DT_Groups should be"
assert dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be"
for link_gid in expected_links:
assert link_gid in node_json, "Expected Link not present"
assert link_gid in dts_in_tree, "Expected Link not present"
def test_get_project_structure(self):
"""
Tests project structure is as expected and contains all datatypes
"""
SELF_DTS_NUMBER = 3
dt_factory_1 = datatypes_factory.DatatypesFactory()
self._create_datatypes(dt_factory_1, SELF_DTS_NUMBER)
dt_group = dt_factory_1.create_datatype_group()
link_ids, expected_links = [], []
# Prepare link towards a simple DT
dt_factory_2 = datatypes_factory.DatatypesFactory()
dt_to_link = dt_factory_2.create_simple_datatype()
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
link_gr = dt_factory_2.create_datatype_group()
dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(link_gr.id)
expected_links.append(link_gr.gid)
# Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree
link_gr = dt_factory_2.create_datatype_group()
dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0]
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Actually create the links from Prj2 into Prj1
FlowService().create_link(link_ids, dt_factory_1.project.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(dt_factory_1.project.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(dt_factory_1.project, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
self.assertEqual(len(expected_links) + SELF_DTS_NUMBER + 2, len(dts_in_tree), "invalid number of nodes in tree")
self.assertFalse(link_gr.gid in dts_in_tree, "DT_group where a single DT is linked is not expected.")
self.assertTrue(dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!")
self.assertTrue(dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!")
project_dts = dao.get_datatypes_in_project(dt_factory_1.project.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
self.assertFalse(dt.gid in node_json, "DTs part of a group should not be")
self.assertFalse(dt.gid in dts_in_tree, "DTs part of a group should not be")
else:
self.assertTrue(dt.gid in node_json, "Simple DTs and DT_Groups should be")
self.assertTrue(dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be")
for link_gid in expected_links:
self.assertTrue(link_gid in node_json, "Expected Link not present")
self.assertTrue(link_gid in dts_in_tree, "Expected Link not present")
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 0, "There should be no operation"
adapter_instance = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 1, "Expected one operation group"
assert all_operations[0][2] == 2, "Expected 2 operations in group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 0, "There should be no operation")
algogroup = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapter3")
group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup)
adapter_instance = flow_service.build_adapter_instance(group)
data = {model.RANGE_PARAMETER_1: "param_5", "param_5": [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 1, "Expected one operation group")
self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group")
operation_group_id = all_operations[0][3]
self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.")
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
new_data[key] = submit_data[key]
if new_data[CommonDetails.CODE_OPERATION_TAG] == '':
new_data[CommonDetails.CODE_OPERATION_TAG] = None
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(operation_group_id=
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException("Inconsistent group, can not be updated!")
datatype_group = dao.get_generic_entity(DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
new_data[CommonDetails.CODE_GID] = datatype.gid
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception as excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def create_group(test_user=None, test_project=None, subject="John Doe"):
"""
Create a group of 2 operations, each with at least one resultant DataType.
"""
if test_user is None:
test_user = TestFactory.create_user()
if test_project is None:
test_project = TestFactory.create_project(test_user)
adapter_inst = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
adapter_inst.generic_attributes.subject = subject
view_model = adapter_inst.get_view_model()()
args = {RANGE_PARAMETER_1: 'param_5', 'param_5': json.dumps({constants.ATT_MINVALUE: 1,
constants.ATT_MAXVALUE: 2.1,
constants.ATT_STEP: 1})}
algo = adapter_inst.stored_adapter
algo_category = dao.get_category_by_id(algo.fk_category)
# Prepare Operations group. Execute them synchronously
service = OperationService()
operations = service.prepare_operations(test_user.id, test_project, algo, algo_category,
view_model=view_model, **args)[0]
service.launch_operation(operations[0].id, False, adapter_inst)
service.launch_operation(operations[1].id, False, adapter_inst)
resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group)
return resulted_dts, operations[0].fk_operation_group
def update_metadata(self, submit_data):
"""
Update DataType/ DataTypeGroup metadata
THROW StructureException when input data is invalid.
"""
new_data = dict()
for key in DataTypeOverlayDetails().meta_attributes_list:
if key in submit_data:
new_data[key] = submit_data[key]
if new_data[CommonDetails.CODE_OPERATION_TAG] == '':
new_data[CommonDetails.CODE_OPERATION_TAG] = None
try:
if (CommonDetails.CODE_OPERATION_GROUP_ID in new_data
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID]
and new_data[CommonDetails.CODE_OPERATION_GROUP_ID] != ''):
# We need to edit a group
all_data_in_group = dao.get_datatype_in_group(operation_group_id=
new_data[CommonDetails.CODE_OPERATION_GROUP_ID])
if len(all_data_in_group) < 1:
raise StructureException("Inconsistent group, can not be updated!")
datatype_group = dao.get_generic_entity(model.DataTypeGroup, all_data_in_group[0].fk_datatype_group)[0]
all_data_in_group.append(datatype_group)
for datatype in all_data_in_group:
new_data[CommonDetails.CODE_GID] = datatype.gid
self._edit_data(datatype, new_data, True)
else:
# Get the required DataType and operation from DB to store changes that will be done in XML.
gid = new_data[CommonDetails.CODE_GID]
datatype = dao.get_datatype_by_gid(gid)
self._edit_data(datatype, new_data)
except Exception, excep:
self.logger.exception(excep)
raise StructureException(str(excep))
def create_group(test_user=None, test_project=None, subject="John Doe"):
"""
Create a group of 2 operations, each with at least one resultant DataType.
"""
if test_user is None:
test_user = TestFactory.create_user()
if test_project is None:
test_project = TestFactory.create_project(test_user)
### Retrieve Adapter instance
algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
algo_category = dao.get_category_by_id(algo_group.fk_category)
algo = dao.get_algorithm_by_group(algo_group.id)
adapter_inst = TestFactory.create_adapter(algo_group=algo_group, test_project=test_project)
adapter_inst.meta_data = {DataTypeMetaData.KEY_SUBJECT: subject}
args = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
### Prepare Operations group. Execute them synchronously
service = OperationService()
operations = service.prepare_operations(test_user.id, test_project.id, algo, algo_category, {}, **args)[0]
service.launch_operation(operations[0].id, False, adapter_inst)
service.launch_operation(operations[1].id, False, adapter_inst)
resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group)
return resulted_dts, operations[0].fk_operation_group
def test_datatypes_groups(self, test_adapter_factory):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
# TODO: re-write this to use groups correctly
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 0, "There should be no operation"
algo = test_adapter_factory(TestAdapter3)
adapter_instance = ABCAdapter.build_adapter(algo)
data = {model_burst.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
FlowService().fire_operation(adapter_instance, self.test_user, self.test_project.id)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 1, "Expected one operation group"
assert all_operations[0][2] == 2, "Expected 2 operations in group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 0, "There should be no operation")
algogroup = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup)
adapter_instance = flow_service.build_adapter_instance(group)
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 1, "Expected one operation group")
self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group")
operation_group_id = all_operations[0][3]
self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.")
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def create_group(test_user=None, test_project=None, subject="John Doe"):
"""
Create a group of 2 operations, each with at least one resultant DataType.
"""
if test_user is None:
test_user = TestFactory.create_user()
if test_project is None:
test_project = TestFactory.create_project(test_user)
### Retrieve Adapter instance
algo_group = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
algo_category = dao.get_category_by_id(algo_group.fk_category)
algo = dao.get_algorithm_by_group(algo_group.id)
adapter_inst = TestFactory.create_adapter(algo_group=algo_group, test_project=test_project)
adapter_inst.meta_data = {DataTypeMetaData.KEY_SUBJECT: subject}
args = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
### Prepare Operations group. Execute them synchronously
service = OperationService()
operations = service.prepare_operations(test_user.id, test_project.id, algo, algo_category, {}, **args)[0]
service.launch_operation(operations[0].id, False, adapter_inst)
service.launch_operation(operations[1].id, False, adapter_inst)
resulted_dts = dao.get_datatype_in_group(operation_group_id=operations[0].fk_operation_group)
return resulted_dts, operations[0].fk_operation_group
def test_get_project_structure(self, datatype_group_factory,
dummy_datatype_index_factory,
project_factory, user_factory):
"""
Tests project structure is as expected and contains all datatypes and created links
"""
user = user_factory()
project1 = project_factory(user, name="TestPS1")
project2 = project_factory(user, name="TestPS2")
dt_group = datatype_group_factory(project=project1)
dt_simple = dummy_datatype_index_factory(state="RAW_DATA",
project=project1)
# Create 3 DTs directly in Project 2
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
dummy_datatype_index_factory(state="RAW_DATA", project=project2)
# Create Links from Project 1 into Project 2
link_ids, expected_links = [], []
link_ids.append(dt_simple.id)
expected_links.append(dt_simple.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
dts = dao.get_datatype_in_group(datatype_group_id=dt_group.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(dt_group.id)
expected_links.append(dt_group.gid)
# Actually create the links from Prj1 into Prj2
AlgorithmService().create_link(link_ids, project2.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(project2.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(
project2, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
assert len(expected_links) + 3 == len(
dts_in_tree), "invalid number of nodes in tree"
assert dt_group.gid in dts_in_tree, "DT_Group should be in the Project Tree!"
assert dt_group.gid in node_json, "DT_Group should be in the Project Tree JSON!"
project_dts = dao.get_datatypes_in_project(project2.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
assert not dt.gid in node_json, "DTs part of a group should not be"
assert not dt.gid in dts_in_tree, "DTs part of a group should not be"
else:
assert dt.gid in node_json, "Simple DTs and DT_Groups should be"
assert dt.gid in dts_in_tree, "Simple DTs and DT_Groups should be"
for link_gid in expected_links:
assert link_gid in node_json, "Expected Link not present"
assert link_gid in dts_in_tree, "Expected Link not present"
def test_datatypes_groups(self, test_adapter_factory,
datatype_group_factory):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
all_operations = dao.get_filtered_operations(self.test_project.id,
None)
assert len(all_operations) == 0, "There should be no operation"
dt_group = datatype_group_factory(project=self.test_project)
model = TestModel()
test_adapter_factory()
adapter = TestFactory.create_adapter(
"tvb.tests.framework.adapters.testadapter1", "TestAdapter1")
operations = dao.get_operations_in_group(dt_group.id)
for op in operations:
model.gid = uuid.uuid4()
op_path = StorageInterface().get_project_folder(
self.test_project.name, str(op.id))
op.view_model_gid = model.gid.hex
op.algorithm = adapter.stored_adapter
h5.store_view_model(model, op_path)
dao.store_entity(op)
all_operations = dao.get_filtered_operations(self.test_project.id,
None)
assert len(all_operations) == 2, "Expected two operation groups"
assert all_operations[0][2] == 6, "Expected 6 operations in one group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[1][0])
self.operation_service.stop_operation(all_operations[1][1])
# Make sure operations are executed
self.operation_service.launch_operation(all_operations[1][0], False)
self.operation_service.launch_operation(all_operations[1][1], False)
resulted_datatypes = dao.get_datatype_in_group(
operation_group_id=operation_group_id)
assert len(
resulted_datatypes) >= 2, "Expected at least 2, but: " + str(
len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def _remove_operation_group(self, operation_group_id, project_id, skip_validation, operations_set, links):
metrics_groups = dao.get_generic_entity(DataTypeGroup, operation_group_id,
'fk_operation_group')
if len(metrics_groups) > 0:
metric_datatype_group_id = metrics_groups[0].id
self._remove_datatype_group_dts(project_id, metric_datatype_group_id, skip_validation,
operations_set, links)
datatypes = dao.get_datatype_in_group(metric_datatype_group_id)
# If this condition is false, then there is a link for the datatype group and we don't need to delete it
if len(datatypes) == 0:
dao.remove_entity(DataTypeGroup, metric_datatype_group_id)
return dao.remove_entity(OperationGroup, operation_group_id)
else:
return True
def test_update_meta_data_group(self, test_adapter_factory, datatype_group_factory):
"""
Test the new update metaData for a group of dataTypes.
"""
test_adapter_factory(adapter_class=DummyAdapter3)
group, _ = datatype_group_factory()
op_group_id = group.fk_operation_group
new_meta_data = {DataTypeOverlayDetails.DATA_SUBJECT: "new subject",
DataTypeOverlayDetails.DATA_STATE: "updated_state",
DataTypeOverlayDetails.CODE_OPERATION_GROUP_ID: op_group_id,
DataTypeOverlayDetails.CODE_OPERATION_TAG: 'newGroupName'}
self.project_service.update_metadata(new_meta_data)
datatypes = dao.get_datatype_in_group(op_group_id)
for datatype in datatypes:
new_datatype = dao.get_datatype_by_id(datatype.id)
assert op_group_id == new_datatype.parent_operation.fk_operation_group
new_group = dao.get_generic_entity(model_operation.OperationGroup, op_group_id)[0]
assert new_group.name == "newGroupName"
self.__check_meta_data(new_meta_data, new_datatype)
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id,
None)
assert len(all_operations) == 0, "There should be no operation"
adapter_instance = TestFactory.create_adapter(
'tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user,
self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id,
None)
assert len(all_operations) == 1, "Expected one operation group"
assert all_operations[0][2] == 2, "Expected 2 operations in group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(
operation_group_id=operation_group_id)
assert len(
resulted_datatypes) >= 2, "Expected at least 2, but: " + str(
len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same DTgroup.
"""
return dao.get_datatype_in_group(datatype_group_id=group)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False, existing_dt_links=None):
"""
Method used for removing a dataType. If the given dataType is a DatatypeGroup
or a dataType from a DataTypeGroup than this method will remove the entire group.
The operation(s) used for creating the dataType(s) will also be removed.
"""
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is None:
self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid)
return
if datatype.parent_operation.fk_launched_in != int(project_id):
self.logger.warning("Datatype with GUID [%s] has been moved to another project and does "
"not need to be deleted anymore." % datatype_gid)
return
is_datatype_group = False
datatype_group = None
new_dt_links = []
# Datatype Groups were already handled when the first DatatypeMeasureIndex has been found
if dao.is_datatype_group(datatype_gid):
is_datatype_group = True
datatype_group = datatype
# Found the first DatatypeMeasureIndex from a group
elif datatype.fk_datatype_group is not None:
is_datatype_group = True
# We load it this way to make sure we have the 'fk_operation_group' in every case
datatype_group_gid = dao.get_datatype_by_id(datatype.fk_datatype_group).gid
datatype_group = h5.load_entity_by_gid(datatype_group_gid)
operations_set = [datatype.fk_from_operation]
correct = True
if is_datatype_group:
operations_set = [datatype_group.fk_from_operation]
self.logger.debug("Removing datatype group %s" % datatype_group)
datatypes = self.get_all_datatypes_from_data(datatype_group)
first_datatype = datatypes[0]
if hasattr(first_datatype, 'fk_source_gid'):
ts = h5.load_entity_by_gid(first_datatype.fk_source_gid)
ts_group = dao.get_datatypegroup_by_op_group_id(ts.parent_operation.fk_operation_group)
dm_group = datatype_group
else:
dt_measure_index = get_class_by_name("{}.{}".format(DATATYPE_MEASURE_INDEX_MODULE,
DATATYPE_MEASURE_INDEX_CLASS))
dm_group = dao.get_datatype_measure_group_from_ts_from_pse(first_datatype.gid, dt_measure_index)
ts_group = datatype_group
links = []
if ts_group:
links.extend(dao.get_links_for_datatype(ts_group.id))
correct = correct and self._remove_operation_group(ts_group.fk_operation_group, project_id,
skip_validation, operations_set, links)
if dm_group:
links.extend(dao.get_links_for_datatype(dm_group.id))
correct = correct and self._remove_operation_group(dm_group.fk_operation_group, project_id,
skip_validation, operations_set, links)
if len(links) > 0:
# We want to get the links for the first TSIndex directly
# This code works for all cases
datatypes = dao.get_datatype_in_group(ts_group.id)
ts = datatypes[0]
new_dt_links = self._add_links_for_datatype_references(ts, links[0].fk_to_project, links[0].id,
existing_dt_links)
else:
self.logger.debug("Removing datatype %s" % datatype)
links = dao.get_links_for_datatype(datatype.id)
if len(links) > 0:
new_dt_links = self._add_links_for_datatype_references(datatype, links[0].fk_to_project, links[0].id,
existing_dt_links)
self._remove_project_node_files(project_id, datatype.gid, links, skip_validation)
# Remove Operation entity in case no other DataType needs them.
project = dao.get_project_by_id(project_id)
for operation_id in operations_set:
dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation")
if len(dependent_dt) > 0:
# Do not remove Operation in case DataType still exist referring it.
continue
correct = correct and dao.remove_entity(Operation, operation_id)
# Make sure Operation folder is removed
self.storage_interface.remove_operation_data(project.name, operation_id)
self.storage_interface.push_folder_to_sync(project.name)
if not correct:
raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid))
return new_dt_links
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same DTgroup.
"""
return dao.get_datatype_in_group(datatype_group_id=group)
def get_datatype_in_group(group):
"""
Return all dataTypes that are the result of the same group of operations.
"""
return dao.get_datatype_in_group(group)
def test_get_project_structure(self):
"""
Tests project structure is as expected and contains all datatypes
"""
SELF_DTS_NUMBER = 3
dt_factory_1 = datatypes_factory.DatatypesFactory()
self._create_datatypes(dt_factory_1, SELF_DTS_NUMBER)
dt_group = dt_factory_1.create_datatype_group()
link_ids, expected_links = [], []
# Prepare link towards a simple DT
dt_factory_2 = datatypes_factory.DatatypesFactory()
dt_to_link = dt_factory_2.create_simple_datatype()
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Prepare links towards a full DT Group, but expecting only the DT_Group in the final tree
link_gr = dt_factory_2.create_datatype_group()
dts = dao.get_datatype_in_group(datatype_group_id=link_gr.id)
link_ids.extend([dt_to_link.id for dt_to_link in dts])
link_ids.append(link_gr.id)
expected_links.append(link_gr.gid)
# Prepare link towards a single DT inside a group, and expecting to find the DT in the final tree
link_gr = dt_factory_2.create_datatype_group()
dt_to_link = dao.get_datatype_in_group(datatype_group_id=link_gr.id)[0]
link_ids.append(dt_to_link.id)
expected_links.append(dt_to_link.gid)
# Actually create the links from Prj2 into Prj1
FlowService().create_link(link_ids, dt_factory_1.project.id)
# Retrieve the raw data used to compose the tree (for easy parsing)
dts_in_tree = dao.get_data_in_project(dt_factory_1.project.id)
dts_in_tree = [dt.gid for dt in dts_in_tree]
# Retrieve the tree json (for trivial validations only, as we can not decode)
node_json = self.project_service.get_project_structure(
dt_factory_1.project, None, DataTypeMetaData.KEY_STATE,
DataTypeMetaData.KEY_SUBJECT, None)
self.assertEqual(
len(expected_links) + SELF_DTS_NUMBER + 2, len(dts_in_tree),
"invalid number of nodes in tree")
self.assertFalse(
link_gr.gid in dts_in_tree,
"DT_group where a single DT is linked is not expected.")
self.assertTrue(dt_group.gid in dts_in_tree,
"DT_Group should be in the Project Tree!")
self.assertTrue(dt_group.gid in node_json,
"DT_Group should be in the Project Tree JSON!")
project_dts = dao.get_datatypes_in_project(dt_factory_1.project.id)
for dt in project_dts:
if dt.fk_datatype_group is not None:
self.assertFalse(dt.gid in node_json,
"DTs part of a group should not be")
self.assertFalse(dt.gid in dts_in_tree,
"DTs part of a group should not be")
else:
self.assertTrue(dt.gid in node_json,
"Simple DTs and DT_Groups should be")
self.assertTrue(dt.gid in dts_in_tree,
"Simple DTs and DT_Groups should be")
for link_gid in expected_links:
self.assertTrue(link_gid in node_json, "Expected Link not present")
self.assertTrue(link_gid in dts_in_tree,
"Expected Link not present")
