def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 0, "There should be no operation")
algogroup = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapter3")
group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup)
adapter_instance = flow_service.build_adapter_instance(group)
data = {model.RANGE_PARAMETER_1: "param_5", "param_5": [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 1, "Expected one operation group")
self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group")
operation_group_id = all_operations[0][3]
self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.")
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 0, "There should be no operation"
adapter_instance = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 1, "Expected one operation group"
assert all_operations[0][2] == 2, "Expected 2 operations in group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 0, "There should be no operation")
algogroup = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup)
adapter_instance = flow_service.build_adapter_instance(group)
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 1, "Expected one operation group")
self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group")
operation_group_id = all_operations[0][3]
self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.")
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def fire_simulation(project_id=1, **kwargs):
project = dao.get_project_by_id(project_id)
flow_service = FlowService()
# below the holy procedure to launch with the correct parameters taken from the defaults
stored_adapter = flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
simulator_adapter = ABCAdapter.build_adapter(stored_adapter)
flatten_interface = simulator_adapter.flaten_input_interface()
itree_mngr = flow_service.input_tree_manager
prepared_flatten_interface = itree_mngr.fill_input_tree_with_options(flatten_interface, project.id,
stored_adapter.fk_category)
launch_args = {}
for entry in prepared_flatten_interface:
value = entry['default']
if isinstance(value, dict):
value = str(value)
if hasattr(value, 'tolist'):
value = value.tolist()
launch_args[entry['name']] = value
launch_args.update(**kwargs)
# end of magic
launched_operation = flow_service.fire_operation(simulator_adapter, project.administrator,
project.id, **launch_args)[0]
return launched_operation
def fire_simulation(project_id=1, **kwargs):
project = dao.get_project_by_id(project_id)
flow_service = FlowService()
# below the holy procedure to launch with the correct parameters taken from the defaults
stored_adapter = flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
simulator_adapter = ABCAdapter.build_adapter(stored_adapter)
flatten_interface = simulator_adapter.flaten_input_interface()
itree_mngr = flow_service.input_tree_manager
prepared_flatten_interface = itree_mngr.fill_input_tree_with_options(flatten_interface, project.id,
stored_adapter.fk_category)
launch_args = {}
for entry in prepared_flatten_interface:
value = entry['default']
if isinstance(value, dict):
value = str(value)
if hasattr(value, 'tolist'):
value = value.tolist()
launch_args[entry['name']] = value
launch_args.update(**kwargs)
# end of magic
launched_operation = flow_service.fire_operation(simulator_adapter, project.administrator,
project.id, **launch_args)[0]
return launched_operation
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id,
None)
assert len(all_operations) == 0, "There should be no operation"
adapter_instance = TestFactory.create_adapter(
'tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user,
self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id,
None)
assert len(all_operations) == 1, "Expected one operation group"
assert all_operations[0][2] == 2, "Expected 2 operations in group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(
operation_group_id=operation_group_id)
assert len(
resulted_datatypes) >= 2, "Expected at least 2, but: " + str(
len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(
operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def import_h5(file_path, project_id):
flow_service = FlowService()
## This ID of a project needs to exists in Db, and it can be taken from the WebInterface:
project = dao.get_project_by_id(project_id)
adapter_instance = ABCAdapter.build_adapter_from_class(TVBImporter)
## Prepare the input algorithms as if they were coming from web UI submit:
launch_args = {"data_file": file_path}
print "We will try to import file at path " + file_path
## launch an operation and have the results stored both in DB and on disk
launched_operations = flow_service.fire_operation(
adapter_instance, project.administrator, project.id, **launch_args
)
print "Operation launched. Check the web UI"
def import_h5(file_path, project_id):
flow_service = FlowService()
## This ID of a project needs to exists in Db, and it can be taken from the WebInterface:
project = dao.get_project_by_id(project_id)
adapter_instance = ABCAdapter.build_adapter_from_class(TVBImporter)
## Prepare the input algorithms as if they were coming from web UI submit:
launch_args = {"data_file": file_path}
print "We will try to import file at path " + file_path
## launch an operation and have the results stored both in DB and on disk
launched_operations = flow_service.fire_operation(adapter_instance,
project.administrator,
project.id,
**launch_args)
print "Operation launched. Check the web UI"
class TestProjectStructure(TransactionalTestCase):
"""
Test ProjectService methods (part related to Project Data Structure).
"""
def transactional_setup_method(self):
"""
Prepare before each test.
"""
self.project_service = ProjectService()
self.flow_service = FlowService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, "ProjectStructure")
self.relevant_filter = StaticFiltersFactory.build_datatype_filters(
single_filter=StaticFiltersFactory.RELEVANT_VIEW)
self.full_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.FULL_VIEW)
def transactional_teardown_method(self):
"""
Clear project folders after testing
"""
self.delete_project_folders()
def test_set_operation_visibility(self):
"""
Check if the visibility for an operation is set correct.
"""
self.__init_algorithmn()
op1 = Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "")
op1 = dao.store_entity(op1)
assert op1.visible, "The operation should be visible."
self.project_service.set_operation_and_group_visibility(op1.gid, False)
updated_op = dao.get_operation_by_id(op1.id)
assert not updated_op.visible, "The operation should not be visible."
def test_set_op_and_group_visibility(self):
"""
When changing the visibility for an operation that belongs to an operation group, we
should also change the visibility for the entire group of operations.
"""
_, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1")
list_of_operations = dao.get_operations_in_group(group_id)
for operation in list_of_operations:
assert operation.visible, "The operation should be visible."
self.project_service.set_operation_and_group_visibility(list_of_operations[0].gid, False)
operations = dao.get_operations_in_group(group_id)
for operation in operations:
assert not operation.visible, "The operation should not be visible."
def test_set_op_group_visibility(self):
"""
Tests if the visibility for an operation group is set correct.
"""
_, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1")
list_of_operations = dao.get_operations_in_group(group_id)
for operation in list_of_operations:
assert operation.visible, "The operation should be visible."
op_group = dao.get_operationgroup_by_id(group_id)
self.project_service.set_operation_and_group_visibility(op_group.gid, False, True)
operations = dao.get_operations_in_group(group_id)
for operation in operations:
assert not operation.visible, "The operation should not be visible."
def test_is_upload_operation(self):
"""
Tests that upload and non-upload algorithms are created and run accordingly
"""
self.__init_algorithmn()
upload_algo = self._create_algo_for_upload()
op1 = Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "")
op2 = Operation(self.test_user.id, self.test_project.id, upload_algo.id, "")
operations = dao.store_entities([op1, op2])
is_upload_operation = self.project_service.is_upload_operation(operations[0].gid)
assert not is_upload_operation, "The operation is not an upload operation."
is_upload_operation = self.project_service.is_upload_operation(operations[1].gid)
assert is_upload_operation, "The operation is an upload operation."
def test_get_upload_operations(self):
"""
Test get_all when filter is for Upload category.
"""
self.__init_algorithmn()
upload_algo = self._create_algo_for_upload()
project = Project("test_proj_2", self.test_user.id, "desc")
project = dao.store_entity(project)
op1 = Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "")
op2 = Operation(self.test_user.id, project.id, upload_algo.id, "", status=STATUS_FINISHED)
op3 = Operation(self.test_user.id, self.test_project.id, upload_algo.id, "")
op4 = Operation(self.test_user.id, self.test_project.id, upload_algo.id, "", status=STATUS_FINISHED)
op5 = Operation(self.test_user.id, self.test_project.id, upload_algo.id, "", status=STATUS_FINISHED)
operations = dao.store_entities([op1, op2, op3, op4, op5])
upload_operations = self.project_service.get_all_operations_for_uploaders(self.test_project.id)
assert 2 == len(upload_operations), "Wrong number of upload operations."
upload_ids = [operation.id for operation in upload_operations]
for i in [3, 4]:
assert operations[i].id in upload_ids, \
"The operation should be an upload operation."
for i in [0, 1, 2]:
assert not operations[i].id in upload_ids, \
"The operation should not be an upload operation."
def test_is_datatype_group(self):
"""
Tests if a datatype is group.
"""
_, dt_group_id, first_dt, _ = self._create_datatype_group()
dt_group = dao.get_generic_entity(DataTypeGroup, dt_group_id)[0]
is_dt_group = self.project_service.is_datatype_group(dt_group.gid)
assert is_dt_group, "The datatype should be a datatype group."
is_dt_group = self.project_service.is_datatype_group(first_dt.gid)
assert not is_dt_group, "The datatype should not be a datatype group."
def test_count_datatypes_in_group(self):
""" Test that counting dataTypes is correct. Happy flow."""
_, dt_group_id, first_dt, _ = self._create_datatype_group()
count = dao.count_datatypes_in_group(dt_group_id)
assert count == 2
count = dao.count_datatypes_in_group(first_dt.id)
assert count == 0, "There should be no dataType."
def test_set_datatype_visibility(self):
"""
Check if the visibility for a datatype is set correct.
"""
# it's a list of 3 elem.
mapped_arrays = self._create_mapped_arrays(self.test_project.id)
for mapped_array in mapped_arrays:
is_visible = dao.get_datatype_by_id(mapped_array[0]).visible
assert is_visible, "The data type should be visible."
self.project_service.set_datatype_visibility(mapped_arrays[0][2], False)
for i in range(len(mapped_arrays)):
is_visible = dao.get_datatype_by_id(mapped_arrays[i][0]).visible
if not i:
assert not is_visible, "The data type should not be visible."
else:
assert is_visible, "The data type should be visible."
def test_set_visibility_for_dt_in_group(self):
"""
Check if the visibility for a datatype from a datatype group is set correct.
"""
_, dt_group_id, first_dt, second_dt = self._create_datatype_group()
assert first_dt.visible, "The data type should be visible."
assert second_dt.visible, "The data type should be visible."
self.project_service.set_datatype_visibility(first_dt.gid, False)
db_dt_group = self.project_service.get_datatype_by_id(dt_group_id)
db_first_dt = self.project_service.get_datatype_by_id(first_dt.id)
db_second_dt = self.project_service.get_datatype_by_id(second_dt.id)
assert not db_dt_group.visible, "The data type should be visible."
assert not db_first_dt.visible, "The data type should not be visible."
assert not db_second_dt.visible, "The data type should be visible."
def test_set_visibility_for_group(self):
"""
Check if the visibility for a datatype group is set correct.
"""
_, dt_group_id, first_dt, second_dt = self._create_datatype_group()
dt_group = dao.get_generic_entity(DataTypeGroup, dt_group_id)[0]
assert dt_group.visible, "The data type group should be visible."
assert first_dt.visible, "The data type should be visible."
assert second_dt.visible, "The data type should be visible."
self.project_service.set_datatype_visibility(dt_group.gid, False)
updated_dt_group = self.project_service.get_datatype_by_id(dt_group_id)
updated_first_dt = self.project_service.get_datatype_by_id(first_dt.id)
updated_second_dt = self.project_service.get_datatype_by_id(second_dt.id)
assert not updated_dt_group.visible, "The data type group should be visible."
assert not updated_first_dt.visible, "The data type should be visible."
assert not updated_second_dt.visible, "The data type should be visible."
def test_getdatatypes_from_dtgroup(self):
"""
Validate that we can retrieve all DTs from a DT_Group
"""
_, dt_group_id, first_dt, second_dt = self._create_datatype_group()
datatypes = self.project_service.get_datatypes_from_datatype_group(dt_group_id)
assert len(datatypes) == 2, "There should be 2 datatypes into the datatype group."
expected_dict = {first_dt.id: first_dt, second_dt.id: second_dt}
actual_dict = {datatypes[0].id: datatypes[0], datatypes[1].id: datatypes[1]}
for key in expected_dict:
expected = expected_dict[key]
actual = actual_dict[key]
assert expected.id == actual.id, "Not the same id."
assert expected.gid == actual.gid, "Not the same gid."
assert expected.type == actual.type, "Not the same type."
assert expected.subject == actual.subject, "Not the same subject."
assert expected.state == actual.state, "Not the same state."
assert expected.visible == actual.visible, "The datatype visibility is not correct."
assert expected.module == actual.module, "Not the same module."
assert expected.user_tag_1 == actual.user_tag_1, "Not the same user_tag_1."
assert expected.invalid == actual.invalid, "The invalid field value is not correct."
assert expected.is_nan == actual.is_nan, "The is_nan field value is not correct."
def test_get_operations_for_dt(self):
"""
Tests method get_operations_for_datatype.
Verifies result dictionary has the correct values
"""
created_ops, datatype_gid = self._create_operations_with_inputs()
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.relevant_filter)
assert len(operations) == 2
assert created_ops[0].id in [operations[0].id, operations[1].id], "Retrieved wrong operations."
assert created_ops[2].id in [operations[0].id, operations[1].id], "Retrieved wrong operations."
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.full_filter)
assert len(operations) == 4
ids = [operations[0].id, operations[1].id, operations[2].id, operations[3].id]
for i in range(4):
assert created_ops[i].id in ids, "Retrieved wrong operations."
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.relevant_filter, True)
assert len(operations) == 1
assert created_ops[4].id == operations[0].id, "Incorrect number of operations."
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.full_filter, True)
assert len(operations) == 2
assert created_ops[4].id in [operations[0].id, operations[1].id], "Retrieved wrong operations."
assert created_ops[5].id in [operations[0].id, operations[1].id], "Retrieved wrong operations."
def test_get_operations_for_dt_group(self):
"""
Tests method get_operations_for_datatype_group.
Verifies filters' influence over results is as expected
"""
created_ops, dt_group_id = self._create_operations_with_inputs(True)
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.relevant_filter)
assert len(ops) == 2
assert created_ops[0].id in [ops[0].id, ops[1].id], "Retrieved wrong operations."
assert created_ops[2].id in [ops[0].id, ops[1].id], "Retrieved wrong operations."
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.full_filter)
assert len(ops) == 4, "Incorrect number of operations."
ids = [ops[0].id, ops[1].id, ops[2].id, ops[3].id]
for i in range(4):
assert created_ops[i].id in ids, "Retrieved wrong operations."
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.relevant_filter, True)
assert len(ops) == 1
assert created_ops[4].id == ops[0].id, "Incorrect number of operations."
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.full_filter, True)
assert len(ops), 2
assert created_ops[4].id in [ops[0].id, ops[1].id], "Retrieved wrong operations."
assert created_ops[5].id in [ops[0].id, ops[1].id], "Retrieved wrong operations."
def test_get_inputs_for_operation(self):
"""
Tests method get_datatype_and_datatypegroup_inputs_for_operation.
Verifies filters' influence over results is as expected
"""
algo = dao.get_algorithm_by_module('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
array_wrappers = self._create_mapped_arrays(self.test_project.id)
ids = []
for datatype in array_wrappers:
ids.append(datatype[0])
datatype = dao.get_datatype_by_id(ids[0])
datatype.visible = False
dao.store_entity(datatype)
parameters = json.dumps({"param_5": "1", "param_1": array_wrappers[0][2],
"param_2": array_wrappers[1][2], "param_3": array_wrappers[2][2], "param_6": "0"})
operation = Operation(self.test_user.id, self.test_project.id, algo.id, parameters)
operation = dao.store_entity(operation)
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.relevant_filter)
assert len(inputs) == 2
assert ids[1] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType."
assert ids[2] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType."
assert not ids[0] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType."
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.full_filter)
assert len(inputs) == 3, "Incorrect number of operations."
assert ids[0] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType."
assert ids[1] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType."
assert ids[2] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType."
project, dt_group_id, first_dt, _ = self._create_datatype_group()
first_dt.visible = False
dao.store_entity(first_dt)
parameters = json.dumps({"other_param": "_", "param_1": first_dt.gid})
operation = Operation(self.test_user.id, project.id, algo.id, parameters)
operation = dao.store_entity(operation)
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.relevant_filter)
assert len(inputs) == 0, "Incorrect number of dataTypes."
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.full_filter)
assert len(inputs) == 1, "Incorrect number of dataTypes."
assert inputs[0].id == dt_group_id, "Wrong dataType."
assert inputs[0].id != first_dt.id, "Wrong dataType."
def test_get_inputs_for_op_group(self):
"""
Tests method get_datatypes_inputs_for_operation_group.
The DataType inputs will be from a DataType group.
"""
project, dt_group_id, first_dt, second_dt = self._create_datatype_group()
first_dt.visible = False
dao.store_entity(first_dt)
second_dt.visible = False
dao.store_entity(second_dt)
op_group = OperationGroup(project.id, "group", "range1[1..2]")
op_group = dao.store_entity(op_group)
params_1 = json.dumps({"param_5": "1", "param_1": first_dt.gid, "param_6": "2"})
params_2 = json.dumps({"param_5": "1", "param_4": second_dt.gid, "param_6": "5"})
algo = dao.get_algorithm_by_module('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
op1 = Operation(self.test_user.id, project.id, algo.id, params_1, op_group_id=op_group.id)
op2 = Operation(self.test_user.id, project.id, algo.id, params_2, op_group_id=op_group.id)
dao.store_entities([op1, op2])
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter)
assert len(inputs) == 0
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter)
assert len(inputs) == 1, "Incorrect number of dataTypes."
assert not first_dt.id == inputs[0].id, "Retrieved wrong dataType."
assert not second_dt.id == inputs[0].id, "Retrieved wrong dataType."
assert dt_group_id == inputs[0].id, "Retrieved wrong dataType."
first_dt.visible = True
dao.store_entity(first_dt)
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter)
assert len(inputs) == 1, "Incorrect number of dataTypes."
assert not first_dt.id == inputs[0].id, "Retrieved wrong dataType."
assert not second_dt.id == inputs[0].id, "Retrieved wrong dataType."
assert dt_group_id == inputs[0].id, "Retrieved wrong dataType."
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter)
assert len(inputs) == 1, "Incorrect number of dataTypes."
assert not first_dt.id == inputs[0].id, "Retrieved wrong dataType."
assert not second_dt.id == inputs[0].id, "Retrieved wrong dataType."
assert dt_group_id == inputs[0].id, "Retrieved wrong dataType."
def test_get_inputs_for_op_group_simple_inputs(self):
"""
Tests method get_datatypes_inputs_for_operation_group.
The dataType inputs will not be part of a dataType group.
"""
# it's a list of 3 elem.
array_wrappers = self._create_mapped_arrays(self.test_project.id)
array_wrapper_ids = []
for datatype in array_wrappers:
array_wrapper_ids.append(datatype[0])
datatype = dao.get_datatype_by_id(array_wrapper_ids[0])
datatype.visible = False
dao.store_entity(datatype)
op_group = OperationGroup(self.test_project.id, "group", "range1[1..2]")
op_group = dao.store_entity(op_group)
params_1 = json.dumps({"param_5": "2", "param_1": array_wrappers[0][2],
"param_2": array_wrappers[1][2], "param_6": "7"})
params_2 = json.dumps({"param_5": "5", "param_3": array_wrappers[2][2],
"param_2": array_wrappers[1][2], "param_6": "6"})
algo = dao.get_algorithm_by_module('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
op1 = Operation(self.test_user.id, self.test_project.id, algo.id, params_1, op_group_id=op_group.id)
op2 = Operation(self.test_user.id, self.test_project.id, algo.id, params_2, op_group_id=op_group.id)
dao.store_entities([op1, op2])
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter)
assert len(inputs) == 2
assert not array_wrapper_ids[0] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType."
assert array_wrapper_ids[1] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType."
assert array_wrapper_ids[2] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType."
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter)
assert len(inputs) == 3, "Incorrect number of dataTypes."
assert array_wrapper_ids[0] in [inputs[0].id, inputs[1].id, inputs[2].id]
assert array_wrapper_ids[1] in [inputs[0].id, inputs[1].id, inputs[2].id]
assert array_wrapper_ids[2] in [inputs[0].id, inputs[1].id, inputs[2].id]
def test_remove_datatype(self):
"""
Tests the deletion of a datatype.
"""
# it's a list of 3 elem.
array_wrappers = self._create_mapped_arrays(self.test_project.id)
dt_list = []
for array_wrapper in array_wrappers:
dt_list.append(dao.get_datatype_by_id(array_wrapper[0]))
self.project_service.remove_datatype(self.test_project.id, dt_list[0].gid)
self._check_if_datatype_was_removed(dt_list[0])
def test_remove_datatype_from_group(self):
"""
Tests the deletion of a datatype group.
"""
project, dt_group_id, first_dt, second_dt = self._create_datatype_group()
datatype_group = dao.get_generic_entity(DataTypeGroup, dt_group_id)[0]
self.project_service.remove_datatype(project.id, first_dt.gid)
self._check_if_datatype_was_removed(first_dt)
self._check_if_datatype_was_removed(second_dt)
self._check_if_datatype_was_removed(datatype_group)
self._check_datatype_group_removed(dt_group_id, datatype_group.fk_operation_group)
def test_remove_datatype_group(self):
"""
Tests the deletion of a datatype group.
"""
project, dt_group_id, first_dt, second_dt = self._create_datatype_group()
datatype_group = dao.get_generic_entity(DataTypeGroup, dt_group_id)[0]
self.project_service.remove_datatype(project.id, datatype_group.gid)
self._check_if_datatype_was_removed(first_dt)
self._check_if_datatype_was_removed(second_dt)
self._check_if_datatype_was_removed(datatype_group)
self._check_datatype_group_removed(dt_group_id, datatype_group.fk_operation_group)
def _create_mapped_arrays(self, project_id):
"""
:param project_id: the project in which the arrays are created
:return: a list of dummy `MappedArray`
"""
count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1]
assert count == 0
group = dao.get_algorithm_by_module('tvb.tests.framework.adapters.ndimensionarrayadapter',
'NDimensionArrayAdapter')
adapter_instance = ABCAdapter.build_adapter(group)
data = {'param_1': 'some value'}
# create 3 data types
self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data)
count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1]
assert count == 1
self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data)
count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1]
assert count == 2
self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data)
array_wrappers, count = self.flow_service.get_available_datatypes(project_id,
"tvb.datatypes.arrays.MappedArray")
assert count == 3
return array_wrappers
def _create_operation(self, project_id, algorithm_id):
"""
dummy operation
:param project_id: the project in which the operation is created
:param algorithm_id: the algorithm to be run for the operation
:return: a dummy `Operation` with the given specifications
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"}
operation = Operation(self.test_user.id, project_id, algorithm.id, 'test params',
meta=json.dumps(meta), status=STATUS_FINISHED)
return dao.store_entity(operation)
def _create_datatype_group(self):
"""
Creates a project, one DataTypeGroup with 2 DataTypes into the new group.
"""
test_project = TestFactory.create_project(self.test_user, "NewProject")
all_operations = dao.get_filtered_operations(test_project.id, None, is_count=True)
assert 0 == all_operations, "There should be no operation."
datatypes, op_group_id = TestFactory.create_group(self.test_user, test_project)
dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id)
return test_project, dt_group.id, datatypes[0], datatypes[1]
def _create_operations_with_inputs(self, is_group_parent=False):
"""
Method used for creating a complex tree of operations.
If 'if_group_parent' is True then a new group will be created and one of its entries it will be used as
input for the returned operations.
"""
group_dts, root_op_group_id = TestFactory.create_group(self.test_user, self.test_project)
if is_group_parent:
datatype_gid = group_dts[0].gid
else:
datatype_gid = TestProjectService._create_value_wrapper(self.test_user, self.test_project)[1]
parameters = json.dumps({"param_name": datatype_gid})
ops = []
for i in range(4):
ops.append(TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project))
if i in [1, 3]:
ops[i].visible = False
ops[i].parameters = parameters
ops[i] = dao.store_entity(ops[i])
# groups
_, ops_group = TestFactory.create_group(self.test_user, self.test_project)
ops_group = dao.get_operations_in_group(ops_group)
assert 2 == len(ops_group)
ops_group[0].parameters = parameters
ops_group[0] = dao.store_entity(ops_group[0])
ops_group[1].visible = False
ops_group[1].parameters = parameters
ops_group[1] = dao.store_entity(ops_group[1])
ops.extend(ops_group)
if is_group_parent:
dt_group = dao.get_datatypegroup_by_op_group_id(root_op_group_id)
return ops, dt_group.id
return ops, datatype_gid
def _check_if_datatype_was_removed(self, datatype):
"""
Check if a certain datatype was removed.
"""
try:
dao.get_datatype_by_id(datatype.id)
raise AssertionError("The datatype was not deleted.")
except Exception:
pass
try:
dao.get_operation_by_id(datatype.fk_from_operation)
raise AssertionError("The operation was not deleted.")
except Exception:
pass
def _check_datatype_group_removed(self, datatype_group_id, operation_groupp_id):
"""
Checks if the DataTypeGroup and OperationGroup was removed.
"""
try:
dao.get_generic_entity(DataTypeGroup, datatype_group_id)
raise AssertionError("The DataTypeGroup entity was not removed.")
except Exception:
pass
try:
dao.get_operationgroup_by_id(operation_groupp_id)
raise AssertionError("The OperationGroup entity was not removed.")
except Exception:
pass
def __init_algorithmn(self):
"""
Insert some starting data in the database.
"""
categ1 = AlgorithmCategory('one', True)
self.categ1 = dao.store_entity(categ1)
ad = Algorithm(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS, categ1.id)
self.algo_inst = dao.store_entity(ad)
@staticmethod
def _create_algo_for_upload():
""" Creates a fake algorithm for an upload category. """
category = dao.store_entity(AlgorithmCategory("upload_category", rawinput=True))
return dao.store_entity(Algorithm("module", "classname", category.id))
class ModelValidator(object):
overwrites = {}
def __init__(self, overwrites=None, settings_file=None):
""" Parameters can be overwritten either from a settigns file or from a dictionary. """
if overwrites is not None:
self.overwrites.update(overwrites)
if settings_file is not None:
settings = open(sys.argv[1]).read()
for line in settings.split('\n'):
key, value = line.split('=')
self.overwrites[key.strip()] = value.strip()
if KEY_PROJECT not in self.overwrites:
raise Exception("Settings file should contain the id of the project: %s=1" % KEY_PROJECT)
self.project = dao.get_project_by_id(self.overwrites[KEY_PROJECT])
self.flow_service = FlowService()
self.operation_service = OperationService()
def launch_validation(self):
"""
Prepare the arguments to be submitted and launch actual operations group.
TODO: Now get the results and check if any errors
"""
stored_adapter = self.flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
simulator_adapter = ABCAdapter.build_adapter(stored_adapter)
launch_args = {}
flatten_interface = simulator_adapter.flaten_input_interface()
itree_mngr = self.flow_service.input_tree_manager
prepared_flatten_interface = itree_mngr.fill_input_tree_with_options(flatten_interface, self.project.id,
stored_adapter.fk_category)
for entry in prepared_flatten_interface:
value = entry['default']
if isinstance(value, dict):
value = str(value)
if hasattr(value, 'tolist'):
value = value.tolist()
launch_args[entry['name']] = value
launch_args.update(self.overwrites)
nr_of_operations = 1
for key in self.overwrites:
if key.startswith(PARAM_RANGE_PREFIX):
range_values = self.operation_service.get_range_values(launch_args, key)
nr_of_operations *= len(range_values)
do_launch = False
print "Warning! This will launch %s operations. Do you agree? (yes/no)" % nr_of_operations
while 1:
accept = raw_input()
if accept.lower() == 'yes':
do_launch = True
break
if accept.lower() == 'no':
do_launch = False
break
print "Please type either yes or no"
if do_launch:
self.launched_operations = self.flow_service.fire_operation(simulator_adapter, self.project.administrator,
self.project.id, **launch_args)
return self.validate_results(0)
else:
return "Operation canceled by user."
def validate_results(self, last_verified_index):
error_count = 0
while last_verified_index < len(self.launched_operations):
operation_to_check = self.launched_operations[last_verified_index]
operation = dao.get_operation_by_id(operation_to_check.id)
if not operation.has_finished:
sleep(10)
if operation.status == STATUS_ERROR:
sys.stdout.write("E(" + str(operation_to_check.id) + ")")
error_count += 1
last_verified_index += 1
sys.stdout.flush()
if operation.status == STATUS_FINISHED:
last_verified_index += 1
sys.stdout.write('.')
sys.stdout.flush()
if error_count:
return "%s operations in error; %s operations successfully." % (error_count,
len(self.launched_operations) - error_count)
return "All operations finished successfully!"
class FlowServiceTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.flow_service module.
"""
def setUp(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
### Insert some starting data in the database.
categ1 = model.AlgorithmCategory('one', True)
self.categ1 = dao.store_entity(categ1)
categ2 = model.AlgorithmCategory('two', rawinput=True)
self.categ2 = dao.store_entity(categ2)
group1 = model.AlgorithmGroup("test_module1", "classname1", categ1.id)
self.algo_group1 = dao.store_entity(group1)
group2 = model.AlgorithmGroup("test_module2", "classname2", categ2.id)
self.algo_group2 = dao.store_entity(group2)
group3 = model.AlgorithmGroup("test_module3", "classname3", categ1.id)
self.algo_group3 = dao.store_entity(group3)
group_v = model.AlgorithmGroup(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS, categ2.id)
self.algo_group_v = dao.store_entity(group_v)
algo_v = model.Algorithm(self.algo_group_v.id, 'ident', name='', req_data='', param_name='', output='')
self.algorithm_v = dao.store_entity(algo_v)
algo1 = model.Algorithm(self.algo_group1.id, 'id', name='', req_data='', param_name='', output='')
self.algorithm1 = dao.store_entity(algo1)
def tearDown(self):
for algo in [self.algorithm1, self.algorithm_v]:
dao.remove_entity(model.Algorithm, algo.id)
for group in [self.algo_group1, self.algo_group2, self.algo_group3, self.algorithm_v]:
dao.remove_entity(model.AlgorithmGroup, group.id)
for categ in [self.categ1, self.categ2]:
dao.remove_entity(model.AlgorithmCategory, categ.id)
def test_groups_for_categories(self):
"""
Test getting algorithms for specific categories.
"""
category1 = self.flow_service.get_groups_for_categories([self.categ1])
category2 = self.flow_service.get_groups_for_categories([self.categ2])
dummy = model.AlgorithmCategory('dummy', rawinput=True)
dummy.id = 999
unexisting_cat = self.flow_service.get_groups_for_categories([dummy])
self.assertEqual(len(category1), 2)
self.assertEqual(len(category2), 2)
self.assertEqual(len(unexisting_cat), 0)
for group in category1:
if group.module not in ["test_module1", "test_module3"]:
self.fail("Some invalid data retrieved")
for group in category2:
if group.module not in ["test_module2", TEST_ADAPTER_VALID_MODULE]:
self.fail("Some invalid data retrieved")
def test_get_broup_by_identifier(self):
"""
Test for the get_algorithm_by_identifier.
"""
algo_ret = self.flow_service.get_algo_group_by_identifier(self.algo_group1.id)
self.assertEqual(algo_ret.id, self.algo_group1.id, "ID-s are different!")
self.assertEqual(algo_ret.module, self.algo_group1.module, "Modules are different!")
self.assertEqual(algo_ret.fk_category, self.algo_group1.fk_category, "Categories are different!")
self.assertEqual(algo_ret.classname, self.algo_group1.classname, "Class names are different!")
def test_build_adapter_instance(self):
"""
Test standard flow for building an adapter instance.
"""
algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
adapter = ABCAdapter.build_adapter(algo_group)
self.assertTrue(isinstance(adapter, ABCSynchronous), "Something went wrong with valid data!")
def test_build_adapter_invalid(self):
"""
Test flow for trying to build an adapter that does not inherit from ABCAdapter.
"""
group = dao.find_group(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_INVALID_CLASS)
self.assertRaises(OperationException, self.flow_service.build_adapter_instance, group)
def test_prepare_adapter(self):
"""
Test preparation of an adapter.
"""
algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
group, interface = self.flow_service.prepare_adapter(self.test_project.id, algo_group)
self.assertTrue(isinstance(group, model.AlgorithmGroup), "Something went wrong with valid data!")
self.assertTrue("name" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["name"], "test", "Bad interface!")
self.assertTrue("type" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["type"], "int", "Bad interface!")
self.assertTrue("default" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["default"], "0", "Bad interface!")
def test_fire_operation(self):
"""
Test preparation of an adapter and launch mechanism.
"""
algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
adapter = self.flow_service.build_adapter_instance(algo_group)
data = {"test": 5}
result = self.flow_service.fire_operation(adapter, self.test_user, self.test_project.id, **data)
self.assertTrue(result.endswith("has finished."), "Operation fail")
def test_get_filtered_by_column(self):
"""
Test the filter function when retrieving dataTypes with a filter
after a column from a class specific table (e.g. DATA_arraywrapper).
"""
operation_1 = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
operation_2 = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
one_dim_array = numpy.arange(5)
two_dim_array = numpy.array([[1, 2], [2, 3], [1, 4]])
self._store_float_array(one_dim_array, "John Doe 1", operation_1.id)
self._store_float_array(one_dim_array, "John Doe 2", operation_1.id)
self._store_float_array(two_dim_array, "John Doe 3", operation_2.id)
count = self.flow_service.get_available_datatypes(self.test_project.id, "tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(count, 3, "Problems with inserting data")
first_filter = FilterChain(fields=[FilterChain.datatype + '._nr_dimensions'], operations=["=="], values=[1])
count = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray", first_filter)[1]
self.assertEqual(count, 2, "Data was not filtered")
second_filter = FilterChain(fields=[FilterChain.datatype + '._nr_dimensions'], operations=["=="], values=[2])
filtered_data = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray", second_filter)[0]
self.assertEqual(len(filtered_data), 1, "Data was not filtered")
self.assertEqual(filtered_data[0][3], "John Doe 3")
third_filter = FilterChain(fields=[FilterChain.datatype + '._length_1d'], operations=["=="], values=[3])
filtered_data = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray", third_filter)[0]
self.assertEqual(len(filtered_data), 1, "Data was not filtered correct")
self.assertEqual(filtered_data[0][3], "John Doe 3")
try:
if os.path.exists('One_dim.txt'):
os.remove('One_dim.txt')
if os.path.exists('Two_dim.txt'):
os.remove('Two_dim.txt')
if os.path.exists('One_dim-1.txt'):
os.remove('One_dim-1.txt')
except Exception:
pass
@staticmethod
def _store_float_array(array_data, subject_name, operation_id):
"""Create Float Array and DB persist it"""
datatype_inst = MappedArray(user_tag_1=subject_name)
datatype_inst.set_operation_id(operation_id)
datatype_inst.array_data = array_data
datatype_inst.type = "MappedArray"
datatype_inst.module = "tvb.datatypes.arrays"
datatype_inst.subject = subject_name
datatype_inst.state = "RAW"
dao.store_entity(datatype_inst)
def test_get_filtered_datatypes(self):
"""
Test the filter function when retrieving dataTypes.
"""
#Create some test operations
start_dates = [datetime.now(),
datetime.strptime("08-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2010", "%m-%d-%Y"),
datetime.strptime("05-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2011", "%m-%d-%Y")]
end_dates = [datetime.now(),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y")]
for i in range(5):
operation = model.Operation(self.test_user.id, self.test_project.id, self.algorithm1.id, 'test params',
status=model.STATUS_FINISHED, start_date=start_dates[i],
completion_date=end_dates[i])
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(self.test_project, str(operation.id))
if i < 4:
datatype_inst = Datatype1()
datatype_inst.type = "Datatype1"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
else:
for _ in range(2):
datatype_inst = Datatype2()
datatype_inst.storage_path = storage_path
datatype_inst.type = "Datatype2"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.string_data = ["data"]
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
returned_data = self.flow_service.get_available_datatypes(self.test_project.id, Datatype1)[0]
for row in returned_data:
if row[1] != 'Datatype1':
self.fail("Some invalid data was returned!")
self.assertEqual(4, len(returned_data), "Invalid length of result")
filter_op = FilterChain(fields=[FilterChain.datatype + ".state", FilterChain.operation + ".start_date"],
values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")], operations=["==", ">"])
returned_data = self.flow_service.get_available_datatypes(self.test_project.id, Datatype1, filter_op)[0]
returned_subjects = [one_data[3] for one_data in returned_data]
if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len(returned_subjects) != 2:
self.fail("DataTypes were not filtered properly!")
class TestFlowService(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.flow_service module.
"""
def transactional_setup_method(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
category = dao.get_uploader_categories()[0]
self.algorithm = dao.store_entity(
model.Algorithm(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, category.id))
def transactional_teardown_method(self):
dao.remove_entity(model.Algorithm, self.algorithm)
def test_get_uploaders(self):
result = self.flow_service.get_upload_algorithms()
assert 29 == len(result)
found = False
for algo in result:
if algo.classname == self.algorithm.classname and algo.module == self.algorithm.module:
found = True
break
assert found, "Uploader incorrectly returned"
def test_get_analyze_groups(self):
category, groups = self.flow_service.get_analyze_groups()
assert category.displayname == 'Analyze'
assert len(groups) > 1
assert isinstance(groups[0], model.AlgorithmTransientGroup)
def test_get_visualizers_for_group(self):
_, op_group_id = TestFactory.create_group(self.test_user,
self.test_project)
dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id)
result = self.flow_service.get_visualizers_for_group(dt_group.gid)
# Only the discreet is expected
assert 1 == len(result)
assert DISCRETE_PSE_ADAPTER_CLASS == result[0].classname
def test_get_launchable_algorithms(self):
factory = DatatypesFactory()
conn = factory.create_connectivity(4)[1]
ts = factory.create_timeseries(conn)
result = self.flow_service.get_launchable_algorithms(ts.gid)
assert 'Analyze' in result
assert 'View' in result
def test_get_roup_by_identifier(self):
"""
Test for the get_algorithm_by_identifier.
"""
algo_ret = self.flow_service.get_algorithm_by_identifier(
self.algorithm.id)
assert algo_ret.id == self.algorithm.id, "ID-s are different!"
assert algo_ret.module == self.algorithm.module, "Modules are different!"
assert algo_ret.fk_category == self.algorithm.fk_category, "Categories are different!"
assert algo_ret.classname == self.algorithm.classname, "Class names are different!"
def test_build_adapter_instance(self):
"""
Test standard flow for building an adapter instance.
"""
adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS)
assert isinstance(
adapter, ABCSynchronous), "Something went wrong with valid data!"
def test_build_adapter_invalid(self):
"""
Test flow for trying to build an adapter that does not inherit from ABCAdapter.
"""
group = dao.get_algorithm_by_module(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_INVALID_CLASS)
with pytest.raises(IntrospectionException):
ABCAdapter.build_adapter(group)
def test_prepare_adapter(self):
"""
Test preparation of an adapter.
"""
stored_adapter = dao.get_algorithm_by_module(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS)
interface = self.flow_service.prepare_adapter(self.test_project.id,
stored_adapter)
assert isinstance(
stored_adapter,
model.Algorithm), "Something went wrong with valid data!"
assert "name" in interface[0], "Bad interface created!"
assert interface[0]["name"] == "test", "Bad interface!"
assert "type" in interface[0], "Bad interface created!"
assert interface[0]["type"] == "int", "Bad interface!"
assert "default" in interface[0], "Bad interface created!"
assert interface[0]["default"] == "0", "Bad interface!"
def test_fire_operation(self):
"""
Test preparation of an adapter and launch mechanism.
"""
adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS)
data = {"test": 5}
result = self.flow_service.fire_operation(adapter, self.test_user,
self.test_project.id, **data)
assert result.endswith("has finished."), "Operation fail"
def test_get_filtered_by_column(self):
"""
Test the filter function when retrieving dataTypes with a filter
after a column from a class specific table (e.g. DATA_arraywrapper).
"""
operation_1 = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
operation_2 = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
one_dim_array = numpy.arange(5)
two_dim_array = numpy.array([[1, 2], [2, 3], [1, 4]])
self._store_float_array(one_dim_array, "John Doe 1", operation_1.id)
self._store_float_array(one_dim_array, "John Doe 2", operation_1.id)
self._store_float_array(two_dim_array, "John Doe 3", operation_2.id)
count = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray")[1]
assert count, 3 == "Problems with inserting data"
first_filter = FilterChain(
fields=[FilterChain.datatype + '._nr_dimensions'],
operations=["=="],
values=[1])
count = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray",
first_filter)[1]
assert count, 2 == "Data was not filtered"
second_filter = FilterChain(
fields=[FilterChain.datatype + '._nr_dimensions'],
operations=["=="],
values=[2])
filtered_data = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray",
second_filter)[0]
assert len(filtered_data), 1 == "Data was not filtered"
assert filtered_data[0][3] == "John Doe 3"
third_filter = FilterChain(
fields=[FilterChain.datatype + '._length_1d'],
operations=["=="],
values=[3])
filtered_data = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray",
third_filter)[0]
assert len(filtered_data), 1 == "Data was not filtered correct"
assert filtered_data[0][3] == "John Doe 3"
try:
if os.path.exists('One_dim.txt'):
os.remove('One_dim.txt')
if os.path.exists('Two_dim.txt'):
os.remove('Two_dim.txt')
if os.path.exists('One_dim-1.txt'):
os.remove('One_dim-1.txt')
except Exception:
pass
@staticmethod
def _store_float_array(array_data, subject_name, operation_id):
"""Create Float Array and DB persist it"""
datatype_inst = MappedArray(user_tag_1=subject_name)
datatype_inst.set_operation_id(operation_id)
datatype_inst.array_data = array_data
datatype_inst.type = "MappedArray"
datatype_inst.module = "tvb.datatypes.arrays"
datatype_inst.subject = subject_name
datatype_inst.state = "RAW"
dao.store_entity(datatype_inst)
def test_get_filtered_datatypes(self):
"""
Test the filter function when retrieving dataTypes.
"""
#Create some test operations
start_dates = [
datetime.now(),
datetime.strptime("08-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2010", "%m-%d-%Y"),
datetime.strptime("05-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2011", "%m-%d-%Y")
]
end_dates = [
datetime.now(),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y")
]
for i in range(5):
operation = model.Operation(self.test_user.id,
self.test_project.id,
self.algorithm.id,
'test params',
status=model.STATUS_FINISHED,
start_date=start_dates[i],
completion_date=end_dates[i])
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(
self.test_project, str(operation.id))
if i < 4:
datatype_inst = Datatype1()
datatype_inst.type = "Datatype1"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
else:
for _ in range(2):
datatype_inst = Datatype2()
datatype_inst.storage_path = storage_path
datatype_inst.type = "Datatype2"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.string_data = ["data"]
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
returned_data = self.flow_service.get_available_datatypes(
self.test_project.id, Datatype1)[0]
for row in returned_data:
if row[1] != 'Datatype1':
raise AssertionError("Some invalid data was returned!")
assert 4 == len(returned_data), "Invalid length of result"
filter_op = FilterChain(
fields=[
FilterChain.datatype + ".state",
FilterChain.operation + ".start_date"
],
values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")],
operations=["==", ">"])
returned_data = self.flow_service.get_available_datatypes(
self.test_project.id, Datatype1, filter_op)[0]
returned_subjects = [one_data[3] for one_data in returned_data]
if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len(
returned_subjects) != 2:
raise AssertionError("DataTypes were not filtered properly!")
class ProjectStructureTest(TransactionalTestCase):
"""
Test ProjectService methods (part related to Project Data Structure).
"""
def setUp(self):
"""
Prepare before each test.
"""
self.project_service = ProjectService()
self.flow_service = FlowService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, "ProjectStructure")
self.relevant_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.RELEVANT_VIEW)
self.full_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.FULL_VIEW)
def tearDown(self):
"""
Clear project folders after testing
"""
self.delete_project_folders()
def test_set_operation_visibility(self):
"""
Check if the visibility for an operation is set correct.
"""
self.__init_algorithmn()
op1 = model.Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "")
op1 = dao.store_entity(op1)
self.assertTrue(op1.visible, "The operation should be visible.")
self.project_service.set_operation_and_group_visibility(op1.gid, False)
updated_op = dao.get_operation_by_id(op1.id)
self.assertFalse(updated_op.visible, "The operation should not be visible.")
def test_set_op_and_group_visibility(self):
"""
When changing the visibility for an operation that belongs to an operation group, we
should also change the visibility for the entire group of operations.
"""
_, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1")
list_of_operations = dao.get_operations_in_group(group_id)
for operation in list_of_operations:
self.assertTrue(operation.visible, "The operation should be visible.")
self.project_service.set_operation_and_group_visibility(list_of_operations[0].gid, False)
operations = dao.get_operations_in_group(group_id)
for operation in operations:
self.assertFalse(operation.visible, "The operation should not be visible.")
def test_set_op_group_visibility(self):
"""
Tests if the visibility for an operation group is set correct.
"""
_, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1")
list_of_operations = dao.get_operations_in_group(group_id)
for operation in list_of_operations:
self.assertTrue(operation.visible, "The operation should be visible.")
op_group = dao.get_operationgroup_by_id(group_id)
self.project_service.set_operation_and_group_visibility(op_group.gid, False, True)
operations = dao.get_operations_in_group(group_id)
for operation in operations:
self.assertFalse(operation.visible, "The operation should not be visible.")
def test_is_upload_operation(self):
"""
Tests that upload and non-upload algorithms are created and run accordingly
"""
self.__init_algorithmn()
upload_algo = self._create_algo_for_upload()
op1 = model.Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "")
op2 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "")
operations = dao.store_entities([op1, op2])
is_upload_operation = self.project_service.is_upload_operation(operations[0].gid)
self.assertFalse(is_upload_operation, "The operation is not an upload operation.")
is_upload_operation = self.project_service.is_upload_operation(operations[1].gid)
self.assertTrue(is_upload_operation, "The operation is an upload operation.")
def test_get_upload_operations(self):
"""
Test get_all when filter is for Upload category.
"""
self.__init_algorithmn()
upload_algo = self._create_algo_for_upload()
project = model.Project("test_proj_2", self.test_user.id, "desc")
project = dao.store_entity(project)
op1 = model.Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, "")
op2 = model.Operation(self.test_user.id, project.id, upload_algo.id, "", status=model.STATUS_FINISHED)
op3 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "")
op4 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "", status=model.STATUS_FINISHED)
op5 = model.Operation(self.test_user.id, self.test_project.id, upload_algo.id, "", status=model.STATUS_FINISHED)
operations = dao.store_entities([op1, op2, op3, op4, op5])
upload_operations = self.project_service.get_all_operations_for_uploaders(self.test_project.id)
self.assertEqual(2, len(upload_operations), "Wrong number of upload operations.")
upload_ids = [operation.id for operation in upload_operations]
for i in [3, 4]:
self.assertTrue(operations[i].id in upload_ids,
"The operation should be an upload operation.")
for i in [0, 1, 2]:
self.assertFalse(operations[i].id in upload_ids,
"The operation should not be an upload operation.")
def test_is_datatype_group(self):
"""
Tests if a datatype is group.
"""
_, dt_group_id, first_dt, _ = self._create_datatype_group()
dt_group = dao.get_generic_entity(model.DataTypeGroup, dt_group_id)[0]
is_dt_group = self.project_service.is_datatype_group(dt_group.gid)
self.assertTrue(is_dt_group, "The datatype should be a datatype group.")
is_dt_group = self.project_service.is_datatype_group(first_dt.gid)
self.assertFalse(is_dt_group, "The datatype should not be a datatype group.")
def test_count_datatypes_in_group(self):
""" Test that counting dataTypes is correct. Happy flow."""
_, dt_group_id, first_dt, _ = self._create_datatype_group()
count = dao.count_datatypes_in_group(dt_group_id)
self.assertEqual(count, 2)
count = dao.count_datatypes_in_group(first_dt.id)
self.assertEqual(count, 0, "There should be no dataType.")
def test_set_datatype_visibility(self):
"""
Check if the visibility for a datatype is set correct.
"""
#it's a list of 3 elem.
mapped_arrays = self._create_mapped_arrays(self.test_project.id)
for mapped_array in mapped_arrays:
is_visible = dao.get_datatype_by_id(mapped_array[0]).visible
self.assertTrue(is_visible, "The data type should be visible.")
self.project_service.set_datatype_visibility(mapped_arrays[0][2], False)
for i in range(len(mapped_arrays)):
is_visible = dao.get_datatype_by_id(mapped_arrays[i][0]).visible
if not i:
self.assertFalse(is_visible, "The data type should not be visible.")
else:
self.assertTrue(is_visible, "The data type should be visible.")
def test_set_visibility_for_dt_in_group(self):
"""
Check if the visibility for a datatype from a datatype group is set correct.
"""
_, dt_group_id, first_dt, second_dt = self._create_datatype_group()
self.assertTrue(first_dt.visible, "The data type should be visible.")
self.assertTrue(second_dt.visible, "The data type should be visible.")
self.project_service.set_datatype_visibility(first_dt.gid, False)
db_dt_group = self.project_service.get_datatype_by_id(dt_group_id)
db_first_dt = self.project_service.get_datatype_by_id(first_dt.id)
db_second_dt = self.project_service.get_datatype_by_id(second_dt.id)
self.assertFalse(db_dt_group.visible, "The data type should be visible.")
self.assertFalse(db_first_dt.visible, "The data type should not be visible.")
self.assertFalse(db_second_dt.visible, "The data type should be visible.")
def test_set_visibility_for_group(self):
"""
Check if the visibility for a datatype group is set correct.
"""
_, dt_group_id, first_dt, second_dt = self._create_datatype_group()
dt_group = dao.get_generic_entity(model.DataTypeGroup, dt_group_id)[0]
self.assertTrue(dt_group.visible, "The data type group should be visible.")
self.assertTrue(first_dt.visible, "The data type should be visible.")
self.assertTrue(second_dt.visible, "The data type should be visible.")
self.project_service.set_datatype_visibility(dt_group.gid, False)
updated_dt_group = self.project_service.get_datatype_by_id(dt_group_id)
updated_first_dt = self.project_service.get_datatype_by_id(first_dt.id)
updated_second_dt = self.project_service.get_datatype_by_id(second_dt.id)
self.assertFalse(updated_dt_group.visible, "The data type group should be visible.")
self.assertFalse(updated_first_dt.visible, "The data type should be visible.")
self.assertFalse(updated_second_dt.visible, "The data type should be visible.")
def test_getdatatypes_from_dtgroup(self):
"""
Validate that we can retrieve all DTs from a DT_Group
"""
_, dt_group_id, first_dt, second_dt = self._create_datatype_group()
datatypes = self.project_service.get_datatypes_from_datatype_group(dt_group_id)
self.assertEqual(len(datatypes), 2, "There should be 2 datatypes into the datatype group.")
expected_dict = {first_dt.id: first_dt, second_dt.id: second_dt}
actual_dict = {datatypes[0].id: datatypes[0], datatypes[1].id: datatypes[1]}
for key in expected_dict.keys():
expected = expected_dict[key]
actual = actual_dict[key]
self.assertEqual(expected.id, actual.id, "Not the same id.")
self.assertEqual(expected.gid, actual.gid, "Not the same gid.")
self.assertEqual(expected.type, actual.type, "Not the same type.")
self.assertEqual(expected.subject, actual.subject, "Not the same subject.")
self.assertEqual(expected.state, actual.state, "Not the same state.")
self.assertEqual(expected.visible, actual.visible, "The datatype visibility is not correct.")
self.assertEqual(expected.module, actual.module, "Not the same module.")
self.assertEqual(expected.user_tag_1, actual.user_tag_1, "Not the same user_tag_1.")
self.assertEqual(expected.invalid, actual.invalid, "The invalid field value is not correct.")
self.assertEqual(expected.is_nan, actual.is_nan, "The is_nan field value is not correct.")
def test_get_operations_for_dt(self):
"""
Tests method get_operations_for_datatype.
Verifies result dictionary has the correct values
"""
created_ops, datatype_gid = self._create_operations_with_inputs()
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.relevant_filter)
self.assertEqual(len(operations), 2)
self.assertTrue(created_ops[0].id in [operations[0].id, operations[1].id], "Retrieved wrong operations.")
self.assertTrue(created_ops[2].id in [operations[0].id, operations[1].id], "Retrieved wrong operations.")
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.full_filter)
self.assertEqual(len(operations), 4)
ids = [operations[0].id, operations[1].id, operations[2].id, operations[3].id]
for i in range(4):
self.assertTrue(created_ops[i].id in ids, "Retrieved wrong operations.")
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.relevant_filter, True)
self.assertEqual(len(operations), 1)
self.assertEqual(created_ops[4].id, operations[0].id, "Incorrect number of operations.")
operations = self.project_service.get_operations_for_datatype(datatype_gid, self.full_filter, True)
self.assertEqual(len(operations), 2)
self.assertTrue(created_ops[4].id in [operations[0].id, operations[1].id], "Retrieved wrong operations.")
self.assertTrue(created_ops[5].id in [operations[0].id, operations[1].id], "Retrieved wrong operations.")
def test_get_operations_for_dt_group(self):
"""
Tests method get_operations_for_datatype_group.
Verifies filters' influence over results is as expected
"""
created_ops, dt_group_id = self._create_operations_with_inputs(True)
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.relevant_filter)
self.assertEqual(len(ops), 2)
self.assertTrue(created_ops[0].id in [ops[0].id, ops[1].id], "Retrieved wrong operations.")
self.assertTrue(created_ops[2].id in [ops[0].id, ops[1].id], "Retrieved wrong operations.")
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.full_filter)
self.assertEqual(len(ops), 4, "Incorrect number of operations.")
ids = [ops[0].id, ops[1].id, ops[2].id, ops[3].id]
for i in range(4):
self.assertTrue(created_ops[i].id in ids, "Retrieved wrong operations.")
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.relevant_filter, True)
self.assertEqual(len(ops), 1)
self.assertEqual(created_ops[4].id, ops[0].id, "Incorrect number of operations.")
ops = self.project_service.get_operations_for_datatype_group(dt_group_id, self.full_filter, True)
self.assertEqual(len(ops), 2)
self.assertTrue(created_ops[4].id in [ops[0].id, ops[1].id], "Retrieved wrong operations.")
self.assertTrue(created_ops[5].id in [ops[0].id, ops[1].id], "Retrieved wrong operations.")
def test_get_inputs_for_operation(self):
"""
Tests method get_datatype_and_datatypegroup_inputs_for_operation.
Verifies filters' influence over results is as expected
"""
algo = dao.get_algorithm_by_module('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
array_wrappers = self._create_mapped_arrays(self.test_project.id)
ids = []
for datatype in array_wrappers:
ids.append(datatype[0])
datatype = dao.get_datatype_by_id(ids[0])
datatype.visible = False
dao.store_entity(datatype)
parameters = json.dumps({"param_5": "1", "param_1": array_wrappers[0][2],
"param_2": array_wrappers[1][2], "param_3": array_wrappers[2][2], "param_6": "0"})
operation = model.Operation(self.test_user.id, self.test_project.id, algo.id, parameters)
operation = dao.store_entity(operation)
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.relevant_filter)
self.assertEqual(len(inputs), 2)
self.assertTrue(ids[1] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.")
self.assertTrue(ids[2] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.")
self.assertFalse(ids[0] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.")
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.full_filter)
self.assertEqual(len(inputs), 3, "Incorrect number of operations.")
self.assertTrue(ids[0] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType.")
self.assertTrue(ids[1] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType.")
self.assertTrue(ids[2] in [inputs[0].id, inputs[1].id, inputs[2].id], "Retrieved wrong dataType.")
project, dt_group_id, first_dt, _ = self._create_datatype_group()
first_dt.visible = False
dao.store_entity(first_dt)
parameters = json.dumps({"other_param": "_", "param_1": first_dt.gid})
operation = model.Operation(self.test_user.id, project.id, algo.id, parameters)
operation = dao.store_entity(operation)
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.relevant_filter)
self.assertEqual(len(inputs), 0, "Incorrect number of dataTypes.")
inputs = self.project_service.get_datatype_and_datatypegroup_inputs_for_operation(operation.gid,
self.full_filter)
self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.")
self.assertEqual(inputs[0].id, dt_group_id, "Wrong dataType.")
self.assertTrue(inputs[0].id != first_dt.id, "Wrong dataType.")
def test_get_inputs_for_op_group(self):
"""
Tests method get_datatypes_inputs_for_operation_group.
The DataType inputs will be from a DataType group.
"""
project, dt_group_id, first_dt, second_dt = self._create_datatype_group()
first_dt.visible = False
dao.store_entity(first_dt)
second_dt.visible = False
dao.store_entity(second_dt)
op_group = model.OperationGroup(project.id, "group", "range1[1..2]")
op_group = dao.store_entity(op_group)
params_1 = json.dumps({"param_5": "1", "param_1": first_dt.gid, "param_6": "2"})
params_2 = json.dumps({"param_5": "1", "param_4": second_dt.gid, "param_6": "5"})
algo = dao.get_algorithm_by_module('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
op1 = model.Operation(self.test_user.id, project.id, algo.id, params_1, op_group_id=op_group.id)
op2 = model.Operation(self.test_user.id, project.id, algo.id, params_2, op_group_id=op_group.id)
dao.store_entities([op1, op2])
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter)
self.assertEqual(len(inputs), 0)
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter)
self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.")
self.assertFalse(first_dt.id == inputs[0].id, "Retrieved wrong dataType.")
self.assertFalse(second_dt.id == inputs[0].id, "Retrieved wrong dataType.")
self.assertTrue(dt_group_id == inputs[0].id, "Retrieved wrong dataType.")
first_dt.visible = True
dao.store_entity(first_dt)
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter)
self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.")
self.assertFalse(first_dt.id == inputs[0].id, "Retrieved wrong dataType.")
self.assertFalse(second_dt.id == inputs[0].id, "Retrieved wrong dataType.")
self.assertTrue(dt_group_id == inputs[0].id, "Retrieved wrong dataType.")
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter)
self.assertEqual(len(inputs), 1, "Incorrect number of dataTypes.")
self.assertFalse(first_dt.id == inputs[0].id, "Retrieved wrong dataType.")
self.assertFalse(second_dt.id == inputs[0].id, "Retrieved wrong dataType.")
self.assertTrue(dt_group_id == inputs[0].id, "Retrieved wrong dataType.")
def test_get_inputs_for_op_group_simple_inputs(self):
"""
Tests method get_datatypes_inputs_for_operation_group.
The dataType inputs will not be part of a dataType group.
"""
#it's a list of 3 elem.
array_wrappers = self._create_mapped_arrays(self.test_project.id)
array_wrapper_ids = []
for datatype in array_wrappers:
array_wrapper_ids.append(datatype[0])
datatype = dao.get_datatype_by_id(array_wrapper_ids[0])
datatype.visible = False
dao.store_entity(datatype)
op_group = model.OperationGroup(self.test_project.id, "group", "range1[1..2]")
op_group = dao.store_entity(op_group)
params_1 = json.dumps({"param_5": "2", "param_1": array_wrappers[0][2],
"param_2": array_wrappers[1][2], "param_6": "7"})
params_2 = json.dumps({"param_5": "5", "param_3": array_wrappers[2][2],
"param_2": array_wrappers[1][2], "param_6": "6"})
algo = dao.get_algorithm_by_module('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
op1 = model.Operation(self.test_user.id, self.test_project.id, algo.id, params_1, op_group_id=op_group.id)
op2 = model.Operation(self.test_user.id, self.test_project.id, algo.id, params_2, op_group_id=op_group.id)
dao.store_entities([op1, op2])
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.relevant_filter)
self.assertEqual(len(inputs), 2)
self.assertFalse(array_wrapper_ids[0] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.")
self.assertTrue(array_wrapper_ids[1] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.")
self.assertTrue(array_wrapper_ids[2] in [inputs[0].id, inputs[1].id], "Retrieved wrong dataType.")
inputs = self.project_service.get_datatypes_inputs_for_operation_group(op_group.id, self.full_filter)
self.assertEqual(len(inputs), 3, "Incorrect number of dataTypes.")
self.assertTrue(array_wrapper_ids[0] in [inputs[0].id, inputs[1].id, inputs[2].id])
self.assertTrue(array_wrapper_ids[1] in [inputs[0].id, inputs[1].id, inputs[2].id])
self.assertTrue(array_wrapper_ids[2] in [inputs[0].id, inputs[1].id, inputs[2].id])
def test_remove_datatype(self):
"""
Tests the deletion of a datatype.
"""
#it's a list of 3 elem.
array_wrappers = self._create_mapped_arrays(self.test_project.id)
dt_list = []
for array_wrapper in array_wrappers:
dt_list.append(dao.get_datatype_by_id(array_wrapper[0]))
self.project_service.remove_datatype(self.test_project.id, dt_list[0].gid)
self._check_if_datatype_was_removed(dt_list[0])
def test_remove_datatype_from_group(self):
"""
Tests the deletion of a datatype group.
"""
project, dt_group_id, first_dt, second_dt = self._create_datatype_group()
datatype_group = dao.get_generic_entity(model.DataTypeGroup, dt_group_id)[0]
self.project_service.remove_datatype(project.id, first_dt.gid)
self._check_if_datatype_was_removed(first_dt)
self._check_if_datatype_was_removed(second_dt)
self._check_if_datatype_was_removed(datatype_group)
self._check_datatype_group_removed(dt_group_id, datatype_group.fk_operation_group)
def test_remove_datatype_group(self):
"""
Tests the deletion of a datatype group.
"""
project, dt_group_id, first_dt, second_dt = self._create_datatype_group()
datatype_group = dao.get_generic_entity(model.DataTypeGroup, dt_group_id)[0]
self.project_service.remove_datatype(project.id, datatype_group.gid)
self._check_if_datatype_was_removed(first_dt)
self._check_if_datatype_was_removed(second_dt)
self._check_if_datatype_was_removed(datatype_group)
self._check_datatype_group_removed(dt_group_id, datatype_group.fk_operation_group)
def _create_mapped_arrays(self, project_id):
"""
:param project_id: the project in which the arrays are created
:return: a list of dummy `MappedArray`
"""
count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(count, 0)
group = dao.get_algorithm_by_module('tvb.tests.framework.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter')
adapter_instance = ABCAdapter.build_adapter(group)
data = {'param_1': 'some value'}
#create 3 data types
self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data)
count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(count, 1)
self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data)
count = self.flow_service.get_available_datatypes(project_id, "tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(count, 2)
self.flow_service.fire_operation(adapter_instance, self.test_user, project_id, **data)
array_wrappers, count = self.flow_service.get_available_datatypes(project_id,
"tvb.datatypes.arrays.MappedArray")
self.assertEqual(count, 3)
return array_wrappers
def _create_operation(self, project_id, algorithm_id):
"""
dummy operation
:param project_id: the project in which the operation is created
:param algorithm_id: the algorithm to be run for the operation
:return: a dummy `Operation` with the given specifications
"""
algorithm = dao.get_algorithm_by_id(algorithm_id)
meta = {DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"}
operation = model.Operation(self.test_user.id, project_id, algorithm.id, 'test params',
meta=json.dumps(meta), status=model.STATUS_FINISHED)
return dao.store_entity(operation)
def _create_datatype_group(self):
"""
Creates a project, one DataTypeGroup with 2 DataTypes into the new group.
"""
test_project = TestFactory.create_project(self.test_user, "NewProject")
all_operations = dao.get_filtered_operations(test_project.id, None, is_count=True)
self.assertEqual(0, all_operations, "There should be no operation.")
datatypes, op_group_id = TestFactory.create_group(self.test_user, test_project)
dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id)
return test_project, dt_group.id, datatypes[0], datatypes[1]
def _create_operations_with_inputs(self, is_group_parent=False):
"""
Method used for creating a complex tree of operations.
If 'if_group_parent' is True then a new group will be created and one of its entries it will be used as
input for the returned operations.
"""
group_dts, root_op_group_id = TestFactory.create_group(self.test_user, self.test_project)
if is_group_parent:
datatype_gid = group_dts[0].gid
else:
datatype_gid = ProjectServiceTest._create_value_wrapper(self.test_user, self.test_project)[1]
parameters = json.dumps({"param_name": datatype_gid})
ops = []
for i in range(4):
ops.append(TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project))
if i in [1, 3]:
ops[i].visible = False
ops[i].parameters = parameters
ops[i] = dao.store_entity(ops[i])
#groups
_, ops_group = TestFactory.create_group(self.test_user, self.test_project)
ops_group = dao.get_operations_in_group(ops_group)
self.assertEqual(2, len(ops_group))
ops_group[0].parameters = parameters
ops_group[0] = dao.store_entity(ops_group[0])
ops_group[1].visible = False
ops_group[1].parameters = parameters
ops_group[1] = dao.store_entity(ops_group[1])
ops.extend(ops_group)
if is_group_parent:
dt_group = dao.get_datatypegroup_by_op_group_id(root_op_group_id)
return ops, dt_group.id
return ops, datatype_gid
def _check_if_datatype_was_removed(self, datatype):
"""
Check if a certain datatype was removed.
"""
try:
dao.get_datatype_by_id(datatype.id)
self.fail("The datatype was not deleted.")
except Exception:
pass
try:
dao.get_operation_by_id(datatype.fk_from_operation)
self.fail("The operation was not deleted.")
except Exception:
pass
def _check_datatype_group_removed(self, datatype_group_id, operation_groupp_id):
"""
Checks if the DataTypeGroup and OperationGroup was removed.
"""
try:
dao.get_generic_entity(model.DataTypeGroup, datatype_group_id)
self.fail("The DataTypeGroup entity was not removed.")
except Exception:
pass
try:
dao.get_operationgroup_by_id(operation_groupp_id)
self.fail("The OperationGroup entity was not removed.")
except Exception:
pass
def __init_algorithmn(self):
"""
Insert some starting data in the database.
"""
categ1 = model.AlgorithmCategory('one', True)
self.categ1 = dao.store_entity(categ1)
ad = model.Algorithm(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS, categ1.id)
self.algo_inst = dao.store_entity(ad)
@staticmethod
def _create_algo_for_upload():
""" Creates a fake algorithm for an upload category. """
category = dao.store_entity(model.AlgorithmCategory("upload_category", rawinput=True))
return dao.store_entity(model.Algorithm("module", "classname", category.id))
class FlowServiceTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.flow_service module.
"""
def setUp(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
### Insert some starting data in the database.
categ1 = model.AlgorithmCategory('one', True)
self.categ1 = dao.store_entity(categ1)
categ2 = model.AlgorithmCategory('two', rawinput=True)
self.categ2 = dao.store_entity(categ2)
group1 = model.AlgorithmGroup("test_module1", "classname1", categ1.id)
self.algo_group1 = dao.store_entity(group1)
group2 = model.AlgorithmGroup("test_module2", "classname2", categ2.id)
self.algo_group2 = dao.store_entity(group2)
group3 = model.AlgorithmGroup("test_module3", "classname3", categ1.id)
self.algo_group3 = dao.store_entity(group3)
group_v = model.AlgorithmGroup(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, categ2.id)
self.algo_group_v = dao.store_entity(group_v)
algo_v = model.Algorithm(self.algo_group_v.id,
'ident',
name='',
req_data='',
param_name='',
output='')
self.algorithm_v = dao.store_entity(algo_v)
algo1 = model.Algorithm(self.algo_group1.id,
'id',
name='',
req_data='',
param_name='',
output='')
self.algorithm1 = dao.store_entity(algo1)
def tearDown(self):
for algo in [self.algorithm1, self.algorithm_v]:
dao.remove_entity(model.Algorithm, algo.id)
for group in [
self.algo_group1, self.algo_group2, self.algo_group3,
self.algorithm_v
]:
dao.remove_entity(model.AlgorithmGroup, group.id)
for categ in [self.categ1, self.categ2]:
dao.remove_entity(model.AlgorithmCategory, categ.id)
def test_groups_for_categories(self):
"""
Test getting algorithms for specific categories.
"""
category1 = self.flow_service.get_groups_for_categories([self.categ1])
category2 = self.flow_service.get_groups_for_categories([self.categ2])
dummy = model.AlgorithmCategory('dummy', rawinput=True)
dummy.id = 999
unexisting_cat = self.flow_service.get_groups_for_categories([dummy])
self.assertEqual(len(category1), 2)
self.assertEqual(len(category2), 2)
self.assertEqual(len(unexisting_cat), 0)
for group in category1:
if group.module not in ["test_module1", "test_module3"]:
self.fail("Some invalid data retrieved")
for group in category2:
if group.module not in ["test_module2", TEST_ADAPTER_VALID_MODULE]:
self.fail("Some invalid data retrieved")
def test_get_broup_by_identifier(self):
"""
Test for the get_algorithm_by_identifier.
"""
algo_ret = self.flow_service.get_algo_group_by_identifier(
self.algo_group1.id)
self.assertEqual(algo_ret.id, self.algo_group1.id,
"ID-s are different!")
self.assertEqual(algo_ret.module, self.algo_group1.module,
"Modules are different!")
self.assertEqual(algo_ret.fk_category, self.algo_group1.fk_category,
"Categories are different!")
self.assertEqual(algo_ret.classname, self.algo_group1.classname,
"Class names are different!")
def test_build_adapter_instance(self):
"""
Test standard flow for building an adapter instance.
"""
algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS)
adapter = ABCAdapter.build_adapter(algo_group)
self.assertTrue(isinstance(adapter, ABCSynchronous),
"Something went wrong with valid data!")
def test_build_adapter_invalid(self):
"""
Test flow for trying to build an adapter that does not inherit from ABCAdapter.
"""
group = dao.find_group(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_INVALID_CLASS)
self.assertRaises(OperationException,
self.flow_service.build_adapter_instance, group)
def test_prepare_adapter(self):
"""
Test preparation of an adapter.
"""
algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS)
group, interface = self.flow_service.prepare_adapter(
self.test_project.id, algo_group)
self.assertTrue(isinstance(group, model.AlgorithmGroup),
"Something went wrong with valid data!")
self.assertTrue("name" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["name"], "test", "Bad interface!")
self.assertTrue("type" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["type"], "int", "Bad interface!")
self.assertTrue("default" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["default"], "0", "Bad interface!")
def test_fire_operation(self):
"""
Test preparation of an adapter and launch mechanism.
"""
algo_group = dao.find_group(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS)
adapter = self.flow_service.build_adapter_instance(algo_group)
data = {"test": 5}
result = self.flow_service.fire_operation(adapter, self.test_user,
self.test_project.id, **data)
self.assertTrue(result.endswith("has finished."), "Operation fail")
def test_get_filtered_by_column(self):
"""
Test the filter function when retrieving dataTypes with a filter
after a column from a class specific table (e.g. DATA_arraywrapper).
"""
operation_1 = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
operation_2 = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
one_dim_array = numpy.arange(5)
two_dim_array = numpy.array([[1, 2], [2, 3], [1, 4]])
self._store_float_array(one_dim_array, "John Doe 1", operation_1.id)
self._store_float_array(one_dim_array, "John Doe 2", operation_1.id)
self._store_float_array(two_dim_array, "John Doe 3", operation_2.id)
count = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(count, 3, "Problems with inserting data")
first_filter = FilterChain(
fields=[FilterChain.datatype + '._nr_dimensions'],
operations=["=="],
values=[1])
count = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray",
first_filter)[1]
self.assertEqual(count, 2, "Data was not filtered")
second_filter = FilterChain(
fields=[FilterChain.datatype + '._nr_dimensions'],
operations=["=="],
values=[2])
filtered_data = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray",
second_filter)[0]
self.assertEqual(len(filtered_data), 1, "Data was not filtered")
self.assertEqual(filtered_data[0][3], "John Doe 3")
third_filter = FilterChain(
fields=[FilterChain.datatype + '._length_1d'],
operations=["=="],
values=[3])
filtered_data = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray",
third_filter)[0]
self.assertEqual(len(filtered_data), 1,
"Data was not filtered correct")
self.assertEqual(filtered_data[0][3], "John Doe 3")
try:
if os.path.exists('One_dim.txt'):
os.remove('One_dim.txt')
if os.path.exists('Two_dim.txt'):
os.remove('Two_dim.txt')
if os.path.exists('One_dim-1.txt'):
os.remove('One_dim-1.txt')
except Exception:
pass
@staticmethod
def _store_float_array(array_data, subject_name, operation_id):
"""Create Float Array and DB persist it"""
datatype_inst = MappedArray(user_tag_1=subject_name)
datatype_inst.set_operation_id(operation_id)
datatype_inst.array_data = array_data
datatype_inst.type = "MappedArray"
datatype_inst.module = "tvb.datatypes.arrays"
datatype_inst.subject = subject_name
datatype_inst.state = "RAW"
dao.store_entity(datatype_inst)
def test_get_filtered_datatypes(self):
"""
Test the filter function when retrieving dataTypes.
"""
#Create some test operations
start_dates = [
datetime.now(),
datetime.strptime("08-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2010", "%m-%d-%Y"),
datetime.strptime("05-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2011", "%m-%d-%Y")
]
end_dates = [
datetime.now(),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y")
]
for i in range(5):
operation = model.Operation(self.test_user.id,
self.test_project.id,
self.algorithm1.id,
'test params',
status=model.STATUS_FINISHED,
start_date=start_dates[i],
completion_date=end_dates[i])
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(
self.test_project, str(operation.id))
if i < 4:
datatype_inst = Datatype1()
datatype_inst.type = "Datatype1"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
else:
for _ in range(2):
datatype_inst = Datatype2()
datatype_inst.storage_path = storage_path
datatype_inst.type = "Datatype2"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.string_data = ["data"]
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
returned_data = self.flow_service.get_available_datatypes(
self.test_project.id, Datatype1)[0]
for row in returned_data:
if row[1] != 'Datatype1':
self.fail("Some invalid data was returned!")
self.assertEqual(4, len(returned_data), "Invalid length of result")
filter_op = FilterChain(
fields=[
FilterChain.datatype + ".state",
FilterChain.operation + ".start_date"
],
values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")],
operations=["==", ">"])
returned_data = self.flow_service.get_available_datatypes(
self.test_project.id, Datatype1, filter_op)[0]
returned_subjects = [one_data[3] for one_data in returned_data]
if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len(
returned_subjects) != 2:
self.fail("DataTypes were not filtered properly!")
adapter_instance = ABCAdapter.build_adapter_from_class(SimulatorAdapter)
## Prepare the input algorithms as if they were coming from web UI submit:
## TODO create helper methods for working with objects instead of strings.
connectivity = dao.get_generic_entity(Connectivity, DataTypeMetaData.DEFAULT_SUBJECT, "subject")[0]
launch_args = dict()
for f in adapter_instance.flaten_input_interface():
launch_args[f["name"]] = str(f["default"]) if 'default' in f else None
launch_args["connectivity"] = connectivity.gid
launch_args["model_parameters_option_Generic2dOscillator_variables_of_interest"] = 'V'
if len(sys.argv) > 1:
launch_args["model_parameters_option_Generic2dOscillator_tau"] = sys.argv[1]
## launch an operation and have the results stored both in DB and on disk
launched_operation = flow_service.fire_operation(adapter_instance, project.administrator,
project.id, **launch_args)[0]
## wait for the operation to finish
while not launched_operation.has_finished:
sleep(5)
launched_operation = dao.get_operation_by_id(launched_operation.id)
if launched_operation.status == model.STATUS_FINISHED:
ts = dao.get_generic_entity(TimeSeriesRegion, launched_operation.id, "fk_from_operation")[0]
LOG.info("TimeSeries result is: %s " % ts.summary_info)
print ts.summary_info
else:
LOG.warning("Operation ended with problems [%s]: [%s]" % (launched_operation.status,
launched_operation.additional_info))
class ModelValidator(object):
overwrites = {}
def __init__(self, overwrites=None, settings_file=None):
""" Parameters can be overwritten either from a settigns file or from a dictionary. """
if overwrites is not None:
self.overwrites.update(overwrites)
if settings_file is not None:
settings = open(sys.argv[1]).read()
for line in settings.split('\n'):
key, value = line.split('=')
self.overwrites[key.strip()] = value.strip()
if KEY_PROJECT not in self.overwrites:
raise Exception(
"Settings file should contain the id of the project: %s=1" %
KEY_PROJECT)
self.project = dao.get_project_by_id(self.overwrites[KEY_PROJECT])
self.flow_service = FlowService()
self.operation_service = OperationService()
def launch_validation(self):
"""
Prepare the arguments to be submitted and launch actual operations group.
TODO: Now get the results and check if any errors
"""
stored_adapter = self.flow_service.get_algorithm_by_module_and_class(
SIMULATOR_MODULE, SIMULATOR_CLASS)
simulator_adapter = ABCAdapter.build_adapter(stored_adapter)
launch_args = {}
flatten_interface = simulator_adapter.flaten_input_interface()
itree_mngr = self.flow_service.input_tree_manager
prepared_flatten_interface = itree_mngr.fill_input_tree_with_options(
flatten_interface, self.project.id, stored_adapter.fk_category)
for entry in prepared_flatten_interface:
value = entry['default']
if isinstance(value, dict):
value = str(value)
if hasattr(value, 'tolist'):
value = value.tolist()
launch_args[entry['name']] = value
launch_args.update(self.overwrites)
nr_of_operations = 1
for key in self.overwrites:
if key.startswith(PARAM_RANGE_PREFIX):
range_values = self.operation_service.get_range_values(
launch_args, key)
nr_of_operations *= len(range_values)
do_launch = False
print "Warning! This will launch %s operations. Do you agree? (yes/no)" % nr_of_operations
while 1:
accept = raw_input()
if accept.lower() == 'yes':
do_launch = True
break
if accept.lower() == 'no':
do_launch = False
break
print "Please type either yes or no"
if do_launch:
self.launched_operations = self.flow_service.fire_operation(
simulator_adapter, self.project.administrator, self.project.id,
**launch_args)
return self.validate_results(0)
else:
return "Operation canceled by user."
def validate_results(self, last_verified_index):
error_count = 0
while last_verified_index < len(self.launched_operations):
operation_to_check = self.launched_operations[last_verified_index]
operation = dao.get_operation_by_id(operation_to_check.id)
if not operation.has_finished:
sleep(10)
if operation.status == STATUS_ERROR:
sys.stdout.write("E(" + str(operation_to_check.id) + ")")
error_count += 1
last_verified_index += 1
sys.stdout.flush()
if operation.status == STATUS_FINISHED:
last_verified_index += 1
sys.stdout.write('.')
sys.stdout.flush()
if error_count:
return "%s operations in error; %s operations successfully." % (
error_count, len(self.launched_operations) - error_count)
return "All operations finished successfully!"
project = dao.get_project_by_id(1)
## Prepare the Adapter
adapter_instance = ABCAdapter.prepare_adapter(FourierAdapter)
## Prepare the input algorithms as if they were coming from web UI submit:
time_series = dao.get_generic_entity(TimeSeriesRegion,
DataTypeMetaData.DEFAULT_SUBJECT,
"subject")
if len(time_series) < 1:
LOG.error("We could not find a compatible TimeSeries Datatype!")
launch_args = {"time_series": time_series[0].gid}
## launch an operation and have the results stored both in DB and on disk
launched_operation = flow_service.fire_operation(adapter_instance,
project.administrator,
project.id,
**launch_args)[0]
## wait for the operation to finish
while not launched_operation.has_finished:
sleep(5)
launched_operation = dao.get_operation_by_id(launched_operation.id)
if launched_operation.status == model.STATUS_FINISHED:
fourier_spectrum = dao.get_generic_entity(FourierSpectrum,
launched_operation.id,
"fk_from_operation")[0]
LOG.info("Fourier Spectrum result is: %s " %
fourier_spectrum.summary_info)
## print fourier_spectrum.summary_info
else:
class TestFlowService(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.flow_service module.
"""
def transactional_setup_method(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
category = dao.get_uploader_categories()[0]
self.algorithm = dao.store_entity(model_operation.Algorithm(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, category.id))
def transactional_teardown_method(self):
dao.remove_entity(model_operation.Algorithm, self.algorithm)
def test_get_uploaders(self):
result = self.flow_service.get_upload_algorithms()
# Not sure if it is correct but I think there are not 29 algorithms anymore here
assert 19 == len(result)
found = False
for algo in result:
if algo.classname == self.algorithm.classname and algo.module == self.algorithm.module:
found = True
break
assert found, "Uploader incorrectly returned"
def test_get_analyze_groups(self):
category, groups = self.flow_service.get_analyze_groups()
assert category.displayname == 'Analyze'
assert len(groups) > 1
assert isinstance(groups[0], model_operation.AlgorithmTransientGroup)
def test_get_visualizers_for_group(self, datatype_group_factory):
group = datatype_group_factory()
dt_group = dao.get_datatypegroup_by_op_group_id(group.fk_from_operation)
result = self.flow_service.get_visualizers_for_group(dt_group.gid)
# Both discrete and isocline are expected due to the 2 ranges set in the factory
assert 2 == len(result)
result_classnames = [res.classname for res in result]
assert IntrospectionRegistry.ISOCLINE_PSE_ADAPTER_CLASS in result_classnames
assert IntrospectionRegistry.DISCRETE_PSE_ADAPTER_CLASS in result_classnames
def test_get_launchable_algorithms(self, time_series_region_index_factory, connectivity_factory,
region_mapping_factory):
conn = connectivity_factory()
rm = region_mapping_factory()
ts = time_series_region_index_factory(connectivity=conn, region_mapping=rm)
result, has_operations_warning = self.flow_service.get_launchable_algorithms(ts.gid)
assert 'Analyze' in result
assert 'View' in result
assert has_operations_warning is False
def test_get_group_by_identifier(self):
"""
Test for the get_algorithm_by_identifier.
"""
algo_ret = self.flow_service.get_algorithm_by_identifier(self.algorithm.id)
assert algo_ret.id == self.algorithm.id, "ID-s are different!"
assert algo_ret.module == self.algorithm.module, "Modules are different!"
assert algo_ret.fk_category == self.algorithm.fk_category, "Categories are different!"
assert algo_ret.classname == self.algorithm.classname, "Class names are different!"
def test_build_adapter_invalid(self):
"""
Test flow for trying to build an adapter that does not inherit from ABCAdapter.
"""
group = dao.get_algorithm_by_module(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_INVALID_CLASS)
with pytest.raises(IntrospectionException):
ABCAdapter.build_adapter(group)
def test_prepare_adapter(self):
"""
Test preparation of an adapter.
"""
stored_adapter = dao.get_algorithm_by_module(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
assert isinstance(stored_adapter, model_operation.Algorithm), "Something went wrong with valid data!"
adapter = self.flow_service.prepare_adapter(stored_adapter)
assert isinstance(adapter, TestAdapter1), "Adapter incorrectly built"
assert adapter.get_form_class() == TestAdapter1Form
assert adapter.get_view_model() == TestModel
def test_fire_operation(self):
"""
Test preparation of an adapter and launch mechanism.
"""
adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
test_user = TestFactory.create_user()
test_project = TestFactory.create_project(admin=test_user)
result = self.flow_service.fire_operation(adapter, test_user, test_project.id,
view_model=adapter.get_view_model()())
assert result.endswith("has finished."), "Operation fail"
class FlowServiceTest(TransactionalTestCase):
"""
This class contains tests for the tvb.core.services.flow_service module.
"""
def setUp(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
category = dao.get_uploader_categories()[0]
self.algorithm = dao.store_entity(model.Algorithm(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, category.id))
def tearDown(self):
dao.remove_entity(model.Algorithm, self.algorithm)
def test_get_uploaders(self):
result = self.flow_service.get_upload_algorithms()
self.assertEqual(29, len(result))
found = False
for algo in result:
if algo.classname == self.algorithm.classname and algo.module == self.algorithm.module:
found = True
break
self.assertTrue(found, "Uploader incorrectly returned")
def test_get_analyze_groups(self):
category, groups = self.flow_service.get_analyze_groups()
self.assertEqual(category.displayname, 'Analyze')
self.assertTrue(len(groups) > 1)
self.assertTrue(isinstance(groups[0], model.AlgorithmTransientGroup))
def test_get_visualizers_for_group(self):
_, op_group_id = TestFactory.create_group(self.test_user, self.test_project)
dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id)
result = self.flow_service.get_visualizers_for_group(dt_group.gid)
# Only the discreet is expected
self.assertEqual(1, len(result))
self.assertEqual(DISCRETE_PSE_ADAPTER_CLASS, result[0].classname)
def test_get_launchable_algorithms(self):
factory = DatatypesFactory()
conn = factory.create_connectivity(4)[1]
ts = factory.create_timeseries(conn)
result = self.flow_service.get_launchable_algorithms(ts.gid)
self.assertTrue('Analyze' in result)
self.assertTrue('View' in result)
def test_get_roup_by_identifier(self):
"""
Test for the get_algorithm_by_identifier.
"""
algo_ret = self.flow_service.get_algorithm_by_identifier(self.algorithm.id)
self.assertEqual(algo_ret.id, self.algorithm.id, "ID-s are different!")
self.assertEqual(algo_ret.module, self.algorithm.module, "Modules are different!")
self.assertEqual(algo_ret.fk_category, self.algorithm.fk_category, "Categories are different!")
self.assertEqual(algo_ret.classname, self.algorithm.classname, "Class names are different!")
def test_build_adapter_instance(self):
"""
Test standard flow for building an adapter instance.
"""
adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
self.assertTrue(isinstance(adapter, ABCSynchronous), "Something went wrong with valid data!")
def test_build_adapter_invalid(self):
"""
Test flow for trying to build an adapter that does not inherit from ABCAdapter.
"""
group = dao.get_algorithm_by_module(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_INVALID_CLASS)
self.assertRaises(IntrospectionException, ABCAdapter.build_adapter, group)
def test_prepare_adapter(self):
"""
Test preparation of an adapter.
"""
stored_adapter = dao.get_algorithm_by_module(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
interface = self.flow_service.prepare_adapter(self.test_project.id, stored_adapter)
self.assertTrue(isinstance(stored_adapter, model.Algorithm), "Something went wrong with valid data!")
self.assertTrue("name" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["name"], "test", "Bad interface!")
self.assertTrue("type" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["type"], "int", "Bad interface!")
self.assertTrue("default" in interface[0], "Bad interface created!")
self.assertEquals(interface[0]["default"], "0", "Bad interface!")
def test_fire_operation(self):
"""
Test preparation of an adapter and launch mechanism.
"""
adapter = TestFactory.create_adapter(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS)
data = {"test": 5}
result = self.flow_service.fire_operation(adapter, self.test_user, self.test_project.id, **data)
self.assertTrue(result.endswith("has finished."), "Operation fail")
def test_get_filtered_by_column(self):
"""
Test the filter function when retrieving dataTypes with a filter
after a column from a class specific table (e.g. DATA_arraywrapper).
"""
operation_1 = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
operation_2 = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
one_dim_array = numpy.arange(5)
two_dim_array = numpy.array([[1, 2], [2, 3], [1, 4]])
self._store_float_array(one_dim_array, "John Doe 1", operation_1.id)
self._store_float_array(one_dim_array, "John Doe 2", operation_1.id)
self._store_float_array(two_dim_array, "John Doe 3", operation_2.id)
count = self.flow_service.get_available_datatypes(self.test_project.id, "tvb.datatypes.arrays.MappedArray")[1]
self.assertEqual(count, 3, "Problems with inserting data")
first_filter = FilterChain(fields=[FilterChain.datatype + '._nr_dimensions'], operations=["=="], values=[1])
count = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray", first_filter)[1]
self.assertEqual(count, 2, "Data was not filtered")
second_filter = FilterChain(fields=[FilterChain.datatype + '._nr_dimensions'], operations=["=="], values=[2])
filtered_data = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray", second_filter)[0]
self.assertEqual(len(filtered_data), 1, "Data was not filtered")
self.assertEqual(filtered_data[0][3], "John Doe 3")
third_filter = FilterChain(fields=[FilterChain.datatype + '._length_1d'], operations=["=="], values=[3])
filtered_data = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb.datatypes.arrays.MappedArray", third_filter)[0]
self.assertEqual(len(filtered_data), 1, "Data was not filtered correct")
self.assertEqual(filtered_data[0][3], "John Doe 3")
try:
if os.path.exists('One_dim.txt'):
os.remove('One_dim.txt')
if os.path.exists('Two_dim.txt'):
os.remove('Two_dim.txt')
if os.path.exists('One_dim-1.txt'):
os.remove('One_dim-1.txt')
except Exception:
pass
@staticmethod
def _store_float_array(array_data, subject_name, operation_id):
"""Create Float Array and DB persist it"""
datatype_inst = MappedArray(user_tag_1=subject_name)
datatype_inst.set_operation_id(operation_id)
datatype_inst.array_data = array_data
datatype_inst.type = "MappedArray"
datatype_inst.module = "tvb.datatypes.arrays"
datatype_inst.subject = subject_name
datatype_inst.state = "RAW"
dao.store_entity(datatype_inst)
def test_get_filtered_datatypes(self):
"""
Test the filter function when retrieving dataTypes.
"""
#Create some test operations
start_dates = [datetime.now(),
datetime.strptime("08-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2010", "%m-%d-%Y"),
datetime.strptime("05-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2011", "%m-%d-%Y")]
end_dates = [datetime.now(),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y")]
for i in range(5):
operation = model.Operation(self.test_user.id, self.test_project.id, self.algorithm.id, 'test params',
status=model.STATUS_FINISHED, start_date=start_dates[i],
completion_date=end_dates[i])
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(self.test_project, str(operation.id))
if i < 4:
datatype_inst = Datatype1()
datatype_inst.type = "Datatype1"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
else:
for _ in range(2):
datatype_inst = Datatype2()
datatype_inst.storage_path = storage_path
datatype_inst.type = "Datatype2"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.string_data = ["data"]
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
returned_data = self.flow_service.get_available_datatypes(self.test_project.id, Datatype1)[0]
for row in returned_data:
if row[1] != 'Datatype1':
self.fail("Some invalid data was returned!")
self.assertEqual(4, len(returned_data), "Invalid length of result")
filter_op = FilterChain(fields=[FilterChain.datatype + ".state", FilterChain.operation + ".start_date"],
values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")], operations=["==", ">"])
returned_data = self.flow_service.get_available_datatypes(self.test_project.id, Datatype1, filter_op)[0]
returned_subjects = [one_data[3] for one_data in returned_data]
if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len(returned_subjects) != 2:
self.fail("DataTypes were not filtered properly!")
