def test_launch_operation_hdd_full_space_started_ops(
self, test_adapter_factory):
"""
Test the actual operation flow by executing a test adapter.
"""
test_adapter_factory(adapter_class=TestAdapterHDDRequired)
space_taken_by_started = 100
adapter = TestFactory.create_adapter(
"tvb.tests.framework.adapters.testadapter3",
"TestAdapterHDDRequired")
form = TestAdapterHDDRequiredForm()
adapter.submit_form(form)
started_operation = model_operation.Operation(
self.test_user.id,
self.test_project.id,
adapter.stored_adapter.id,
"",
status=model_operation.STATUS_STARTED,
estimated_disk_size=space_taken_by_started)
view_model = adapter.get_view_model()()
dao.store_entity(started_operation)
TvbProfile.current.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(view_model) +
space_taken_by_started - 1)
with pytest.raises(NoMemoryAvailableException):
self.operation_service.initiate_operation(self.test_user,
self.test_project,
adapter,
model_view=view_model)
self._assert_no_ddti()
def _export_linked_datatypes(self, project, zip_file):
linked_paths = ProjectService().get_linked_datatypes_storage_path(
project)
if not linked_paths:
# do not export an empty operation
return
# Make a import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
op = model_operation.Operation(None, None, project.id, algo.id)
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model_operation.STATUS_FINISHED)
op_folder = self.files_helper.get_operation_folder(
op.project.name, op.id)
op_folder_name = os.path.basename(op_folder)
# add linked datatypes to archive in the import operation
for pth in linked_paths:
zip_pth = op_folder_name + '/' + os.path.basename(pth)
zip_file.write(pth, zip_pth)
# remove these files, since we only want them in export archive
self.files_helper.remove_folder(op_folder)
def test_launch_operation_hdd_with_space_started_ops(
self, test_adapter_factory):
"""
Test the actual operation flow by executing a test adapter.
"""
test_adapter_factory(adapter_class=TestAdapterHDDRequired)
space_taken_by_started = 100
adapter = TestFactory.create_adapter(
"tvb.tests.framework.adapters.testadapter3",
"TestAdapterHDDRequired")
form = TestAdapterHDDRequiredForm()
form.fill_from_post({'_test': "100"})
adapter.submit_form(form)
started_operation = model_operation.Operation(
self.test_user.id,
self.test_project.id,
adapter.stored_adapter.id,
"",
status=model_operation.STATUS_STARTED,
estimated_disk_size=space_taken_by_started)
dao.store_entity(started_operation)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(**data) + space_taken_by_started)
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self.operation_service.initiate_operation(self.test_user,
self.test_project, adapter,
tmp_folder, **data)
self._assert_stored_dt2()
def test_get_filtered_datatypes(self, dummy_datatype_index_factory):
"""
Test the filter function when retrieving dataTypes.
"""
#Create some test operations
start_dates = [datetime.now(),
datetime.strptime("08-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2010", "%m-%d-%Y"),
datetime.strptime("05-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2011", "%m-%d-%Y")]
end_dates = [datetime.now(),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y")]
for i in range(5):
operation = model_operation.Operation(self.test_user.id, self.test_project.id, self.algorithm.id, 'test params',
status=model_operation.STATUS_FINISHED, start_date=start_dates[i],
completion_date=end_dates[i])
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(self.test_project, str(operation.id))
if i < 4:
datatype_inst = dummy_datatype_index_factory()
datatype_inst.type = "DummyDataTypeIndex"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.fk_from_operation = operation.id
dao.store_entity(datatype_inst)
else:
for _ in range(2):
datatype_inst = dummy_datatype_index_factory()
datatype_inst.storage_path = storage_path
datatype_inst.type = "DummyDataTypeIndex"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.string_data = ["data"]
datatype_inst.fk_from_operation = operation.id
dao.store_entity(datatype_inst)
returned_data = self.flow_service.get_available_datatypes(self.test_project.id, DummyDataTypeIndex)[0]
for row in returned_data:
if row[1] != 'DummyDataTypeIndex':
raise AssertionError("Some invalid data was returned!")
assert 4 == len(returned_data), "Invalid length of result"
filter_op = FilterChain(fields=[FilterChain.datatype + ".state", FilterChain.operation + ".start_date"],
values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")], operations=["==", ">"])
returned_data = self.flow_service.get_available_datatypes(self.test_project.id, Datatype1, filter_op)[0]
returned_subjects = [one_data[3] for one_data in returned_data]
if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len(returned_subjects) != 2:
raise AssertionError("DataTypes were not filtered properly!")
def _export_linked_datatypes(self, project):
linked_paths = ProjectService().get_linked_datatypes_storage_path(project)
if not linked_paths:
# do not export an empty operation
return None, None
# Make an import operation which will contain links to other projects
algo = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE, TVB_IMPORTER_CLASS)
op = model_operation.Operation(None, None, project.id, algo.id)
op.project = project
op.algorithm = algo
op.id = 'links-to-external-projects'
op.start_now()
op.mark_complete(model_operation.STATUS_FINISHED)
return linked_paths, op
def test_adapter_launch(self, connectivity_factory, region_mapping_factory,
time_series_region_index_factory):
"""
Test that the adapters launches and successfully generates a datatype measure entry.
"""
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"
}
algo = FlowService().get_algorithm_by_module_and_class(
IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS)
self.operation = model_operation.Operation(
self.test_user.id,
self.test_project.id,
algo.id,
json.dumps(''),
meta=json.dumps(meta),
status=model_operation.STATUS_STARTED)
self.operation = dao.store_entity(self.operation)
# Get connectivity, region_mapping and a dummy time_series_region
connectivity = connectivity_factory()
region_mapping = region_mapping_factory()
dummy_time_series_index = time_series_region_index_factory(
connectivity=connectivity, region_mapping=region_mapping)
dummy_time_series_index.start_time = 0.0
dummy_time_series_index.sample_period = 1.0
dummy_time_series_index = \
dao.get_generic_entity(dummy_time_series_index.__class__, dummy_time_series_index.gid, 'gid')[0]
ts_metric_adapter = TimeseriesMetricsAdapter()
form = TimeseriesMetricsAdapterForm()
view_model = form.get_view_model()()
view_model.time_series = UUID(dummy_time_series_index.gid)
form.fill_trait(view_model)
ts_metric_adapter.submit_form(form)
resulted_metric = ts_metric_adapter.launch(view_model)
assert isinstance(
resulted_metric,
DatatypeMeasureIndex), "Result should be a datatype measure."
assert len(resulted_metric.metrics) >= len(list(ts_metric_adapter.get_form().algorithms.choices)), \
"At least a result should have been generated for every metric."
for metric_value in json.loads(resulted_metric.metrics).values():
assert isinstance(metric_value, (float, int))
def test_write_operation_metadata(self):
"""
Test that a correct XML is created for an operation.
"""
operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
expected_file = self.files_helper.get_operation_meta_file_path(self.PROJECT_NAME, operation.id)
assert not os.path.exists(expected_file)
self.files_helper.write_operation_metadata(operation)
assert os.path.exists(expected_file)
operation_meta = XMLReader(expected_file).read_metadata()
loaded_operation = model_operation.Operation(None, None, None, None)
loaded_operation.from_dict(operation_meta, dao, user_id=self.test_user.id)
expected_dict = operation.to_dict()[1]
found_dict = loaded_operation.to_dict()[1]
for key, value in expected_dict.items():
assert str(value) == str(found_dict[key])
# Now validate that operation metaData can be also updated
assert "new_group_name" != found_dict['user_group']
self.files_helper.update_operation_metadata(self.PROJECT_NAME, "new_group_name", operation.id)
found_dict = XMLReader(expected_file).read_metadata()
assert "new_group_name" == found_dict['user_group']
def test_adapter_huge_memory_requirement(self, test_adapter_factory):
"""
Test that an MemoryException is raised in case adapter cant launch due to lack of memory.
"""
# Prepare adapter
test_adapter_factory(adapter_class=TestAdapterHugeMemoryRequired)
adapter = TestFactory.create_adapter(
"tvb.tests.framework.adapters.testadapter3",
"TestAdapterHugeMemoryRequired")
# Simulate receiving POST data
form = TestAdapterHugeMemoryRequiredForm()
adapter.submit_form(form)
view_model = form.get_view_model()()
view_model.test = 5
# Prepare operation for launch
operation = model_operation.Operation(
self.test_user.id,
self.test_project.id,
adapter.stored_adapter.id,
json.dumps({'gid': view_model.gid.hex}),
json.dumps({}),
status=model_operation.STATUS_STARTED)
operation = dao.store_entity(operation)
# Store ViewModel in H5
parent_folder = FilesHelper().get_project_folder(
self.test_project, str(operation.id))
view_model_path = os.path.join(
parent_folder,
h5.path_for(parent_folder, ViewModelH5, view_model.gid))
with ViewModelH5(view_model_path, view_model) as view_model_h5:
view_model_h5.store(view_model)
# Launch operation
with pytest.raises(NoMemoryAvailableException):
OperationService().initiate_prelaunch(operation, adapter)
def test_adapter_huge_memory_requirement(self, test_adapter_factory):
"""
Test that an MemoryException is raised in case adapter cant launch due to lack of memory.
"""
test_adapter_factory(adapter_class=TestAdapterHugeMemoryRequired)
adapter = TestFactory.create_adapter(
"tvb.tests.framework.adapters.testadapter3",
"TestAdapterHugeMemoryRequired")
form = TestAdapterHugeMemoryRequiredForm()
adapter.submit_form(form)
data = {"test": 5}
operation = model_operation.Operation(
self.test_user.id,
self.test_project.id,
adapter.stored_adapter.id,
json.dumps(data),
json.dumps({}),
status=model_operation.STATUS_STARTED)
operation = dao.store_entity(operation)
with pytest.raises(NoMemoryAvailableException):
OperationService().initiate_prelaunch(operation, adapter)
def test_reduce_dimension_component(self):
"""
This method tests if the data passed to the launch method of
the NDimensionArrayAdapter adapter is correct. The passed data should be a list
of arrays with one dimension.
"""
inserted_count = FlowService().get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray")[1]
assert inserted_count == 0, "Expected to find no data."
#create an operation
algorithm_id = FlowService().get_algorithm_by_module_and_class(
'tvb.tests.framework.adapters.ndimensionarrayadapter',
'NDimensionArrayAdapter').id
operation = model_operation.Operation(
self.test_user.id,
self.test_project.id,
algorithm_id,
'test params',
meta=json.dumps({DataTypeMetaData.KEY_STATE: "RAW_DATA"}),
status=model_operation.STATUS_FINISHED)
operation = dao.store_entity(operation)
#save the array wrapper in DB
adapter_instance = NDimensionArrayAdapter()
PARAMS = {}
self.operation_service.initiate_prelaunch(operation, adapter_instance,
{}, **PARAMS)
inserted_data = FlowService().get_available_datatypes(
self.test_project.id, "tvb.datatypes.arrays.MappedArray")[0]
assert len(inserted_data) == 1, "Problems when inserting data"
gid = inserted_data[0][2]
entity = dao.get_datatype_by_gid(gid)
#from the 3D array do not select any array
PARAMS = {
"python_method": "reduce_dimension",
"input_data": gid,
"input_data_dimensions_0": "requiredDim_1",
"input_data_dimensions_1": "",
"input_data_dimensions_2": ""
}
try:
self.operation_service.initiate_prelaunch(operation,
adapter_instance, {},
**PARAMS)
raise AssertionError(
"Test should not pass. The resulted array should be a 1D array."
)
except Exception:
# OK, do nothing; we were expecting to produce a 1D array
pass
#from the 3D array select only a 1D array
first_dim = [gid + '_1_0', 'requiredDim_1']
PARAMS = {
"python_method": "reduce_dimension",
"input_data": gid,
"input_data_dimensions_0": first_dim,
"input_data_dimensions_1": gid + "_2_1"
}
self.operation_service.initiate_prelaunch(operation, adapter_instance,
{}, **PARAMS)
expected_result = entity.array_data[:, 0, 1]
actual_result = adapter_instance.launch_param
assert len(actual_result) == len(
expected_result), "Not the same size for results!"
assert numpy.equal(actual_result, expected_result).all()
#from the 3D array select a 2D array
first_dim = [gid + '_1_0', gid + '_1_1', 'requiredDim_2']
PARAMS = {
"python_method": "reduce_dimension",
"input_data": gid,
"input_data_dimensions_0": first_dim,
"input_data_dimensions_1": gid + "_2_1"
}
self.operation_service.initiate_prelaunch(operation, adapter_instance,
{}, **PARAMS)
expected_result = entity.array_data[slice(0, None), [0, 1], 1]
actual_result = adapter_instance.launch_param
assert len(actual_result) == len(
expected_result), "Not the same size for results!"
assert numpy.equal(actual_result, expected_result).all()
#from 3D array select 1D array by applying SUM function on the first
#dimension and average function on the second dimension
PARAMS = {
"python_method": "reduce_dimension",
"input_data": gid,
"input_data_dimensions_0": ["requiredDim_1", "func_sum"],
"input_data_dimensions_1": "func_average",
"input_data_dimensions_2": ""
}
self.operation_service.initiate_prelaunch(operation, adapter_instance,
{}, **PARAMS)
aux = numpy.sum(entity.array_data, axis=0)
expected_result = numpy.average(aux, axis=0)
actual_result = adapter_instance.launch_param
assert len(actual_result) == len(
expected_result), "Not the same size of results!"
assert numpy.equal(actual_result, expected_result).all()
#from 3D array select a 2D array and apply op. on the second dimension
PARAMS = {
"python_method":
"reduce_dimension",
"input_data":
gid,
"input_data_dimensions_0": [
"requiredDim_2", "func_sum", "expected_shape_x,512",
"operations_x,>"
],
"input_data_dimensions_1":
"",
"input_data_dimensions_2":
""
}
try:
self.operation_service.initiate_prelaunch(operation,
adapter_instance, {},
**PARAMS)
raise AssertionError(
"Test should not pass! The second dimension of the array should be >512."
)
except Exception:
# OK, do nothing;
pass
