def _process_input_zip(self, zip_arch, result_folder, remote_prefix,
file_name_base, expected_pairs, fix_number=True):
"""
Read entries in uploaded ZIP.
Raise Exception in case pairs HDR/IMG are not matched or number "expected_pairs" is not met.
:return string with HDR list (to be passed to DTI pipeline).
"""
hdr_files = []
for file_name in zip_arch.namelist():
if not file_name.startswith(file_name_base) or file_name.endswith("/"):
continue
if file_name.endswith(".hdr"):
pair_img = file_name.replace(".hdr", ".img")
if pair_img not in zip_arch.namelist():
raise ConnectException("Could not find pair for HDR file :" + str(file_name))
new_file_name = os.path.join(result_folder, file_name_base + str(len(hdr_files)) + ".hdr")
src = zip_arch.open(file_name, 'rU')
FilesHelper.copy_file(src, new_file_name)
hdr_files.append(os.path.join(remote_prefix, os.path.split(new_file_name)[1]))
new_file_name = new_file_name.replace(".hdr", ".img")
src = zip_arch.open(pair_img, 'rU')
FilesHelper.copy_file(src, new_file_name)
elif not file_name.endswith(".img"):
self.logger.warning("Ignored file :" + str(file_name))
if len(hdr_files) < expected_pairs or (fix_number and len(hdr_files) > expected_pairs):
raise ConnectException("Invalid number of files "+ str(len(hdr_files)) + " expected "+ str(expected_pairs))
result = ""
for hdr_name in hdr_files:
result = result + hdr_name + " "
return result
def launch(self, weights, tracts, input_data):
"""
Execute import operations: process the weights and tracts csv files, then use
the reference connectivity passed as input_data for the rest of the attributes.
"""
dti_service = dtipipelineservice.DTIPipelineService()
dti_service._process_csv_file(weights, dti_service.WEIGHTS_FILE)
dti_service._process_csv_file(tracts, dti_service.TRACT_FILE)
weights_matrix = read_list_data(os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE))
tract_matrix = read_list_data(os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE))
FilesHelper.remove_files([os.path.join(os.path.dirname(weights), dti_service.WEIGHTS_FILE),
os.path.join(os.path.dirname(tracts), dti_service.TRACT_FILE)])
if weights_matrix.shape[0] != input_data.orientations.shape[0]:
raise LaunchException("The csv files define %s nodes but the connectivity you selected as reference has only %s nodes."%(
weights_matrix.shape[0], input_data.orientations.shape[0]))
result = Connectivity()
result.storage_path = self.storage_path
result.nose_correction = input_data.nose_correction
result.centres = input_data.centres
result.region_labels = input_data.region_labels
result.weights = weights_matrix
result.tract_lengths = tract_matrix
result.orientations = input_data.orientations
result.areas = input_data.areas
result.cortical = input_data.cortical
result.hemispheres = input_data.hemispheres
return result
def setUp(self):
"""
Set up the context needed by the tests.
"""
# self.clean_database()
self.files_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, self.PROJECT_NAME)
def export_project(self, project):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
:param project)name: name of the project to be exported
:param export_folder: folder where to store export result( e.g zip file)
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
bursts_dict = {}
datatype_burst_mapping = {}
bursts_count = dao.get_bursts_for_project(project.id, count=True)
for start_idx in range(0, bursts_count, BURST_PAGE_SIZE):
bursts = dao.get_bursts_for_project(project.id,
page_start=start_idx,
page_end=start_idx +
BURST_PAGE_SIZE)
for burst in bursts:
self._build_burst_export_dict(burst, bursts_dict)
datatypes_count = dao.get_datatypes_for_project(project.id, count=True)
for start_idx in range(0, datatypes_count, DATAYPES_PAGE_SIZE):
datatypes = dao.get_datatypes_for_project(project.id,
page_start=start_idx,
page_end=start_idx +
DATAYPES_PAGE_SIZE)
for datatype in datatypes:
datatype_burst_mapping[datatype.gid] = datatype.fk_parent_burst
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name,
self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
bursts_file_name = os.path.join(project_folder, BURST_INFO_FILE)
burst_info = {
BURSTS_DICT_KEY: bursts_dict,
DT_BURST_MAP: datatype_burst_mapping
}
with open(bursts_file_name, 'w') as bursts_file:
bursts_file.write(json.dumps(burst_info))
# pack project content into a ZIP file
result_zip = files_helper.zip_folder(result_path, project_folder)
# remove these files, since we only want them in export archive
os.remove(bursts_file_name)
return result_zip
def setUp(self):
"""
Reset the database before each test.
"""
EVENTS_FOLDER = ''
# self.clean_database()
self.project_service = ProjectService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
def setUp(self):
"""
Reset the database before each test.
"""
# self.clean_database()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
self.service = DTIPipelineService('127.0.0.1', 'root')
self.helper = FilesHelper()
def _update_datatype_disk_size(self, file_path):
"""
Computes and updates the disk_size attribute of the DataType, for which was created the given file.
"""
file_handler = FilesHelper()
datatype_gid = self._get_manager(file_path).get_gid_attribute()
datatype = dao.get_datatype_by_gid(datatype_gid)
if datatype is not None:
datatype.disk_size = file_handler.compute_size_on_disk(file_path)
dao.store_entity(datatype)
def get_gifty_file_name(project_id, desired_name):
"""
Compute non-existent file name, in the TEMP folder of
the given project.
Try desired_name, and if already exists, try adding a number.
"""
if project_id:
project = dao.get_project_by_id(project_id)
file_helper = FilesHelper()
temp_path = file_helper.get_project_folder(project,
FilesHelper.TEMP_FOLDER)
return get_unique_file_name(temp_path, desired_name)[0]
return get_unique_file_name(cfg.TVB_STORAGE, desired_name)[0]
def _build_data_export_folder(self, data):
"""
This method computes the folder where results of an export operation will be
stored for a while (e.g until download is done; or for 1 day)
"""
now = datetime.now()
date_str = "%d-%d-%d_%d-%d-%d_%d" % (now.year, now.month, now.day,
now.hour, now.minute, now.second,
now.microsecond)
tmp_str = date_str + "@" + data.gid
data_export_folder = os.path.join(self.export_folder, tmp_str)
files_helper = FilesHelper()
files_helper.check_created(data_export_folder)
return data_export_folder
def tearDown(self):
"""
Remove project folders and restore config file
"""
FilesHelper().remove_project_structure(self.test_project.name)
# self.clean_database()
cfg.CURRENT_DIR = self.old_config_file
def test_launch_operation_HDD_with_space(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb_test.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test": 100}
TVBSettings.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self.operation_service.initiate_operation(
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
method_name=ABCAdapter.LAUNCH_METHOD,
**data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT,
"Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_initiate_operation(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb_test.adapters.testadapter1"
class_name = "TestAdapter1"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test1_val1": 5, "test1_val2": 5}
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
res = self.operation_service.initiate_operation(
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
method_name=ABCAdapter.LAUNCH_METHOD,
**data)
self.assertTrue(
res.index("has finished.") > 10, "Operation didn't finish")
group = dao.find_group(module, class_name)
self.assertEqual(group.module, 'tvb_test.adapters.testadapter1',
"Wrong data stored.")
self.assertEqual(group.classname, 'TestAdapter1', "Wrong data stored.")
dts = dao.get_values_of_datatype(self.test_project.id, Datatype1)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT,
"Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def tearDown(self):
"""
Remove project folders and clean up database.
"""
FilesHelper().remove_project_structure(self.test_project.name)
self.delete_project_folders()
cfg.CURRENT_DIR = self.old_config_file
def setUp(self):
export_manager = ExportManager()
self.datatypeFactory = DatatypesFactory()
self.test_project = self.datatypeFactory.get_project()
# Generate simple data type and export it to H5 file
self.datatype = self.datatypeFactory.create_datatype_with_storage()
_, exported_h5_file, _ = export_manager.export_data(self.datatype, self.TVB_EXPORTER, self.test_project)
# Copy H5 file to another location since the original one / exported
# will be deleted with the project
_, h5_file_name = os.path.split(exported_h5_file)
shutil.copy(exported_h5_file, cfg.TVB_TEMP_FOLDER)
self.h5_file_path = os.path.join(cfg.TVB_TEMP_FOLDER, h5_file_name)
self.assertTrue(os.path.exists(self.h5_file_path), "Simple data type was not exported correct")
# Generate data type group and export it to ZIP file
self.datatype_group = self.datatypeFactory.create_datatype_group()
_, self.zip_file_path, _ = export_manager.export_data(self.datatype_group, self.TVB_EXPORTER, self.test_project)
self.assertTrue(os.path.exists(self.zip_file_path), "Data type group was not exported correct")
FilesHelper().remove_project_structure(self.test_project.name)
self.clean_database(delete_folders=False)
# Recreate project, but a clean one where to import data
self.datatypeFactory = DatatypesFactory()
self.test_project = self.datatypeFactory.get_project()
self.test_user = self.datatypeFactory.get_user()
def test_launch_two_ops_HDD_full_space(self):
"""
Launch two operations and give available space for user so that the first should finish,
but after the update to the user hdd size the second should not.
"""
module = "tvb_test.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test": 100}
TVBSettings.MAX_DISK_SPACE = (1 + float(adapter.get_required_disk_size(**data)))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
#Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
TVBSettings.MAX_DISK_SPACE = float(datatype.disk_size - 1) + float(adapter.get_required_disk_size(**data) - 1)
self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user,
self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 1)
def _burst_create_connectivity(self):
"""
Create a connectivity that will be used in "non-dummy" burst launches (with the actual simulator).
TODO: This is duplicate code from burstservice_test. Should go into the 'generic' DataType factory
once that is done.
"""
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW"
}
algorithm, algo_group = FlowService(
).get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
self.operation = model.Operation(self.test_user.id,
self.test_project.id,
algo_group.id,
json.dumps(''),
meta=json.dumps(meta),
status="STARTED",
method_name=ABCAdapter.LAUNCH_METHOD)
self.operation = dao.store_entity(self.operation)
storage_path = FilesHelper().get_project_folder(
self.test_project, str(self.operation.id))
connectivity = Connectivity(storage_path=storage_path)
connectivity.weights = numpy.ones((74, 74))
connectivity.centres = numpy.ones((74, 3))
adapter_instance = StoreAdapter([connectivity])
OperationService().initiate_prelaunch(self.operation, adapter_instance,
{})
return algorithm.id, connectivity
def test_launch_operation_HDD_full_space_started_ops(self):
"""
Test the actual operation flow by executing a test adapter.
"""
space_taken_by_started = 100
module = "tvb_test.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
started_operation = model.Operation(
self.test_user.id,
self.test_project.id,
group.id,
"",
status=model.STATUS_STARTED,
result_disk_size=space_taken_by_started)
dao.store_entity(started_operation)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TVBSettings.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(**data) + space_taken_by_started -
1)
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self.assertRaises(NoMemoryAvailableException,
self.operation_service.initiate_operation,
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
method_name=ABCAdapter.LAUNCH_METHOD,
**data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 0)
def __init__(self):
now = datetime.now()
micro_postfix = "_%d" % now.microsecond
# Here create all structures needed later for data types creation
self.files_helper = FilesHelper()
# First create user
user = model.User("datatype_factory_user" + micro_postfix, "test_pass",
"*****@*****.**" + micro_postfix, True, "user")
self.user = dao.store_entity(user)
# Now create a project
project_service = ProjectService()
data = dict(name='DatatypesFactoryProject' + micro_postfix,
description='test_desc',
users=[])
self.project = project_service.store_project(self.user, True, None,
**data)
# Create algorithm
alg_category = model.AlgorithmCategory('one', True)
dao.store_entity(alg_category)
alg_group = model.AlgorithmGroup("test_module1", "classname1",
alg_category.id)
dao.store_entity(alg_group)
algorithm = model.Algorithm(alg_group.id,
'id',
name='',
req_data='',
param_name='',
output='')
self.algorithm = dao.store_entity(algorithm)
#Create an operation
self.meta = {
DataTypeMetaData.KEY_SUBJECT: self.USER_FULL_NAME,
DataTypeMetaData.KEY_STATE: self.DATATYPE_STATE
}
operation = model.Operation(self.user.id,
self.project.id,
self.algorithm.id,
'test parameters',
meta=json.dumps(self.meta),
status="FINISHED",
method_name=ABCAdapter.LAUNCH_METHOD)
self.operation = dao.store_entity(operation)
def set_operation_id(self, operation_id):
"""
Setter for FK_operation_id.
"""
self.fk_from_operation = operation_id
parent_project = dao.get_project_for_operation(operation_id)
self.storage_path = FilesHelper().get_project_folder(
parent_project, str(operation_id))
self._storage_manager = None
class DTITest(TransactionalTestCase):
"""
Test basic functionality of DTI Import Service.
"""
### First dataSet
FILE_1 = os.path.join(os.path.dirname(current_pack.__file__), "data",
"TVB_ConnectionCapacityMatrix.csv")
FILE_2 = os.path.join(os.path.dirname(current_pack.__file__), "data",
"TVB_ConnectionDistanceMatrix.csv")
### Second dataSet
FILE_3 = os.path.join(os.path.dirname(current_pack.__file__), "data",
"TVB_ConnectionCapacityMatrix_3.csv")
FILE_4 = os.path.join(os.path.dirname(current_pack.__file__), "data",
"TVB_ConnectionDistanceMatrix_3.csv")
def setUp(self):
"""
Reset the database before each test.
"""
# self.clean_database()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
self.service = DTIPipelineService('127.0.0.1', 'root')
self.helper = FilesHelper()
def test_process_csv(self):
"""
Test that a CSV generated on the server is correctly processed.
"""
folder = self.helper.get_project_folder(self.test_project, "TEMP")
for file_name in [self.FILE_1, self.FILE_2, self.FILE_3, self.FILE_4]:
intermediate_file = os.path.join(folder,
os.path.split(file_name)[1])
self.helper.copy_file(file_name, intermediate_file)
result_file = 'weights.txt' if 'Capacity' in file_name else 'tracts.txt'
result_file = os.path.join(folder, result_file)
self.service._process_csv_file(intermediate_file, result_file)
matrix = read_list_data(result_file)
self.assertEqual(96, len(matrix))
self.assertEqual(96, len(matrix[0]))
def _create_connectivity(self, nodes_number):
"""
Create a connectivity entity and return its GID
"""
storage_path = FilesHelper().get_project_folder(self.test_project, str(self.operation.id))
connectivity = Connectivity(storage_path=storage_path)
connectivity.weights = numpy.ones((nodes_number, nodes_number))
connectivity.centres = numpy.ones((nodes_number, 3))
adapter_instance = StoreAdapter([connectivity])
OperationService().initiate_prelaunch(self.operation, adapter_instance, {})
return dao.get_datatype_by_id(connectivity.id).gid
def test_adapter_launch(self):
"""
Test that the adapters launches and successfully generates a datatype measure entry.
"""
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW"
}
algo_group = FlowService().get_algorithm_by_module_and_class(
SIMULATOR_MODULE, SIMULATOR_CLASS)[1]
self.operation = model.Operation(self.test_user.id,
self.test_project.id,
algo_group.id,
json.dumps(''),
meta=json.dumps(meta),
status=model.STATUS_STARTED,
method_name=ABCAdapter.LAUNCH_METHOD)
self.operation = dao.store_entity(self.operation)
storage_path = FilesHelper().get_project_folder(
self.test_project, str(self.operation.id))
dummy_input = numpy.arange(1, 10001).reshape(10, 10, 10, 10)
dummy_time = numpy.arange(1, 11)
# Get connectivity
connectivities = FlowService().get_available_datatypes(
self.test_project.id, "tvb.datatypes.connectivity.Connectivity")
self.assertEqual(len(connectivities), 1)
connectivity_gid = connectivities[0][2]
dummy_time_series = TimeSeriesRegion()
dummy_time_series.storage_path = storage_path
dummy_time_series.write_data_slice(dummy_input)
dummy_time_series.write_time_slice(dummy_time)
dummy_time_series.close_file()
dummy_time_series.start_time = 0.0
dummy_time_series.sample_period = 1.0
dummy_time_series.connectivity = connectivity_gid
adapter_instance = StoreAdapter([dummy_time_series])
OperationService().initiate_prelaunch(self.operation, adapter_instance,
{})
dummy_time_series = dao.get_generic_entity(dummy_time_series.__class__,
dummy_time_series.gid,
'gid')[0]
ts_metric_adapter = TimeseriesMetricsAdapter()
resulted_metric = ts_metric_adapter.launch(dummy_time_series)
self.assertTrue(isinstance(resulted_metric, DatatypeMeasure),
"Result should be a datatype measure.")
self.assertTrue(
len(resulted_metric.metrics) == len(
ts_metric_adapter.available_algorithms.keys()),
"A result should have been generated for every metric.")
def test_get_filtered_datatypes(self):
"""
Test the filter function when retrieving dataTypes.
"""
#Create some test operations
start_dates = [datetime.now(),
datetime.strptime("08-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2010", "%m-%d-%Y"),
datetime.strptime("05-06-2010", "%m-%d-%Y"),
datetime.strptime("07-21-2011", "%m-%d-%Y")]
end_dates = [datetime.now(),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2010", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y"),
datetime.strptime("08-12-2011", "%m-%d-%Y")]
for i in range(5):
operation = model.Operation(self.test_user.id, self.test_project.id, self.algo_inst.id, 'test params',
status="FINISHED", start_date=start_dates[i], completion_date=end_dates[i])
operation = dao.store_entity(operation)
storage_path = FilesHelper().get_project_folder(self.test_project, str(operation.id))
if i < 4:
datatype_inst = Datatype1()
datatype_inst.type = "Datatype1"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
else:
for _ in range(2):
datatype_inst = Datatype2()
datatype_inst.storage_path = storage_path
datatype_inst.type = "Datatype2"
datatype_inst.subject = "John Doe" + str(i)
datatype_inst.state = "RAW"
datatype_inst.string_data = ["data"]
datatype_inst.set_operation_id(operation.id)
dao.store_entity(datatype_inst)
returned_data = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb_test.datatypes.datatype1.Datatype1")
for row in returned_data:
if row[1] != 'Datatype1':
self.fail("Some invalid data was returned!")
self.assertEqual(4, len(returned_data), "Invalid length of result")
filter_op = FilterChain(fields=[FilterChain.datatype + ".state", FilterChain.operation + ".start_date"],
values=["RAW", datetime.strptime("08-01-2010", "%m-%d-%Y")], operations=["==", ">"])
returned_data = self.flow_service.get_available_datatypes(self.test_project.id,
"tvb_test.datatypes.datatype1.Datatype1", filter_op)
returned_subjects = [one_data[3] for one_data in returned_data]
if "John Doe0" not in returned_subjects or "John Doe1" not in returned_subjects or len(returned_subjects) != 2:
self.fail("DataTypes were not filtered properly!")
def tearDown(self):
"""
Clean-up tests data
"""
project = self.datatypeFactory.get_project()
FilesHelper().remove_project_structure(project.name)
# Remove EXPORT folder
export_folder = os.path.join(cfg.TVB_STORAGE,
ExportManager.EXPORT_FOLDER_NAME)
if os.path.exists(export_folder):
shutil.rmtree(export_folder)
def test_delete_dt_free_HDD_space(self):
"""
Launch two operations and give enough available space for user so that both should finish.
"""
module = "tvb_test.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test": 100}
TVBSettings.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 0)
self.operation_service.initiate_operation(
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
method_name=ABCAdapter.LAUNCH_METHOD,
**data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT,
"Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
#Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
ProjectService().remove_datatype(self.test_project.id, datatype.gid)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 0)
self.operation_service.initiate_operation(
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
method_name=ABCAdapter.LAUNCH_METHOD,
**data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT,
"Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def setUp(self):
"""
Prepare before each test.
"""
self.project_service = ProjectService()
self.flow_service = FlowService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, "ProjectStructure")
self.relevant_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.RELEVANT_VIEW)
self.full_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.FULL_VIEW)
def test_import_surface_gifti_data(self):
"""
This method tests import of a surface from GIFTI file.
!!! Important: We changed this test to execute only GIFTI parse
because storing surface it takes too long (~ 9min) since
normals needs to be calculated.
"""
operation_id = self.datatypeFactory.get_operation().id
storage_path = FilesHelper().get_operation_folder(self.test_project.name, operation_id)
parser = GIFTIParser(storage_path, operation_id)
surface = parser.parse(self.GIFTI_SURFACE_FILE)
self.assertEqual(131342, len(surface.vertices))
self.assertEqual(262680, len(surface.triangles))
def test_import_timeseries_gifti_data(self):
"""
This method tests import of a time series from GIFTI file.
!!! Important: We changed this test to execute only GIFTI parse
because storing surface it takes too long (~ 9min) since
normals needs to be calculated.
"""
operation_id = self.datatypeFactory.get_operation().id
storage_path = FilesHelper().get_operation_folder(self.test_project.name, operation_id)
parser = GIFTIParser(storage_path, operation_id)
time_series = parser.parse(self.GIFTI_TIME_SERIES_FILE)
data_shape = time_series.read_data_shape()
self.assertEqual(135, data_shape[0])
self.assertEqual(143479, data_shape[1])
def _create_timeseries(self):
"""Launch adapter to persist a TimeSeries entity"""
storage_path = FilesHelper().get_project_folder(
self.test_project, str(self.operation.id))
time_series = TimeSeries()
time_series.sample_period = 10.0
time_series.start_time = 0.0
time_series.storage_path = storage_path
time_series.write_data_slice(numpy.array([1.0, 2.0, 3.0]))
time_series.close_file()
time_series.sample_period_unit = 'ms'
self._store_entity(time_series, "TimeSeries",
"tvb.datatypes.time_series")
timeseries = self.flow_service.get_available_datatypes(
self.test_project.id, "tvb.datatypes.time_series.TimeSeries")
self.assertEqual(len(timeseries), 1, "Should be only one TimeSeries")
def __init__(self, remote_machine=None, remote_user=None):
"""
:param remote_machine: IP for the remote machine
:param remote_user: Username valid on remote_machine. No further password should be needed for connecting.
"""
self.logger = get_logger(self.__class__.__module__)
self.remote_machine = remote_machine
self.remote_user = remote_user
self.flow_service = FlowService()
self.file_handler = FilesHelper()
folder_default_data = os.path.dirname(demo_root.__file__)
file_order = os.path.join(folder_default_data, self.FILE_NODES_ORDER)
self.expected_nodes_order = read_list_data(file_order, dtype=numpy.int32, usecols=[0])
zip_path = os.path.join(folder_default_data, self.CONNECTIVITY_DEFAULT)
if not (os.path.exists(zip_path) and os.path.isfile(zip_path)):
raise ConnectException("Could not find default Connectivity for the pipeline! " + str(zip_path))
self.default_connectivity_zip_path = zip_path
