def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.user_service = UserService()
self.flow_service = FlowService()
analyze_category = self.flow_service.get_launchable_non_viewers()
self.analyze_category_link = '/flow/step/' + str(analyze_category.id)
self.analyze_adapters = None
self.connectivity_tab_link = '/flow/step_connectivity'
view_category = self.flow_service.get_visualisers_category()
conn_id = self.flow_service.get_algorithm_by_module_and_class(CONNECTIVITY_MODULE, CONNECTIVITY_CLASS)[1].id
connectivity_link = self.get_url_adapter(view_category.id, conn_id)
self.connectivity_submenu = [dict(title="Large Scale Connectivity", subsection="connectivity",
description="View Connectivity Regions. Perform Connectivity lesions",
link=connectivity_link),
dict(title="Local Connectivity", subsection="local",
link='/spatial/localconnectivity/step_1/1',
description="Create or view existent Local Connectivity entities.")]
self.burst_submenu = [dict(link='/burst', subsection='burst',
title='Simulation Cockpit', description='Manage simulations'),
dict(link='/burst/dynamic', subsection='dynamic',
title='Phase plane', description='Configure model dynamics')]
def fire_simulation(project_id=1, **kwargs):
project = dao.get_project_by_id(project_id)
flow_service = FlowService()
# below the holy procedure to launch with the correct parameters taken from the defaults
stored_adapter = flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
simulator_adapter = ABCAdapter.build_adapter(stored_adapter)
flatten_interface = simulator_adapter.flaten_input_interface()
itree_mngr = flow_service.input_tree_manager
prepared_flatten_interface = itree_mngr.fill_input_tree_with_options(flatten_interface, project.id,
stored_adapter.fk_category)
launch_args = {}
for entry in prepared_flatten_interface:
value = entry['default']
if isinstance(value, dict):
value = str(value)
if hasattr(value, 'tolist'):
value = value.tolist()
launch_args[entry['name']] = value
launch_args.update(**kwargs)
# end of magic
launched_operation = flow_service.fire_operation(simulator_adapter, project.administrator,
project.id, **launch_args)[0]
return launched_operation
def fire_simulation(project_id=1, **kwargs):
project = dao.get_project_by_id(project_id)
flow_service = FlowService()
# below the holy procedure to launch with the correct parameters taken from the defaults
stored_adapter = flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
simulator_adapter = ABCAdapter.build_adapter(stored_adapter)
flatten_interface = simulator_adapter.flaten_input_interface()
itree_mngr = flow_service.input_tree_manager
prepared_flatten_interface = itree_mngr.fill_input_tree_with_options(flatten_interface, project.id,
stored_adapter.fk_category)
launch_args = {}
for entry in prepared_flatten_interface:
value = entry['default']
if isinstance(value, dict):
value = str(value)
if hasattr(value, 'tolist'):
value = value.tolist()
launch_args[entry['name']] = value
launch_args.update(**kwargs)
# end of magic
launched_operation = flow_service.fire_operation(simulator_adapter, project.administrator,
project.id, **launch_args)[0]
return launched_operation
def test_launch_two_ops_HDD_full_space(self):
"""
Launch two operations and give available space for user so that the first should finish,
but after the update to the user hdd size the second should not.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = 1 + float(adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
datatype = self._assert_stored_dt2()
# Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
# plus what is estimated to be required from the next one (transform from B to MB)
TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + float(
adapter.get_required_disk_size(**data) - 1
)
self.assertRaises(
NoMemoryAvailableException,
self.operation_service.initiate_operation,
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
**data
)
self._assert_stored_dt2()
def test_launch_operation_HDD_full_space_started_ops(self):
"""
Test the actual operation flow by executing a test adapter.
"""
space_taken_by_started = 100
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
started_operation = model.Operation(
self.test_user.id,
self.test_project.id,
group.id,
"",
status=model.STATUS_STARTED,
estimated_disk_size=space_taken_by_started,
)
dao.store_entity(started_operation)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) + space_taken_by_started - 1)
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.assertRaises(
NoMemoryAvailableException,
self.operation_service.initiate_operation,
self.test_user,
self.test_project.id,
adapter,
tmp_folder,
**data
)
self._assert_no_dt2()
def _store_imported_datatypes_in_db(self, project, all_datatypes,
dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id],
project.id)
def test_initiate_operation(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb.tests.framework.adapters.testadapter1"
class_name = "TestAdapter1"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test1_val1": 5, "test1_val2": 5}
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
res = self.operation_service.initiate_operation(
self.test_user, self.test_project.id, adapter, tmp_folder, **data
)
self.assertTrue(res.index("has finished.") > 10, "Operation didn't finish")
group = dao.find_group(module, class_name)
self.assertEqual(group.module, "tvb.tests.framework.adapters.testadapter1", "Wrong data stored.")
self.assertEqual(group.classname, "TestAdapter1", "Wrong data stored.")
dts, count = dao.get_values_of_datatype(self.test_project.id, Datatype1)
self.assertEqual(count, 1)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def _import(self, import_file_path, sensors_type, expected_data):
"""
This method is used for importing sensors
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.sensors_importer', 'Sensors_Importer')
args = {'sensors_file': import_file_path, 'sensors_type': sensors_type}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **args)
data_types = FlowService().get_available_datatypes(
self.test_project.id,
expected_data.module + "." + expected_data.type)[0]
assert 1 == len(
data_types), "Project should contain only one data type = Sensors."
time_series = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert time_series is not None, "Sensors instance should not be none"
return time_series
def import_surface_zip(user,
project,
zip_path,
surface_type,
zero_based='True'):
### Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.zip_surface_importer',
'ZIPSurfaceImporter')
form = ZIPSurfaceImporterForm()
form.fill_from_post({
'_uploaded': Part(zip_path, HeaderMap({}), ''),
'_zero_based_triangles': zero_based,
'_should_center': 'True',
'_surface_type': surface_type,
'_Data_Subject': 'John Doe'
})
form.uploaded.data = zip_path
importer.submit_form(form)
### Launch import Operation
FlowService().fire_operation(importer, user, project.id,
**form.get_form_values())
data_types = FlowService().get_available_datatypes(
project.id, SurfaceIndex)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
surface.user_tag_3 = ''
assert surface is not None, "Surface should not be None"
return surface
def _store_imported_datatypes_in_db(self, project, all_datatypes,
dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
# Compute disk size. Similar to ABCAdapter._capture_operation_results.
# No need to close the h5 as we have not written to it.
associated_file = os.path.join(
datatype.storage_path, datatype.get_storage_file_name())
datatype.disk_size = FilesHelper.compute_size_on_disk(
associated_file)
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id],
project.id)
def launch_simulation_workflow(json_path, prj_id):
"""
:param json_path: Path towards a local JSON file exported from GUI
:param prj_id: This ID of a project needs to exists in DB, and it can be taken from the WebInterface
"""
project = dao.get_project_by_id(prj_id)
with open(json_path, 'rb') as input_file:
simulation_json = input_file.read()
simulation_json = json.loads(simulation_json)
LOG.info("Simulation JSON loaded from file '%s': \n %s", json_path,
simulation_json)
importer = ImportService()
simulation_config = importer.load_burst_entity(simulation_json, prj_id)
LOG.info("Simulation Workflow configuration object loaded: \n %s",
simulation_config)
flow_service = FlowService()
stored_adapter = flow_service.get_algorithm_by_module_and_class(
SIMULATOR_MODULE, SIMULATOR_CLASS)
LOG.info("Found Simulation algorithm in local DB: \n %s",
stored_adapter)
burst_service = BurstService()
burst_service.launch_burst(simulation_config, 0, stored_adapter.id,
project.administrator.id, LAUNCH_NEW)
LOG.info(
"Check in the web GUI for your operation. It should be starting now ..."
)
def import_surface_gifti(user, project, path):
"""
This method is used for importing data in GIFIT format
:param import_file_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter('tvb.adapters.uploaders.gifti_surface_importer', 'GIFTISurfaceImporter')
form = GIFTISurfaceImporterForm()
form.fill_from_post({'_file_type': form.get_view_model().KEY_OPTION_READ_METADATA,
'_data_file': Part(path, HeaderMap({}), ''),
'_data_file_part2': Part('', HeaderMap({}), ''),
'_should_center': 'False',
'_Data_Subject': 'John Doe',
})
form.data_file.data = path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer, user, project.id, view_model=view_model)
surface = CorticalSurface
data_types = FlowService().get_available_datatypes(project.id,
surface.__module__ + "." + surface.__name__)[0]
assert 1, len(data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None == "TimeSeries should not be none"
return surface
class _BaseLinksTest(TransactionalTestCase):
GEORGE1st = "george the grey"
GEORGE2nd = "george"
def setUpTVB(self):
"""
Creates a user, an algorithm and 2 projects
Project src_project will have an operation and 2 datatypes
Project dest_project will be empty.
Initializes a flow and a project service
"""
datatype_factory = DatatypesFactory()
self.user = datatype_factory.user
self.src_project = datatype_factory.project
self.red_datatype = datatype_factory.create_simple_datatype(subject=self.GEORGE1st)
self.blue_datatype = datatype_factory.create_datatype_with_storage(subject=self.GEORGE2nd)
# create the destination project
self.dest_project = TestFactory.create_project(admin=datatype_factory.user, name="destination")
self.flow_service = FlowService()
self.project_service = ProjectService()
def tearDown(self):
self.clean_database(delete_folders=True)
def red_datatypes_in(self, project_id):
return self.flow_service.get_available_datatypes(project_id, Datatype1)[1]
def blue_datatypes_in(self, project_id):
return self.flow_service.get_available_datatypes(project_id, Datatype2)[1]
def initialize_two_projects(self, simple_datatype_factory,
datatype_with_storage_factory):
"""
Creates a user, an algorithm and 2 projects
Project src_project will have an operation and 2 datatypes
Project dest_project will be empty.
Initializes a flow and a project service
"""
self.clean_database(delete_folders=True)
src_user = TestFactory.create_user('Source_User')
self.src_usr_id = src_user.id
self.src_project = TestFactory.create_project(src_user,
"Source_Project")
self.red_datatype = simple_datatype_factory(subject=self.GEORGE1st)
self.blue_datatype = datatype_with_storage_factory(
subject=self.GEORGE2nd)
# create the destination project
dst_user = TestFactory.create_user('Destination_User')
self.dst_usr_id = dst_user.id
self.dest_project = TestFactory.create_project(dst_user,
"Destination_Project")
self.flow_service = FlowService()
self.project_service = ProjectService()
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 0, "There should be no operation"
adapter_instance = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3')
data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
assert len(all_operations) == 1, "Expected one operation group"
assert all_operations[0][2] == 2, "Expected 2 operations in group"
operation_group_id = all_operations[0][3]
assert operation_group_id != None, "The operation should be part of a group."
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def import_surface_obj(user, project, obj_path, surface_type):
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.obj_importer', 'ObjSurfaceImporter')
form = ObjSurfaceImporterForm()
form.fill_from_post({
'data_file': Part(obj_path, HeaderMap({}), ''),
'surface_type': surface_type,
'Data_Subject': 'John Doe'
})
form.data_file.data = obj_path
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, SurfaceIndex)[0]
assert 1, len(
data_types) == "Project should contain only one data type."
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert surface is not None, "Surface should not be None"
return surface
def _store_imported_datatypes_in_db(self, project, all_datatypes, dt_burst_mappings, burst_ids_mapping):
def by_time(dt):
return dt.create_date or datetime.now()
if burst_ids_mapping is None:
burst_ids_mapping = {}
if dt_burst_mappings is None:
dt_burst_mappings = {}
all_datatypes.sort(key=by_time)
for datatype in all_datatypes:
old_burst_id = dt_burst_mappings.get(datatype.gid)
if old_burst_id is not None:
datatype.fk_parent_burst = burst_ids_mapping[old_burst_id]
datatype_allready_in_tvb = dao.get_datatype_by_gid(datatype.gid)
if not datatype_allready_in_tvb:
# Compute disk size. Similar to ABCAdapter._capture_operation_results.
# No need to close the h5 as we have not written to it.
associated_file = os.path.join(datatype.storage_path, datatype.get_storage_file_name())
datatype.disk_size = FilesHelper.compute_size_on_disk(associated_file)
self.store_datatype(datatype)
else:
FlowService.create_link([datatype_allready_in_tvb.id], project.id)
def import_sensors(user, project, zip_path, sensors_type):
"""
This method is used for importing sensors
:param zip_path: absolute path of the file to be imported
"""
# Retrieve Adapter instance
importer = TestFactory.create_adapter(
'tvb.adapters.uploaders.sensors_importer', 'SensorsImporter')
form = SensorsImporterForm()
form.fill_from_post({
'sensors_file': Part(zip_path, HeaderMap({}), ''),
'sensors_type': sensors_type,
'Data_Subject': 'John Doe'
})
form.sensors_file.data = zip_path
form.sensors_type.data = sensors_type
view_model = form.get_view_model()()
form.fill_trait(view_model)
importer.submit_form(form)
# Launch import Operation
FlowService().fire_operation(importer,
user,
project.id,
view_model=view_model)
data_types = FlowService().get_available_datatypes(
project.id, SensorsIndex)[0]
assert 1 == len(
data_types), "Project should contain only one data type = Sensors."
sensors = ABCAdapter.load_entity_by_gid(data_types[0][2])
assert sensors is not None, "Sensors instance should not be none"
return sensors
def update():
"""
Update TVB code to SVN revision version 4455.
This update was done for release 1.0.2.
"""
projects_count = dao.get_all_projects(is_count=True)
for page_start in range(0, projects_count, PAGE_SIZE):
projects_page = dao.get_all_projects(page_start=page_start,
page_size=PAGE_SIZE)
for project in projects_page:
try:
user = dao.get_system_user()
adapter = ObjSurfaceImporter()
FlowService().fire_operation(adapter,
user,
project.id,
visible=False,
surface_type=EEG_CAP,
data_file=DATA_FILE_EEG_CAP)
adapter = ObjSurfaceImporter()
FlowService().fire_operation(adapter,
user,
project.id,
visible=False,
surface_type=FACE,
data_file=DATA_FILE_FACE)
except Exception as excep:
LOGGER.exception(excep)
TvbProfile.current.manager.add_entries_to_config_file(
{stored.KEY_MATLAB_EXECUTABLE: get_matlab_executable()})
def __init__(self, conf):
"""
:param conf: burst configuration entity
"""
self.logger = get_logger(__name__)
self.flow_service = FlowService()
self.conf = conf
def update():
"""
Update TVB code to SVN revision version 4455.
This update was done for release 1.0.2.
"""
projects_count = dao.get_all_projects(is_count=True)
for page_start in range(0, projects_count, PAGE_SIZE):
projects_page = dao.get_all_projects(page_start=page_start,
page_size=PAGE_SIZE)
for project in projects_page:
try:
user = dao.get_system_user()
adapter = ObjSurfaceImporter()
FlowService().fire_operation(adapter,
user,
project.id,
visible=False,
surface_type=EEG_CAP,
data_file=DATA_FILE_EEG_CAP)
adapter = ObjSurfaceImporter()
FlowService().fire_operation(adapter,
user,
project.id,
visible=False,
surface_type=FACE,
data_file=DATA_FILE_FACE)
except Exception, excep:
LOGGER.exception(excep)
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.user_service = UserService()
self.flow_service = FlowService()
self.analyze_category_link = '/flow/step_analyzers'
self.analyze_adapters = None
self.connectivity_tab_link = '/flow/step_connectivity'
view_category = self.flow_service.get_visualisers_category()
conn_id = self.flow_service.get_algorithm_by_module_and_class(CONNECTIVITY_MODULE, CONNECTIVITY_CLASS).id
connectivity_link = self.get_url_adapter(view_category.id, conn_id)
self.connectivity_submenu = [dict(title="Large Scale Connectivity", link=connectivity_link,
subsection=WebStructure.SUB_SECTION_CONNECTIVITY,
description="View Connectivity Regions. Perform Connectivity lesions"),
dict(title="Local Connectivity", link='/spatial/localconnectivity/step_1/1',
subsection=WebStructure.SUB_SECTION_LOCAL_CONNECTIVITY,
description="Create or view existent Local Connectivity entities.")]
allen_algo = self.flow_service.get_algorithm_by_module_and_class(ALLEN_CREATOR_MODULE, ALLEN_CREATOR_CLASS)
if allen_algo:
# Only add the Allen Creator if AllenSDK is installed
allen_link = self.get_url_adapter(allen_algo.fk_category, allen_algo.id)
self.connectivity_submenu.append(dict(title="Allen Connectome Downloader", link=allen_link,
subsection=WebStructure.SUB_SECTION_ALLEN,
description="Download a mouse connectivity from Allen dataset"))
self.burst_submenu = [dict(link='/burst', subsection=WebStructure.SUB_SECTION_BURST,
title='Simulation Cockpit', description='Manage simulations'),
dict(link='/burst/dynamic', subsection='dynamic',
title='Phase plane', description='Configure model dynamics')]
def test_launch_two_ops_HDD_full_space(self):
"""
Launch two operations and give available space for user so that the first should finish,
but after the update to the user hdd size the second should not.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = (1 + float(adapter.get_required_disk_size(**data)))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
#Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \
float(adapter.get_required_disk_size(**data) - 1)
self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user,
self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
self.assertEqual(len(dts), 1)
def test_launch_two_ops_HDD_full_space(self):
"""
Launch two operations and give available space for user so that the first should finish,
but after the update to the user hdd size the second should not.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = (1 + float(adapter.get_required_disk_size(**data)))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
#Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \
float(adapter.get_required_disk_size(**data) - 1)
self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user,
self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
self.assertEqual(len(dts), 1)
def test_datatypes_groups(self):
"""
Tests if the dataType group is set correct on the dataTypes resulted from the same operation group.
"""
flow_service = FlowService()
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 0, "There should be no operation")
algogroup = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapter3")
group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup)
adapter_instance = flow_service.build_adapter_instance(group)
data = {model.RANGE_PARAMETER_1: "param_5", "param_5": [1, 2]}
## Create Group of operations
flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data)
all_operations = dao.get_filtered_operations(self.test_project.id, None)
self.assertEqual(len(all_operations), 1, "Expected one operation group")
self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group")
operation_group_id = all_operations[0][3]
self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.")
self.operation_service.stop_operation(all_operations[0][0])
self.operation_service.stop_operation(all_operations[0][1])
## Make sure operations are executed
self.operation_service.launch_operation(all_operations[0][0], False)
self.operation_service.launch_operation(all_operations[0][1], False)
resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id)
self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)))
dt = dao.get_datatype_by_id(resulted_datatypes[0].id)
datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id)
self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def _importSurface(self, import_file_path=None):
"""
This method is used for importing data in GIFIT format
:param import_file_path: absolute path of the file to be imported
"""
### Retrieve Adapter instance
group = dao.find_group('tvb.adapters.uploaders.gifti_surface_importer',
'GIFTISurfaceImporter')
importer = ABCAdapter.build_adapter(group)
args = {
'data_file': import_file_path,
DataTypeMetaData.KEY_SUBJECT: ""
}
### Launch import Operation
FlowService().fire_operation(importer, self.test_user,
self.test_project.id, **args)
surface = CorticalSurface()
data_types = FlowService().get_available_datatypes(
self.test_project.id, surface.module + "." + surface.type)[0]
self.assertEqual(1, len(data_types),
"Project should contain only one data type.")
surface = ABCAdapter.load_entity_by_gid(data_types[0][2])
self.assertTrue(surface is not None, "TimeSeries should not be none")
return surface
def test_launch_operation_HDD_full_space_started_ops(self):
"""
Test the actual operation flow by executing a test adapter.
"""
space_taken_by_started = 100
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
started_operation = model.Operation(
self.test_user.id,
self.test_project.id,
group.id,
"",
status=model.STATUS_STARTED,
estimated_disk_size=space_taken_by_started)
dao.store_entity(started_operation)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(**data) + space_taken_by_started -
1)
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self.assertRaises(NoMemoryAvailableException,
self.operation_service.initiate_operation,
self.test_user, self.test_project.id, adapter,
tmp_folder, **data)
self._assert_no_dt2()
def test_initiate_operation(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb.tests.framework.adapters.testadapter1"
class_name = "TestAdapter1"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test1_val1": 5, "test1_val2": 5}
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
res = self.operation_service.initiate_operation(
self.test_user, self.test_project.id, adapter, tmp_folder, **data)
self.assertTrue(
res.index("has finished.") > 10, "Operation didn't finish")
group = dao.find_group(module, class_name)
self.assertEqual(group.module,
'tvb.tests.framework.adapters.testadapter1',
"Wrong data stored.")
self.assertEqual(group.classname, 'TestAdapter1', "Wrong data stored.")
dts, count = dao.get_values_of_datatype(self.test_project.id,
Datatype1)
self.assertEqual(count, 1)
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT,
"Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_launch_two_ops_HDD_full_space(self):
"""
Launch two operations and give available space for user so that the first should finish,
but after the update to the user hdd size the second should not.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = (
1 + float(adapter.get_required_disk_size(**data)))
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self.operation_service.initiate_operation(self.test_user,
self.test_project.id,
adapter, tmp_folder, **data)
datatype = self._assert_stored_dt2()
#Now update the maximum disk size to be less than size of the previously resulted datatypes (transform kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size - 1) + \
float(adapter.get_required_disk_size(**data) - 1)
self.assertRaises(NoMemoryAvailableException,
self.operation_service.initiate_operation,
self.test_user, self.test_project.id, adapter,
tmp_folder, **data)
self._assert_stored_dt2()
def test_launch_two_ops_HDD_with_space(self):
"""
Launch two operations and give enough available space for user so that both should finish.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = 2 * float(
adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self.operation_service.initiate_operation(self.test_user,
self.test_project.id,
adapter, tmp_folder, **data)
datatype = self._assert_stored_dt2()
#Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size) + float(
adapter.get_required_disk_size(**data))
self.operation_service.initiate_operation(self.test_user,
self.test_project.id,
adapter, tmp_folder, **data)
self._assert_stored_dt2(2)
def test_delete_dt_free_HDD_space(self):
"""
Launch two operations and give enough available space for user so that both should finish.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(
adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project,
"TEMP")
self._assert_no_dt2()
self.operation_service.initiate_operation(self.test_user,
self.test_project.id,
adapter, tmp_folder, **data)
datatype = self._assert_stored_dt2()
# Now free some space and relaunch
ProjectService().remove_datatype(self.test_project.id, datatype.gid)
self._assert_no_dt2()
self.operation_service.initiate_operation(self.test_user,
self.test_project.id,
adapter, tmp_folder, **data)
self._assert_stored_dt2()
def setUp(self):
"""
Reset the database before each test.
"""
initialize_storage()
user = model.User("test_user", "test_pass", "*****@*****.**", True,
"user")
self.test_user = dao.store_entity(user)
data = dict(name='test_proj', description='desc', users=[])
self.test_project = ProjectService().store_project(
self.test_user, True, None, **data)
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "INTERMEDIATE"
}
algo_group = dao.find_group(SIMULATOR_MODULE, SIMULATOR_CLASS)
self.simulator_adapter = FlowService().build_adapter_instance(
algo_group)
self.operation = model.Operation(self.test_user.id,
self.test_project.id,
algo_group.id,
json.dumps(SIMULATOR_PARAMETERS),
meta=json.dumps(meta),
status=model.STATUS_STARTED,
method_name=ABCAdapter.LAUNCH_METHOD)
self.operation = dao.store_entity(self.operation)
SIMULATOR_PARAMETERS['connectivity'] = self._create_connectivity(
self.CONNECTIVITY_NODES)
def test_adapter_memory(self):
"""
Test that a method not implemented exception is raised in case the
get_required_memory_size method is not implemented.
"""
algo_group = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapterHDDRequired")
adapter = FlowService().build_adapter_instance(algo_group)
self.assertEqual(42, adapter.get_required_memory_size())
def transactional_setup_method(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
category = dao.get_uploader_categories()[0]
self.algorithm = dao.store_entity(model_operation.Algorithm(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, category.id))
def transactional_setup_method(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
category = dao.get_uploader_categories()[0]
self.algorithm = dao.store_entity(model.Algorithm(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, category.id))
def setUp(self):
"""
Reset the database before each test.
"""
self.clean_database()
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
self.operation = TestFactory.create_operation(
test_user=self.test_user, test_project=self.test_project)
def test_adapter_launch(self):
"""
Test that the adapters launches and successfully generates a datatype measure entry.
"""
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"
}
algo_group = FlowService().get_algorithm_by_module_and_class(
SIMULATOR_MODULE, SIMULATOR_CLASS)[1]
self.operation = model.Operation(self.test_user.id,
self.test_project.id,
algo_group.id,
json.dumps(''),
meta=json.dumps(meta),
status=model.STATUS_STARTED,
method_name=ABCAdapter.LAUNCH_METHOD)
self.operation = dao.store_entity(self.operation)
storage_path = FilesHelper().get_project_folder(
self.test_project, str(self.operation.id))
dummy_input = numpy.arange(1, 10001).reshape(10, 10, 10, 10)
dummy_time = numpy.arange(1, 11)
# Get connectivity
connectivities = FlowService().get_available_datatypes(
self.test_project.id, "tvb.datatypes.connectivity.Connectivity")[0]
self.assertEqual(2, len(connectivities))
connectivity_gid = connectivities[0][2]
dummy_time_series = TimeSeriesRegion()
dummy_time_series.storage_path = storage_path
dummy_time_series.write_data_slice(dummy_input)
dummy_time_series.write_time_slice(dummy_time)
dummy_time_series.close_file()
dummy_time_series.start_time = 0.0
dummy_time_series.sample_period = 1.0
dummy_time_series.connectivity = connectivity_gid
adapter_instance = StoreAdapter([dummy_time_series])
OperationService().initiate_prelaunch(self.operation, adapter_instance,
{})
dummy_time_series = dao.get_generic_entity(dummy_time_series.__class__,
dummy_time_series.gid,
'gid')[0]
ts_metric_adapter = TimeseriesMetricsAdapter()
resulted_metric = ts_metric_adapter.launch(dummy_time_series)
self.assertTrue(isinstance(resulted_metric, DatatypeMeasure),
"Result should be a datatype measure.")
self.assertTrue(
len(resulted_metric.metrics) >= len(
ts_metric_adapter.available_algorithms.keys()),
"At least a result should have been generated for every metric.")
for metric_value in resulted_metric.metrics.values():
self.assertTrue(isinstance(metric_value, (float, int)))
def setUp(self):
"""
Sets up the testing environment;
saves config file;
creates a test user, a test project;
creates burst, operation, flow and workflow services
"""
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
self.workflow_service = WorkflowService()
self.burst_service = BurstService()
self.operation_service = OperationService()
self.flow_service = FlowService()
def create_adapter(algo_group=None, test_project=None, ):
"""
:returns: Adapter Class after initialization.
"""
if algo_group is None:
algo_group = dao.find_group('tvb.tests.framework.adapters.ndimensionarrayadapter', 'NDimensionArrayAdapter')
if test_project is None:
test_user = TestFactory.create_user()
test_project = TestFactory.create_project(test_user)
group, _ = FlowService().prepare_adapter(test_project.id, algo_group)
return FlowService().build_adapter_instance(group)
def test_launch_operation_HDD_with_space(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
self._assert_stored_dt2()
def test_launch_operation_HDD_full_space(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data) - 1)
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.assertRaises(NoMemoryAvailableException, self.operation_service.initiate_operation, self.test_user,
self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
self._assert_no_dt2()
def __init__(self, conf):
"""
:param conf: burst configuration entity
"""
self.logger = get_logger(__name__)
self.flow_service = FlowService()
self.conf = conf
def setUp(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
### Insert some starting data in the database.
categ1 = model.AlgorithmCategory('one', True)
self.categ1 = dao.store_entity(categ1)
categ2 = model.AlgorithmCategory('two', rawinput=True)
self.categ2 = dao.store_entity(categ2)
group1 = model.AlgorithmGroup("test_module1", "classname1", categ1.id)
self.algo_group1 = dao.store_entity(group1)
group2 = model.AlgorithmGroup("test_module2", "classname2", categ2.id)
self.algo_group2 = dao.store_entity(group2)
group3 = model.AlgorithmGroup("test_module3", "classname3", categ1.id)
self.algo_group3 = dao.store_entity(group3)
group_v = model.AlgorithmGroup(TEST_ADAPTER_VALID_MODULE, TEST_ADAPTER_VALID_CLASS, categ2.id)
self.algo_group_v = dao.store_entity(group_v)
algo_v = model.Algorithm(self.algo_group_v.id, 'ident', name='', req_data='', param_name='', output='')
self.algorithm_v = dao.store_entity(algo_v)
algo1 = model.Algorithm(self.algo_group1.id, 'id', name='', req_data='', param_name='', output='')
self.algorithm1 = dao.store_entity(algo1)
def __init__(self):
BaseController.__init__(self)
self.flow_service = FlowService()
self.logger = get_logger(__name__)
editable_entities = [dict(link='/spatial/stimulus/region/step_1_submit/1/1', title='Region Stimulus',
subsection='regionstim', description='Create a new Stimulus on Region level'),
dict(link='/spatial/stimulus/surface/step_1_submit/1/1', title='Surface Stimulus',
subsection='surfacestim', description='Create a new Stimulus on Surface level')]
self.submenu_list = editable_entities
def setUp(self):
"""
Reset the database before each test.
"""
self.clean_database()
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
self.operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
class _BaseLinksTest(TransactionalTestCase):
GEORGE1st = "george the grey"
GEORGE2nd = "george"
def _initialize_two_projects(self):
"""
Creates a user, an algorithm and 2 projects
Project src_project will have an operation and 2 datatypes
Project dest_project will be empty.
Initializes a flow and a project service
"""
self.datatype_factory_src = DatatypesFactory()
self.src_project = self.datatype_factory_src.project
self.src_usr_id = self.datatype_factory_src.user.id
self.red_datatype = self.datatype_factory_src.create_simple_datatype(subject=self.GEORGE1st)
self.blue_datatype = self.datatype_factory_src.create_datatype_with_storage(subject=self.GEORGE2nd)
# create the destination project
self.datatype_factory_dest = DatatypesFactory()
self.dest_project = self.datatype_factory_dest.project
self.dest_usr_id = self.datatype_factory_dest.user.id
self.flow_service = FlowService()
self.project_service = ProjectService()
def transactional_setup_method(self):
self.clean_database(delete_folders=True)
self._initialize_two_projects()
def transactional_teardown_method(self):
self.clean_database(delete_folders=True)
def red_datatypes_in(self, project_id):
return self.flow_service.get_available_datatypes(project_id, Datatype1)[1]
def blue_datatypes_in(self, project_id):
return self.flow_service.get_available_datatypes(project_id, Datatype2)[1]
def import_h5(file_path, project_id):
flow_service = FlowService()
## This ID of a project needs to exists in Db, and it can be taken from the WebInterface:
project = dao.get_project_by_id(project_id)
adapter_instance = ABCAdapter.build_adapter_from_class(TVBImporter)
## Prepare the input algorithms as if they were coming from web UI submit:
launch_args = {"data_file": file_path}
print "We will try to import file at path " + file_path
## launch an operation and have the results stored both in DB and on disk
launched_operations = flow_service.fire_operation(
adapter_instance, project.administrator, project.id, **launch_args
)
print "Operation launched. Check the web UI"
def setUp(self):
""" Prepare some entities to work with during tests:"""
self.flow_service = FlowService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(admin=self.test_user)
category = dao.get_uploader_categories()[0]
self.algorithm = dao.store_entity(model.Algorithm(TEST_ADAPTER_VALID_MODULE,
TEST_ADAPTER_VALID_CLASS, category.id))
def test_launch_operation_HDD_with_space(self):
"""
Test the actual operation flow by executing a test adapter.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
output = adapter.get_output()
output_type = output[0].__name__
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
dts = dao.get_values_of_datatype(self.test_project.id, Datatype2)[0]
self.assertEqual(len(dts), 1)
datatype = dao.get_datatype_by_id(dts[0][0])
self.assertEqual(datatype.subject, DataTypeMetaData.DEFAULT_SUBJECT, "Wrong data stored.")
self.assertEqual(datatype.type, output_type, "Wrong data stored.")
def test_delete_dt_free_HDD_space(self):
"""
Launch two operations and give enough available space for user so that both should finish.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = float(adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self._assert_no_dt2()
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
datatype = self._assert_stored_dt2()
# Now free some space and relaunch
ProjectService().remove_datatype(self.test_project.id, datatype.gid)
self._assert_no_dt2()
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter, tmp_folder, **data)
self._assert_stored_dt2()
def transactional_setup_method(self):
"""
Sets up the testing environment;
saves config file;
creates a test user, a test project;
creates burst, operation, flow and workflow services
"""
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user)
self.workflow_service = WorkflowService()
self.burst_service = BurstService()
self.operation_service = OperationService()
self.flow_service = FlowService()
def setUp(self):
"""
Prepare before each test.
"""
self.project_service = ProjectService()
self.flow_service = FlowService()
self.structure_helper = FilesHelper()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, "ProjectStructure")
self.relevant_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.RELEVANT_VIEW)
self.full_filter = StaticFiltersFactory.build_datatype_filters(single_filter=StaticFiltersFactory.FULL_VIEW)
def test_launch_two_ops_HDD_with_space(self):
"""
Launch two operations and give enough available space for user so that both should finish.
"""
module = "tvb.tests.framework.adapters.testadapter3"
class_name = "TestAdapterHDDRequired"
group = dao.find_group(module, class_name)
adapter = FlowService().build_adapter_instance(group)
data = {"test": 100}
TvbProfile.current.MAX_DISK_SPACE = 2 * float(adapter.get_required_disk_size(**data))
tmp_folder = FilesHelper().get_project_folder(self.test_project, "TEMP")
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
datatype = self._assert_stored_dt2()
#Now update the maximum disk size to be the size of the previously resulted datatypes (transform from kB to MB)
#plus what is estimated to be required from the next one (transform from B to MB)
TvbProfile.current.MAX_DISK_SPACE = float(datatype.disk_size) + float(adapter.get_required_disk_size(**data))
self.operation_service.initiate_operation(self.test_user, self.test_project.id, adapter,
tmp_folder, method_name=ABCAdapter.LAUNCH_METHOD, **data)
self._assert_stored_dt2(2)
def setUp(self):
"""
Reset the database before each test.
"""
self.import_service = ImportService()
self.flow_service = FlowService()
self.project_service = ProjectService()
self.test_user = TestFactory.create_user()
self.test_project = TestFactory.create_project(self.test_user, name="GeneratedProject", description="test_desc")
self.operation = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
self.adapter_instance = TestFactory.create_adapter(test_project=self.test_project)
TestFactory.import_cff(test_user=self.test_user, test_project=self.test_project)
self.zip_path = None
def __init__(self, overwrites=None, settings_file=None):
""" Parameters can be overwritten either from a settigns file or from a dictionary. """
if overwrites is not None:
self.overwrites.update(overwrites)
if settings_file is not None:
settings = open(sys.argv[1]).read()
for line in settings.split('\n'):
key, value = line.split('=')
self.overwrites[key.strip()] = value.strip()
if KEY_PROJECT not in self.overwrites:
raise Exception("Settings file should contain the id of the project: %s=1" % KEY_PROJECT)
self.project = dao.get_project_by_id(self.overwrites[KEY_PROJECT])
self.flow_service = FlowService()
self.operation_service = OperationService()
def launch_simulation_workflow(json_path, prj_id):
"""
:param json_path: Path towards a local JSON file exported from GUI
:param prj_id: This ID of a project needs to exists in DB, and it can be taken from the WebInterface
"""
project = dao.get_project_by_id(prj_id)
with open(json_path, 'rb') as input_file:
simulation_json = input_file.read()
simulation_json = json.loads(simulation_json)
LOG.info("Simulation JSON loaded from file '%s': \n %s", json_path, simulation_json)
importer = ImportService()
simulation_config = importer.load_burst_entity(simulation_json, prj_id)
LOG.info("Simulation Workflow configuration object loaded: \n %s", simulation_config)
flow_service = FlowService()
simulator_algorithm, _ = flow_service.get_algorithm_by_module_and_class(SIMULATOR_MODULE, SIMULATOR_CLASS)
LOG.info("Found Simulation algorithm in local DB: \n %s", simulator_algorithm)
burst_service = BurstService()
burst_service.launch_burst(simulation_config, 0, simulator_algorithm.id, project.administrator.id, LAUNCH_NEW)
LOG.info("Check in the web GUI for your operation. It should be starting now ...")
def setUp(self):
"""
Reset the database before each test.
"""
initialize_storage()
self.datatypes_factory = DatatypesFactory()
self.test_user = self.datatypes_factory.get_user()
self.test_project = self.datatypes_factory.get_project()
self.connectivity = self.datatypes_factory.create_connectivity(self.CONNECTIVITY_NODES)[1]
algo_group = dao.find_group(SIMULATOR_MODULE, SIMULATOR_CLASS)
algorithm = dao.get_algorithm_by_group(algo_group.id)
self.simulator_adapter = FlowService().build_adapter_instance(algo_group)
self.operation = TestFactory.create_operation(algorithm, self.test_user, self.test_project,
model.STATUS_STARTED, json.dumps(SIMULATOR_PARAMETERS))
SIMULATOR_PARAMETERS['connectivity'] = self.connectivity.gid