def test_store_simulator_view_model_eeg(connectivity_index_factory, surface_index_factory, region_mapping_factory,
sensors_index_factory, operation_factory):
conn = connectivity_index_factory()
surface_idx, surface = surface_index_factory(cortical=True)
region_mapping = region_mapping_factory()
sensors_idx, sensors = sensors_index_factory()
proj = ProjectionSurfaceEEG(sensors=sensors, sources=surface, projection_data=numpy.ones(3))
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
prj_db_db = h5.store_complete(proj, storage_path)
prj_db_db.fk_from_operation = op.id
dao.store_entity(prj_db_db)
seeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid)
seeg_monitor.region_mapping = region_mapping.gid.hex
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
sim_view_model.monitors = [seeg_monitor]
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
loaded_sim_view_model = h5.load_view_model(sim_view_model.gid, storage_path)
assert isinstance(sim_view_model, SimulatorAdapterModel)
assert isinstance(loaded_sim_view_model, SimulatorAdapterModel)
assert sim_view_model.monitors[0].projection == loaded_sim_view_model.monitors[0].projection
def test_server_fire_simulation(self, mocker, connectivity_factory):
self._mock_user(mocker)
input_folder = self.storage_interface.get_project_folder(self.test_project.name)
sim_dir = os.path.join(input_folder, 'test_sim')
if not os.path.isdir(sim_dir):
os.makedirs(sim_dir)
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_factory().gid
h5.store_view_model(simulator, sim_dir)
zip_filename = os.path.join(input_folder, RequestFileKey.SIMULATION_FILE_NAME.value)
self.storage_interface.write_zip_folder(zip_filename, sim_dir)
# Mock flask.request.files to return a dictionary
request_mock = mocker.patch.object(flask, 'request', spec={})
fp = open(zip_filename, 'rb')
request_mock.files = {RequestFileKey.SIMULATION_FILE_KEY.value: FileStorage(fp, os.path.basename(zip_filename))}
def launch_sim(self, user_id, project, algorithm, zip_folder_path, simulator_file):
return Operation('', '', '', {})
# Mock simulation launch and current user
mocker.patch.object(SimulatorService, 'prepare_simulation_on_server', launch_sim)
operation_gid, status = self.simulation_resource.post(project_gid=self.test_project.gid)
fp.close()
assert type(operation_gid) is str
assert status == 201
def test_happy_flow_launch(self, connectivity_index_factory, operation_factory):
"""
Test that launching a simulation from UI works.
"""
model = SimulatorAdapterModel()
model.connectivity = connectivity_index_factory(self.CONNECTIVITY_NODES).gid
model.simulation_length = 32
TestFactory.launch_synchronously(self.test_user.id, self.test_project, self.simulator_adapter, model)
sim_result = dao.get_generic_entity(TimeSeriesRegionIndex, 'TimeSeriesRegion', 'time_series_type')[0]
assert (sim_result.data_length_1d, sim_result.data_length_2d, sim_result.data_length_3d,
sim_result.data_length_4d) == (32, 1, self.CONNECTIVITY_NODES, 1)
def test_store_simulator_view_model(connectivity_index_factory, operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
loaded_sim_view_model = h5.load_view_model(sim_view_model.gid, storage_path)
assert isinstance(sim_view_model, SimulatorAdapterModel)
assert isinstance(loaded_sim_view_model, SimulatorAdapterModel)
def test_update_simulation_fields(self, tmph5factory):
"""
Test that updates the simulation fields of the burst
"""
stored_burst = TestFactory.store_burst(self.test_project.id)
connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project)
op = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)
simulation = SimulatorAdapterModel()
simulation.connectivity = UUID(connectivity.gid)
burst_config = self.burst_service.update_simulation_fields(stored_burst, op.id, simulation.gid)
assert burst_config.id == stored_burst.id, "The loaded burst does not have the same ID"
assert burst_config.fk_simulation == op.id, "The loaded burst does not have the fk simulation that it was given"
assert burst_config.simulator_gid == simulation.gid.hex, "The loaded burst does not have the simulation gid that it was given"
def test_store_simulator_view_model_noise(connectivity_index_factory, operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
sim_view_model.integrator = HeunStochasticViewModel()
sim_view_model.integrator.noise.noise_seed = 45
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
loaded_sim_view_model = h5.load_view_model(sim_view_model.gid, storage_path)
assert isinstance(sim_view_model, SimulatorAdapterModel)
assert isinstance(loaded_sim_view_model, SimulatorAdapterModel)
assert sim_view_model.integrator.noise.noise_seed == loaded_sim_view_model.integrator.noise.noise_seed == 45
def test_gather_view_model_references(connectivity_index_factory,
operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
only_vm_references = h5.gather_view_model_references(
sim_view_model.gid, storage_path, True)
all_references = h5.gather_view_model_references(sim_view_model.gid,
storage_path)
assert len(only_vm_references) == 5
assert len(all_references) == 6
def init_session_at_sim_reset(self, burst_config, last_loaded_form_url):
# type: (BurstConfiguration, str) -> None
self.set_burst_config(burst_config)
self.set_simulator(SimulatorAdapterModel())
self.add_last_loaded_form_url_to_session(last_loaded_form_url)
common.add2session(self.KEY_IS_SIMULATOR_COPY, False)
common.add2session(self.KEY_IS_SIMULATOR_LOAD, False)
common.add2session(self.KEY_IS_SIMULATOR_BRANCH, False)
def set_simulator(self, simulator=None):
# type: (SimulatorAdapterModel) -> None
"""
Create a new simulator instance only if one does not exist in the context.
"""
if not simulator and not self.simulator:
simulator = SimulatorAdapterModel()
if simulator:
common.add2session(self.KEY_SIMULATOR_CONFIG, simulator)
def test_gather_view_model_and_datatype_references(connectivity_index_factory,
operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
op = operation_factory()
storage_path = StorageInterface().get_project_folder(
op.project.name, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
only_vm_references, _ = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path, True)
vm_references, dt_references = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path)
assert len(only_vm_references) == 5
assert len(vm_references) + len(dt_references) == 6
def run_simulation():
log = get_logger(__name__)
# This ID of a project needs to exists in DB, and it can be taken from the WebInterface:
project = dao.get_project_by_id(1)
# Find a structural Connectivity and load it in memory
connectivity_index = dao.get_generic_entity(
ConnectivityIndex, DataTypeMetaData.DEFAULT_SUBJECT, "subject")[0]
connectivity = h5.load_from_index(connectivity_index)
# Load the SimulatorAdapter algorithm from DB
cached_simulator_algorithm = AlgorithmService(
).get_algorithm_by_module_and_class(IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS)
# Instantiate a SimulatorService and launch the configured simulation
simulator_model = SimulatorAdapterModel()
simulator_model.connectivity = connectivity.gid
simulator_model.simulation_length = 100
simulator_model.coupling = Scaling()
simulator_service = SimulatorService()
burst = BurstConfiguration(project.id, name="Simulation")
dao.store_entity(burst)
launched_operation = simulator_service.async_launch_and_prepare_simulation(
burst, project.administrator, project, cached_simulator_algorithm,
simulator_model)
# wait for the operation to finish
while not launched_operation.has_finished:
sleep(5)
launched_operation = dao.get_operation_by_id(launched_operation.id)
if launched_operation.status == STATUS_FINISHED:
ts = dao.get_generic_entity(TimeSeriesRegionIndex,
launched_operation.id,
"fk_from_operation")[0]
log.info("TimeSeries result is: %s " % ts)
else:
log.warning(
"Operation ended with problems [%s]: [%s]" %
(launched_operation.status, launched_operation.additional_info))
def test_export_simulator_configuration(self, operation_factory, connectivity_factory):
"""
Test export of a simulator configuration
"""
operation = operation_factory()
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_factory(4).gid
burst_configuration = BurstConfiguration(self.test_project.id)
burst_configuration.fk_simulation = operation.id
burst_configuration.simulator_gid = simulator.gid.hex
burst_configuration = dao.store_entity(burst_configuration)
storage_path = FilesHelper().get_project_folder(self.test_project, str(operation.id))
h5.store_view_model(simulator, storage_path)
export_file = self.export_manager.export_simulator_configuration(burst_configuration.id)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(export_file), "Could not find export file: %s on disk." % export_file
assert zipfile.is_zipfile(export_file), "Generated file is not a valid ZIP file"
def test_estimate_hdd(self, connectivity_index_factory):
"""
Test that occupied HDD estimation for simulation results considers simulation length.
"""
model = SimulatorAdapterModel()
model.connectivity = connectivity_index_factory(self.CONNECTIVITY_NODES).gid
estimate1 = self._estimate_hdd(model)
assert estimate1 > 1
# Change simulation length and monitor period, we expect a direct proportional increase in estimated HDD
factor = 3
model.simulation_length = float(model.simulation_length) * factor
period = float(model.monitors[0].period)
model.monitors[0].period = period / factor
estimate2 = self._estimate_hdd(model)
assert estimate1 == estimate2 // factor // factor
# Change number of nodes in connectivity. Expect HDD estimation increase.
model.connectivity = connectivity_index_factory(self.CONNECTIVITY_NODES * factor).gid
estimate3 = self._estimate_hdd(model)
assert estimate2 == estimate3 / factor
def test_gather_view_model_and_datatype_references_multiple_monitors(
connectivity_index_factory, operation_factory, sensors_index_factory,
surface_index_factory, region_mapping_index_factory):
conn = connectivity_index_factory()
_, surface = surface_index_factory(cortical=True)
region_mapping_idx = region_mapping_index_factory(
conn_gid=conn.gid, surface_gid=surface.gid.hex)
sensors_idx, sensors = sensors_index_factory()
proj = ProjectionSurfaceEEG(sensors=sensors,
sources=surface,
projection_data=numpy.ones(3))
op = operation_factory()
storage_path = StorageInterface().get_project_folder(
op.project.name, str(op.id))
prj_db = h5.store_complete(proj, storage_path)
prj_db.fk_from_operation = op.id
dao.store_entity(prj_db)
seeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid)
seeg_monitor.region_mapping = region_mapping_idx.gid
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
sim_view_model.monitors = [TemporalAverageViewModel(), seeg_monitor]
op = operation_factory()
storage_path = StorageInterface().get_project_folder(
op.project.name, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
only_vm_references, _ = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path, True)
assert len(only_vm_references) == 7
vm_references, dt_references = h5.gather_references_of_view_model(
sim_view_model.gid, storage_path)
assert len(vm_references + dt_references) == 12
def test_estimate_execution_time(self, connectivity_index_factory):
"""
Test that get_execution_time_approximation considers the correct params
"""
model = SimulatorAdapterModel()
model.connectivity = connectivity_index_factory(self.CONNECTIVITY_NODES).gid
self.simulator_adapter.configure(model)
estimation1 = self.simulator_adapter.get_execution_time_approximation(model)
# import surfaceData and region mapping
cortex_file = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip')
surface = TestFactory.import_surface_zip(self.test_user, self.test_project, cortex_file, CORTICAL)
rm_file = path.join(path.dirname(tvb_data.regionMapping.__file__), 'regionMapping_16k_76.txt')
region_mapping = TestFactory.import_region_mapping(self.test_user, self.test_project, rm_file, surface.gid,
model.connectivity.hex)
local_conn = TestFactory.create_local_connectivity(self.test_user, self.test_project, surface.gid)
cortex_model = CortexViewModel()
cortex_model.region_mapping_data = region_mapping.gid
cortex_model.fk_surface_gid = surface.gid
cortex_model.local_connectivity = local_conn.gid
model.surface = cortex_model
# Estimation when the surface input parameter is set
self.simulator_adapter.configure(model)
estimation2 = self.simulator_adapter.get_execution_time_approximation(model)
assert estimation1 == estimation2 // 500
model.surface = None
# Modify integration step and simulation length:
initial_simulation_length = model.simulation_length
initial_integration_step = model.integrator.dt
for factor in (2, 4, 10):
model.simulation_length = initial_simulation_length * factor
model.integrator.dt = initial_integration_step / factor
self.simulator_adapter.configure(model)
estimation3 = self.simulator_adapter.get_execution_time_approximation(model)
assert estimation1 == estimation3 // factor // factor
# Check that no division by zero happens
model.integrator.dt = 0
estimation4 = self.simulator_adapter.get_execution_time_approximation(model)
assert estimation4 > 0
# even with length zero, still a positive estimation should be returned
model.simulation_length = 0
estimation5 = self.simulator_adapter.get_execution_time_approximation(model)
assert estimation5 > 0
def fire_simulation(project_id, simulator):
project = dao.get_project_by_id(project_id)
assert isinstance(simulator, Simulator)
# Load the SimulatorAdapter algorithm from DB
cached_simulator_algorithm = AlgorithmService(
).get_algorithm_by_module_and_class(IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS)
simulator_model = SimulatorAdapterModel()
simulator_model.connectivity = simulator.connectivity.gid
simulator_model.simulation_length = simulator.simulation_length
# Instantiate a SimulatorService and launch the configured simulation
simulator_service = SimulatorService()
burst = BurstConfiguration(project.id)
burst.name = "Sim " + str(datetime.now())
burst.start_time = datetime.now()
dao.store_entity(burst)
launched_operation = simulator_service.async_launch_and_prepare_simulation(
burst, project.administrator, project, cached_simulator_algorithm,
simulator_model)
LOG.info("Operation launched ....")
return launched_operation
def fire_simulation_example(tvb_client_instance):
logger.info("Requesting projects for logged user")
projects_of_user = tvb_client_instance.get_project_list()
assert len(projects_of_user) > 0
logger.info("TVB has {} projects for this user".format(len(projects_of_user)))
project_gid = projects_of_user[0].gid
logger.info("Requesting datatypes from project {}...".format(project_gid))
data_in_project = tvb_client_instance.get_data_in_project(project_gid)
logger.info("We have {} datatypes".format(len(data_in_project)))
logger.info("Requesting operations from project {}...".format(project_gid))
ops_in_project, _ = tvb_client_instance.get_operations_in_project(project_gid, 1)
logger.info("Displayname of the first operation is: {}".format(ops_in_project[0].displayname))
connectivity_gid = None
datatypes_type = []
for datatype in data_in_project:
datatypes_type.append(datatype.type)
if datatype.type == ConnectivityIndex().display_type:
connectivity_gid = datatype.gid
logger.info("The datatypes in project are: {}".format(datatypes_type))
if connectivity_gid:
logger.info("Preparing the simulator...")
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_gid
simulator.simulation_length = 100
logger.info("Starting the simulation...")
operation_gid = tvb_client_instance.fire_simulation(project_gid, simulator)
logger.info("Monitoring the simulation operation...")
monitor_operation(tvb_client_instance, operation_gid)
logger.info("Requesting the results of the simulation...")
simulation_results = tvb_client_instance.get_operation_results(operation_gid)
datatype_names = []
for datatype in simulation_results:
datatype_names.append(datatype.name)
logger.info("The resulted datatype are: {}".format(datatype_names))
time_series_gid = simulation_results[1].gid
logger.info("Download the time series file...")
time_series_path = tvb_client_instance.retrieve_datatype(time_series_gid, tvb_client_instance.temp_folder)
logger.info("The time series file location is: {}".format(time_series_path))
logger.info("Requesting algorithms to run on time series...")
algos = tvb_client_instance.get_operations_for_datatype(time_series_gid)
algo_names = [algo.displayname for algo in algos]
logger.info("Possible algorithms are {}".format(algo_names))
logger.info("Launch Fourier Analyzer...")
fourier_model = FFTAdapterModel()
fourier_model.time_series = time_series_gid
fourier_model.window_function = 'hamming'
operation_gid = tvb_client_instance.launch_operation(project_gid, FourierAdapter, fourier_model)
logger.info("Fourier Analyzer operation has launched with gid {}".format(operation_gid))
data_in_project = tvb_client_instance.get_data_in_project(project_gid)
logger.info("We have {} datatypes".format(len(data_in_project)))
for datatype in data_in_project:
if datatype.type == 'FourierSpectrum':
ggid = datatype.gid
extra_info = tvb_client_instance.get_extra_info(ggid)
logger.info("The extra information for Fourier {}".format(extra_info))
break
logger.info("Download the connectivity file...")
connectivity_path = tvb_client_instance.retrieve_datatype(connectivity_gid, tvb_client_instance.temp_folder)
logger.info("The connectivity file location is: {}".format(connectivity_path))
logger.info("Loading an entire Connectivity datatype in memory...")
connectivity = tvb_client_instance.load_datatype_from_file(connectivity_path)
logger.info("Info on current Connectivity: {}".format(connectivity.summary_info()))
logger.info("Loading a chuck from the time series H5 file, as this can be very large...")
with TimeSeriesH5(time_series_path) as time_series_h5:
data_shape = time_series_h5.read_data_shape()
chunk = time_series_h5.read_data_slice(
tuple([slice(20), slice(data_shape[1]), slice(data_shape[2]), slice(data_shape[3])]))
assert chunk.shape[0] == 20
assert chunk.shape[1] == data_shape[1]
assert chunk.shape[2] == data_shape[2]
assert chunk.shape[3] == data_shape[3]
return project_gid, time_series_gid