def build(user=None,
project=None,
op=None,
nr_regions=76,
monitor=TemporalAverageViewModel(),
with_surface=False,
conn_gid=None):
model = SimulatorAdapterModel()
model.monitors = [monitor]
if not op:
op = operation_factory(test_user=user, test_project=project)
if conn_gid:
model.connectivity = conn_gid
if not with_surface and not conn_gid:
model.connectivity = connectivity_index_factory(nr_regions, op).gid
model.simulation_length = 100
if with_surface:
rm_idx = region_mapping_index_factory()
model.connectivity = rm_idx.fk_connectivity_gid
model.surface = CortexViewModel()
model.surface.surface_gid = rm_idx.fk_surface_gid
model.surface.region_mapping_data = rm_idx.gid
model.simulation_length = 10
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(model, storage_path)
return storage_path, model.gid
def test_export_simulator_configuration(self, operation_factory,
connectivity_factory):
"""
Test export of a simulator configuration
"""
operation = operation_factory()
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_factory(4).gid
burst_configuration = BurstConfiguration(self.test_project.id)
burst_configuration.fk_simulation_id = operation.id
burst_configuration.simulator_gid = simulator.gid.hex
burst_configuration = dao.store_entity(burst_configuration)
storage_path = FilesHelper().get_project_folder(
self.test_project, str(operation.id))
h5_path = h5.path_for(storage_path, SimulatorH5, simulator.gid)
with SimulatorH5(h5_path) as h5_file:
h5_file.store(simulator)
export_file = self.export_manager.export_simulator_configuration(
burst_configuration.id)
assert export_file is not None, "Export process should return path to export file"
assert os.path.exists(
export_file
), "Could not find export file: %s on disk." % export_file
assert zipfile.is_zipfile(
export_file), "Generated file is not a valid ZIP file"
def test_fire_simulation(self):
simulation_api = SimulationApi(self.base_url)
session_stored_simulator = SimulatorAdapterModel()
TestFactory.import_zip_connectivity(self.test_user, self.test_project)
connectivity = TestFactory.get_entity(self.test_project, ConnectivityIndex)
session_stored_simulator.connectivity = UUID(connectivity.gid)
simulation_api.fire_simulation(self.test_project.gid, session_stored_simulator, None, self.temp_folder)
def deserialize_simulator(simulator_gid, storage_path):
simulator_in_path = h5.path_for(storage_path, SimulatorH5,
simulator_gid)
simulator_in = SimulatorAdapterModel()
with SimulatorH5(simulator_in_path) as simulator_in_h5:
simulator_in_h5.load_into(simulator_in)
simulator_in.connectivity = simulator_in_h5.connectivity.load()
simulator_in.stimulus = simulator_in_h5.stimulus.load()
simulator_in.history_gid = simulator_in_h5.simulation_state.load()
for monitor in simulator_in.monitors:
if isinstance(monitor, Projection):
with SimulatorH5(simulator_in_path) as simulator_in_h5:
monitor_h5_path = simulator_in_h5.get_reference_path(
monitor.gid)
monitor_h5_class = h5_factory.monitor_h5_factory(type(monitor))
with monitor_h5_class(monitor_h5_path) as monitor_h5:
sensors_gid = monitor_h5.sensors.load()
projection_gid = monitor_h5.projection.load()
region_mapping_gid = monitor_h5.region_mapping.load()
sensors_index = ABCAdapter.load_entity_by_gid(sensors_gid.hex)
projection_index = ABCAdapter.load_entity_by_gid(
projection_gid.hex)
sensors_class = monitor.projection_class().sensors.field_type
sensors = h5.load_from_index(sensors_index,
dt_class=sensors_class)
projection_class = monitor.projection_class()
projection = h5.load_from_index(projection_index,
dt_class=projection_class)
region_mapping = ABCAdapter.load_traited_by_gid(
region_mapping_gid)
monitor.sensors = sensors
monitor.projection = projection
monitor.region_mapping = region_mapping
if simulator_in.surface:
cortex_path = h5.path_for(storage_path, CortexH5,
simulator_in.surface.gid)
with CortexH5(cortex_path) as cortex_h5:
simulator_in.surface.local_connectivity = cortex_h5.local_connectivity.load(
)
simulator_in.surface.region_mapping_data = cortex_h5.region_mapping_data.load(
)
rm_index = dao.get_datatype_by_gid(
simulator_in.surface.region_mapping_data.hex)
simulator_in.surface.surface_gid = uuid.UUID(
rm_index.fk_surface_gid)
return simulator_in
def test_fire_simulation(self):
simulation_api = SimulationApi(self.base_url)
session_stored_simulator = SimulatorAdapterModel()
zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip')
TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John")
connectivity = TestFactory.get_entity(self.test_project, ConnectivityIndex)
session_stored_simulator.connectivity = UUID(connectivity.gid)
simulation_api.fire_simulation(self.test_project.gid, session_stored_simulator, None, self.temp_folder)
def deserialize_simulator(simulator_gid, storage_path):
simulator_in_path = h5.path_for(storage_path, SimulatorH5,
simulator_gid)
simulator_in = SimulatorAdapterModel()
with SimulatorH5(simulator_in_path) as simulator_in_h5:
simulator_in_h5.load_into(simulator_in)
simulator_in.connectivity = simulator_in_h5.connectivity.load()
simulator_in.stimulus = simulator_in_h5.stimulus.load()
simulator_in.history_gid = simulator_in_h5.simulation_state.load()
if isinstance(simulator_in.monitors[0], Projection):
# TODO: simplify this part
with SimulatorH5(simulator_in_path) as simulator_in_h5:
monitor_h5_path = simulator_in_h5.get_reference_path(
simulator_in.monitors[0].gid)
monitor_h5_class = h5_factory.monitor_h5_factory(
type(simulator_in.monitors[0]))
with monitor_h5_class(monitor_h5_path) as monitor_h5:
sensors_gid = monitor_h5.sensors.load()
region_mapping_gid = monitor_h5.region_mapping.load()
sensors = ABCAdapter.load_traited_by_gid(sensors_gid)
if isinstance(simulator_in.monitors[0], EEG):
sensors = SensorsEEG.build_sensors_subclass(sensors)
elif isinstance(simulator_in.monitors[0], MEG):
sensors = SensorsMEG.build_sensors_subclass(sensors)
elif isinstance(simulator_in.monitors[0], iEEG):
sensors = SensorsInternal.build_sensors_subclass(sensors)
simulator_in.monitors[0].sensors = sensors
region_mapping = ABCAdapter.load_traited_by_gid(region_mapping_gid)
simulator_in.monitors[0].region_mapping = region_mapping
if simulator_in.surface:
cortex_path = h5.path_for(storage_path, CortexH5,
simulator_in.surface.gid)
with CortexH5(cortex_path) as cortex_h5:
simulator_in.surface.local_connectivity = cortex_h5.local_connectivity.load(
)
simulator_in.surface.region_mapping_data = cortex_h5.region_mapping_data.load(
)
rm_index = dao.get_datatype_by_gid(
simulator_in.surface.region_mapping_data.hex)
simulator_in.surface.surface_gid = uuid.UUID(
rm_index.surface_gid)
return simulator_in
def test_estimate_hdd(self, connectivity_index_factory):
"""
Test that occupied HDD estimation for simulation results considers simulation length.
"""
factor = 5
model = SimulatorAdapterModel()
model.connectivity = connectivity_index_factory(
self.CONNECTIVITY_NODES).gid
estimate1 = self._estimate_hdd(model)
assert estimate1 > 1
## Change simulation length and monitor period, we expect a direct proportial increase in estimated HDD
model.simulation_length = float(model.simulation_length) * factor
period = float(model.monitors[0].period)
model.monitors[0].period = period / factor
estimate2 = self._estimate_hdd(model)
assert estimate1 == estimate2 // factor // factor
## Change number of nodes in connectivity. Expect HDD estimation increase.
model.connectivity = connectivity_index_factory(
self.CONNECTIVITY_NODES * factor).gid
estimate3 = self._estimate_hdd(model)
assert estimate2 == estimate3 / factor
def build(test_user, test_project, simulation_length=10, is_group=False):
model = SimulatorAdapterModel()
model.connectivity = connectivity_index_factory().gid
model.simulation_length = simulation_length
burst = BurstConfiguration(test_project.id, name="Sim " + str(datetime.now()))
burst.start_time = datetime.now()
algorithm = dao.get_algorithm_by_module(SIMULATOR_MODULE, SIMULATOR_CLASS)
service = SimulatorService()
if is_group:
range_param = RangeParameter("conduction_speed", float, Range(lo=50.0, hi=100.0, step=20.0))
burst.range1 = range_param.to_json()
burst = BurstService().prepare_burst_for_pse(burst)
op = service.async_launch_and_prepare_pse(burst, test_user, test_project, algorithm,
range_param, None, model)
else:
dao.store_entity(burst)
op = service.async_launch_and_prepare_simulation(burst, test_user, test_project, algorithm, model)
return op
def test_happy_flow_launch(self, connectivity_index_factory,
operation_factory):
"""
Test that launching a simulation from UI works.
"""
model = SimulatorAdapterModel()
model.connectivity = connectivity_index_factory(
self.CONNECTIVITY_NODES).gid
model.simulation_length = 32
self.operation = operation_factory()
# TODO: should store model in H5 and keep GID as param on operation to fix this
OperationService().initiate_prelaunch(self.operation,
self.simulator_adapter)
sim_result = dao.get_generic_entity(TimeSeriesRegionIndex,
'TimeSeriesRegion',
'time_series_type')[0]
assert (sim_result.data_length_1d, sim_result.data_length_2d,
sim_result.data_length_3d,
sim_result.data_length_4d) == (32, 1, self.CONNECTIVITY_NODES,
1)
def fire_simulation_example(tvb_client_instance):
logger.info("Requesting projects for logged user")
projects_of_user = tvb_client_instance.get_project_list()
assert len(projects_of_user) > 0
logger.info("TVB has {} projects for this user".format(
len(projects_of_user)))
project_gid = projects_of_user[0].gid
logger.info("Requesting datatypes from project {}...".format(project_gid))
data_in_project = tvb_client_instance.get_data_in_project(project_gid)
logger.info("We have {} datatypes".format(len(data_in_project)))
logger.info("Requesting operations from project {}...".format(project_gid))
ops_in_project, _ = tvb_client_instance.get_operations_in_project(
project_gid, 1)
logger.info("Displayname of the first operation is: {}".format(
ops_in_project[0].displayname))
connectivity_gid = None
datatypes_type = []
for datatype in data_in_project:
datatypes_type.append(datatype.type)
if datatype.type == ConnectivityIndex().display_type:
connectivity_gid = datatype.gid
logger.info("The datatypes in project are: {}".format(datatypes_type))
if connectivity_gid:
logger.info("Preparing the simulator...")
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_gid
simulator.simulation_length = 100
logger.info("Starting the simulation...")
operation_gid = tvb_client_instance.fire_simulation(
project_gid, simulator)
logger.info("Monitoring the simulation operation...")
monitor_operation(tvb_client_instance, operation_gid)
logger.info("Requesting the results of the simulation...")
simulation_results = tvb_client_instance.get_operation_results(
operation_gid)
datatype_names = []
for datatype in simulation_results:
datatype_names.append(datatype.name)
logger.info("The resulted datatype are: {}".format(datatype_names))
time_series_gid = simulation_results[1].gid
logger.info("Download the time series file...")
time_series_path = tvb_client_instance.retrieve_datatype(
time_series_gid, tvb_client_instance.temp_folder)
logger.info(
"The time series file location is: {}".format(time_series_path))
logger.info("Requesting algorithms to run on time series...")
algos = tvb_client_instance.get_operations_for_datatype(
time_series_gid)
algo_names = [algo.displayname for algo in algos]
logger.info("Possible algorithms are {}".format(algo_names))
logger.info("Launch Fourier Analyzer...")
fourier_model = FFTAdapterModel()
fourier_model.time_series = time_series_gid
fourier_model.window_function = 'hamming'
operation_gid = tvb_client_instance.launch_operation(
project_gid, FourierAdapter, fourier_model)
logger.info(
"Fourier Analyzer operation has launched with gid {}".format(
operation_gid))
logger.info("Download the connectivity file...")
connectivity_path = tvb_client_instance.retrieve_datatype(
connectivity_gid, tvb_client_instance.temp_folder)
logger.info(
"The connectivity file location is: {}".format(connectivity_path))
logger.info("Loading an entire Connectivity datatype in memory...")
connectivity = tvb_client_instance.load_datatype_from_file(
connectivity_path)
logger.info("Info on current Connectivity: {}".format(
connectivity.summary_info()))
logger.info(
"Loading a chuck from the time series H5 file, as this can be very large..."
)
with TimeSeriesH5(time_series_path) as time_series_h5:
data_shape = time_series_h5.read_data_shape()
chunk = time_series_h5.read_data_slice(
tuple([
slice(20),
slice(data_shape[1]),
slice(data_shape[2]),
slice(data_shape[3])
]))
assert chunk.shape[0] == 20
assert chunk.shape[1] == data_shape[1]
assert chunk.shape[2] == data_shape[2]
assert chunk.shape[3] == data_shape[3]
return project_gid, time_series_gid
logger.info("Requesting datatypes from project {}...".format(project_gid))
data_in_project = tvb_client.get_data_in_project(project_gid)
logger.info("We have {} datatypes".format(len(data_in_project)))
connectivity_gid = None
datatypes_type = []
for datatype in data_in_project:
datatypes_type.append(datatype.type)
if datatype.type == ConnectivityIndex().display_type:
connectivity_gid = datatype.gid
logger.info("The datatypes in projecct are: {}".format(datatypes_type))
if connectivity_gid:
logger.info("Preparing the simulator...")
simulator = SimulatorAdapterModel()
simulator.connectivity = uuid.UUID(connectivity_gid)
simulator.simulation_length = 100
logger.info("Starting the simulation...")
operation_gid = tvb_client.fire_simulation(project_gid, simulator)
logger.info("Monitoring the simulation operation...")
while True:
status = tvb_client.get_operation_status(operation_gid)
if status in [STATUS_FINISHED, STATUS_CANCELED, STATUS_ERROR]:
break
time.sleep(20)
logger.info(
"The simulation has finished with status: {}".format(status))
logger.info("Requesting the results of the simulation...")
logger.info("Requesting datatypes from project {}...".format(project_gid))
data_in_project = tvb_client.get_data_in_project(project_gid)
logger.info("We have {} datatypes".format(len(data_in_project)))
connectivity_gid = None
datatypes_type = []
for datatype in data_in_project:
datatypes_type.append(datatype.type)
if datatype.type == ConnectivityIndex().display_type:
connectivity_gid = datatype.gid
logger.info("The datatypes in project are: {}".format(datatypes_type))
if connectivity_gid:
logger.info("Preparing the simulator...")
simulator = SimulatorAdapterModel()
simulator.connectivity = connectivity_gid
simulator.simulation_length = 100
logger.info("Starting the simulation...")
operation_gid = tvb_client.fire_simulation(project_gid, simulator)
logger.info("Monitoring the simulation operation...")
monitor_operation(tvb_client, operation_gid)
logger.info("Requesting the results of the simulation...")
simulation_results = tvb_client.get_operation_results(operation_gid)
datatype_names = []
for datatype in simulation_results:
datatype_names.append(datatype.name)
logger.info("The resulted datatype are: {}".format(datatype_names))
def test_estimate_execution_time(self, connectivity_index_factory):
"""
Test that get_execution_time_approximation considers the correct params
"""
## Compute reference estimation
self.test_user = TestFactory.create_user("Simulator_Adapter_User")
self.test_project = TestFactory.create_project(
self.test_user, "Simulator_Adapter_Project")
simulator_adapter_model = SimulatorAdapterModel()
connectivity = connectivity_index_factory(76)
simulator_adapter_model.connectivity = connectivity.gid
self.simulator_adapter.configure(simulator_adapter_model)
estimation1 = self.simulator_adapter.get_execution_time_approximation(
simulator_adapter_model)
# import surfaceData
cortex_data = path.join(path.dirname(tvb_data.__file__), 'surfaceData',
'cortex_16384.zip')
surface = TestFactory.import_surface_zip(self.test_user,
self.test_project,
cortex_data, CORTICAL)
cortex_model = CortexViewModel()
# import region mapping for cortex_model (surface)
text_file = path.join(path.dirname(tvb_data.__file__), 'regionMapping',
'regionMapping_16k_76.txt')
region_mapping = TestFactory.import_region_mapping(
self.test_user, self.test_project, text_file, surface.gid,
connectivity.gid)
cortex_model.region_mapping_data = region_mapping.gid
cortex_model.fk_surface_gid = surface.gid
simulator_adapter_model.surface = cortex_model
## Estimation when the surface input parameter is set
self.simulator_adapter.configure(simulator_adapter_model)
estimation2 = self.simulator_adapter.get_execution_time_approximation(
simulator_adapter_model)
assert estimation1 == estimation2 // 500
simulator_adapter_model.surface = None
## Modify integration step and simulation length:
initial_simulation_length = simulator_adapter_model.simulation_length
initial_integration_step = simulator_adapter_model.integrator.dt
for factor in (2, 4, 10):
simulator_adapter_model.simulation_length = initial_simulation_length * factor
simulator_adapter_model.integrator.dt = initial_integration_step / factor
self.simulator_adapter.configure(simulator_adapter_model)
estimation3 = self.simulator_adapter.get_execution_time_approximation(
simulator_adapter_model)
assert estimation1 == estimation3 // factor // factor
## Check that no division by zero happens
simulator_adapter_model.integrator.dt = 0
estimation4 = self.simulator_adapter.get_execution_time_approximation(
simulator_adapter_model)
assert estimation4 > 0
## even with length zero, still a positive estimation should be returned
simulator_adapter_model.simulation_length = 0
estimation5 = self.simulator_adapter.get_execution_time_approximation(
simulator_adapter_model)
assert estimation5 > 0