def load_burst_read_only(self, burst_config_id):
try:
burst_config = self.burst_service.load_burst_configuration(
burst_config_id)
storage_path = StorageInterface().get_project_folder(
self.context.project.name, str(burst_config.fk_simulation))
simulator = h5.load_view_model(burst_config.simulator_gid,
storage_path)
last_loaded_form_url = self.get_url_for_final_fragment(
burst_config)
self.context.init_session_at_burst_loading(burst_config, simulator,
last_loaded_form_url)
form = self.prepare_first_fragment()
self.monitors_handler.build_list_of_monitors_from_view_models(
self.context.simulator)
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
is_simulation_readonly_load=True,
is_first_fragment=True)
return rendering_rules.to_dict()
except Exception:
# Most probably Burst was removed. Delete it from session, so that client
# has a good chance to get a good response on refresh
self.logger.exception("Error loading burst")
self.context.remove_burst_config_from_session()
raise
def do_operation_launch(simulator_gid,
available_disk_space,
is_group_launch,
base_url,
operation_id,
plain_dir='/root/plain'):
try:
log.info("Preparing HPC launch for simulation with id={}".format(
simulator_gid))
populate_datatypes_registry()
log.info("Current TVB profile has HPC run=: {}".format(
TvbProfile.current.hpc.IS_HPC_RUN))
storage_interface = StorageInterface()
_request_passfile(simulator_gid, operation_id, base_url,
storage_interface.get_password_file(simulator_gid))
storage_interface.decrypt_results_to_dir(simulator_gid, plain_dir)
log.info("Current wdir is: {}".format(plain_dir))
view_model = h5.load_view_model(simulator_gid, plain_dir)
adapter_instance = HPCSimulatorAdapter(plain_dir, is_group_launch)
_update_operation_status(STATUS_STARTED, simulator_gid, operation_id,
base_url)
adapter_instance._prelaunch(None, view_model, available_disk_space)
_encrypt_results(adapter_instance, storage_interface, simulator_gid)
_update_operation_status(STATUS_FINISHED, simulator_gid, operation_id,
base_url)
except Exception as excep:
log.error("Could not execute operation {}".format(str(sys.argv[1])))
log.exception(excep)
_update_operation_status(STATUS_ERROR, simulator_gid, operation_id,
base_url)
raise excep
def _run_cluster_job(operation_identifier, user_name_label, adapter_instance):
"""
Threaded Popen
It is the function called by the ClusterSchedulerClient in a Thread.
This function starts a new process.
"""
# Load operation so we can estimate the execution time
operation = dao.get_operation_by_id(operation_identifier)
view_model = h5.load_view_model(operation)
# kwargs = adapter_instance.prepare_ui_inputs(kwargs)
time_estimate = int(adapter_instance.get_execution_time_approximation(view_model))
hours = int(time_estimate / 3600)
minutes = (int(time_estimate) % 3600) / 60
seconds = int(time_estimate) % 60
# Anything lower than 5 hours just use default walltime
if hours < 5:
walltime = "05:00:00"
else:
if hours < 10:
hours = "0%d" % hours
else:
hours = str(hours)
walltime = "%s:%s:%s" % (hours, str(minutes), str(seconds))
call_arg = TvbProfile.current.cluster.SCHEDULE_COMMAND % (operation_identifier, user_name_label, walltime)
LOGGER.info(call_arg)
process_ = Popen([call_arg], stdout=PIPE, shell=True)
job_id = process_.stdout.read().replace('\n', '').split(TvbProfile.current.cluster.JOB_ID_STRING)[-1]
LOGGER.info("Got jobIdentifier = %s for CLUSTER operationID = %s" % (job_id, operation_identifier))
operation_identifier = OperationProcessIdentifier(operation_identifier, job_id=job_id)
dao.store_entity(operation_identifier)
def test_store_simulator_view_model_eeg(connectivity_index_factory, surface_index_factory, region_mapping_factory,
sensors_index_factory, operation_factory):
conn = connectivity_index_factory()
surface_idx, surface = surface_index_factory(cortical=True)
region_mapping = region_mapping_factory()
sensors_idx, sensors = sensors_index_factory()
proj = ProjectionSurfaceEEG(sensors=sensors, sources=surface, projection_data=numpy.ones(3))
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
prj_db_db = h5.store_complete(proj, storage_path)
prj_db_db.fk_from_operation = op.id
dao.store_entity(prj_db_db)
seeg_monitor = EEGViewModel(projection=proj.gid, sensors=sensors.gid)
seeg_monitor.region_mapping = region_mapping.gid.hex
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
sim_view_model.monitors = [seeg_monitor]
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
loaded_sim_view_model = h5.load_view_model(sim_view_model.gid, storage_path)
assert isinstance(sim_view_model, SimulatorAdapterModel)
assert isinstance(loaded_sim_view_model, SimulatorAdapterModel)
assert sim_view_model.monitors[0].projection == loaded_sim_view_model.monitors[0].projection
def prepare_data_for_burst_copy(self, burst_config_id, burst_name_format, project):
burst_config = self.load_burst_configuration(burst_config_id)
burst_config_copy = burst_config.clone()
count = dao.count_bursts_with_name(burst_config.name, burst_config.fk_project)
burst_config_copy.name = burst_name_format.format(burst_config.name, count + 1)
storage_path = self.storage_interface.get_project_folder(project.name, str(burst_config.fk_simulation))
simulator = h5.load_view_model(burst_config.simulator_gid, storage_path)
simulator.generic_attributes = GenericAttributes()
return simulator, burst_config_copy
def test_store_simulator_view_model(connectivity_index_factory, operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
loaded_sim_view_model = h5.load_view_model(sim_view_model.gid, storage_path)
assert isinstance(sim_view_model, SimulatorAdapterModel)
assert isinstance(loaded_sim_view_model, SimulatorAdapterModel)
def test_store_simulator_view_model_noise(connectivity_index_factory, operation_factory):
conn = connectivity_index_factory()
sim_view_model = SimulatorAdapterModel()
sim_view_model.connectivity = conn.gid
sim_view_model.integrator = HeunStochasticViewModel()
sim_view_model.integrator.noise.noise_seed = 45
op = operation_factory()
storage_path = FilesHelper().get_project_folder(op.project, str(op.id))
h5.store_view_model(sim_view_model, storage_path)
loaded_sim_view_model = h5.load_view_model(sim_view_model.gid, storage_path)
assert isinstance(sim_view_model, SimulatorAdapterModel)
assert isinstance(loaded_sim_view_model, SimulatorAdapterModel)
assert sim_view_model.integrator.noise.noise_seed == loaded_sim_view_model.integrator.noise.noise_seed == 45
def load_view_model(self, operation):
storage_path = self.storage_interface.get_project_folder(operation.project.name, str(operation.id))
input_gid = operation.view_model_gid
return h5.load_view_model(input_gid, storage_path)
def load_view_model(self, operation):
storage_path = self.file_handler.get_project_folder(operation.project, str(operation.id))
input_gid = operation.view_model_gid
return h5.load_view_model(input_gid, storage_path)
def load_view_model(self, operation):
storage_path = self.file_handler.get_project_folder(
operation.project, str(operation.id))
input_gid = json.loads(operation.parameters)['gid']
return h5.load_view_model(input_gid, storage_path)
