def do_operation_launch(operation_id):
"""
Event attached to the local queue for executing an operation, when we will have resources available.
"""
log = get_logger('tvb.core.operation_async_launcher')
burst_service = BurstService()
try:
log.debug("Loading operation with id=%s" % operation_id)
curent_operation = dao.get_operation_by_id(operation_id)
stored_adapter = curent_operation.algorithm
log.debug("Importing Algorithm: " + str(stored_adapter.classname) +
" for Operation:" + str(curent_operation.id))
adapter_instance = ABCAdapter.build_adapter(stored_adapter)
# Un-comment bellow for profiling an operation:
# import cherrypy.lib.profiler as profiler
# p = profiler.Profiler("/Users/lia.domide/TVB/profiler/")
# p.run(OperationService().initiate_prelaunch, curent_operation, adapter_instance, {}, **PARAMS)
OperationService().initiate_prelaunch(curent_operation,
adapter_instance)
if curent_operation.fk_operation_group:
parent_burst = dao.get_generic_entity(
BurstConfiguration, curent_operation.fk_operation_group,
'fk_operation_group')[0]
operations_in_group = dao.get_operations_in_group(
curent_operation.fk_operation_group)
if parent_burst.fk_metric_operation_group:
operations_in_group.extend(
dao.get_operations_in_group(
parent_burst.fk_metric_operation_group))
burst_finished = True
for operation in operations_in_group:
if not has_finished(operation.status):
burst_finished = False
break
if burst_finished and parent_burst is not None and parent_burst.status != BurstConfiguration.BURST_ERROR:
burst_service.mark_burst_finished(parent_burst)
else:
parent_burst = burst_service.get_burst_for_operation_id(
operation_id)
if parent_burst is not None:
burst_service.mark_burst_finished(parent_burst)
log.debug("Successfully finished operation " + str(operation_id))
except Exception as excep:
log.error("Could not execute operation " + str(operation_id))
log.exception(excep)
parent_burst = burst_service.get_burst_for_operation_id(operation_id)
if parent_burst is not None:
burst_service.mark_burst_finished(parent_burst,
error_message=str(excep))
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.files_helper = FilesHelper()
def _reset_model(self, session_stored_simulator):
session_stored_simulator.model = type(session_stored_simulator.model)()
vi_indexes = MonitorForm.determine_indexes_for_chosen_vars_of_interest(
session_stored_simulator)
vi_indexes = numpy.array(list(vi_indexes.values()))
for monitor in session_stored_simulator.monitors:
monitor.variables_of_interest = vi_indexes
def reset_at_connectivity_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise
parameters because they might not fit to the new Connectivity's nr of regions.
"""
if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity:
self._reset_model(session_stored_simulator)
if issubclass(type(session_stored_simulator.integrator),
IntegratorStochastic):
session_stored_simulator.integrator.noise = type(
session_stored_simulator.integrator.noise)()
def reset_at_surface_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a surface-simulation and changes the Surface, we want to reset the Model
parameters because they might not fit to the new Surface's nr of vertices.
"""
if is_simulator_copy and (
session_stored_simulator.surface is None and form.surface.value
or session_stored_simulator.surface and form.surface.value !=
session_stored_simulator.surface.surface_gid):
self._reset_model(session_stored_simulator)
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo,
session_stored_simulator):
try:
operation = self.operation_service.prepare_operation(
user.id, project.id, simulator_algo,
session_stored_simulator.gid)
ga = self.operation_service._prepare_metadata(
simulator_algo.algorithm_category, {}, None, burst_config.gid)
session_stored_simulator.generic_attributes = ga
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
h5.store_view_model(session_stored_simulator, storage_path)
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, operation.id, session_stored_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
simulator_vm = h5.load_view_model_from_file(simulator_file)
operation = self.operation_service.prepare_operation(
user_id, project.id, algorithm, simulator_vm.gid)
storage_operation_path = self.files_helper.get_project_folder(
project, str(operation.id))
self.async_launch_simulation_on_server(operation, zip_folder_path,
storage_operation_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path,
storage_operation_path):
try:
for file in os.listdir(zip_folder_path):
shutil.move(os.path.join(zip_folder_path, file),
storage_operation_path)
try:
OperationService().launch_operation(operation.id, True)
shutil.rmtree(zip_folder_path)
return operation
except Exception as excep:
self.logger.error(excep)
except Exception as excep:
self.logger.error(excep)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
operations = []
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
first_simulator = None
ga = self.operation_service._prepare_metadata(
simulator_algo.algorithm_category, {}, operation_group,
burst_config.gid)
session_stored_simulator.generic_attributes = ga
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(
simulator, range_param1.name, param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[
range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self.operation_service.prepare_operation(
user.id, project.id, simulator_algo, simulator.gid,
operation_group, ranges)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
h5.store_view_model(simulator, storage_path)
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
first_operation = operations[0]
storage_path = self.files_helper.get_project_folder(
project, str(first_operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, first_operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=first_operation.id,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group, fk_parent_burst=burst_config.gid)
dao.store_entity(metrics_datatype_group)
wf_errs = 0
for operation in operations:
try:
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
return first_operation
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def load_from_zip(self, zip_file, project):
import_service = ImportService()
simulator_folder = import_service.import_simulator_configuration_zip(
zip_file)
simulator_h5_filename = DirLoader(
simulator_folder,
None).find_file_for_has_traits_type(SimulatorAdapterModel)
simulator_h5_filepath = os.path.join(simulator_folder,
simulator_h5_filename)
simulator = h5.load_view_model_from_file(simulator_h5_filepath)
burst_config = self.burst_service.load_burst_configuration_from_folder(
simulator_folder, project)
return simulator, burst_config
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.algorithm_service = AlgorithmService()
self.storage_interface = StorageInterface()
@staticmethod
def _reset_model(session_stored_simulator):
session_stored_simulator.model = type(session_stored_simulator.model)()
vi_indexes = session_stored_simulator.determine_indexes_for_chosen_vars_of_interest(
)
vi_indexes = numpy.array(list(vi_indexes.values()))
for monitor in session_stored_simulator.monitors:
monitor.variables_of_interest = vi_indexes
def reset_at_connectivity_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise
parameters because they might not fit to the new Connectivity's nr of regions.
"""
if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity:
self._reset_model(session_stored_simulator)
if issubclass(type(session_stored_simulator.integrator),
IntegratorStochastic):
session_stored_simulator.integrator.noise = type(
session_stored_simulator.integrator.noise)()
def reset_at_surface_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a surface-simulation and changes the Surface, we want to reset the Model
parameters because they might not fit to the new Surface's nr of vertices.
"""
if is_simulator_copy and (
session_stored_simulator.surface is None and form.surface.value
or session_stored_simulator.surface and form.surface.value !=
session_stored_simulator.surface.surface_gid):
self._reset_model(session_stored_simulator)
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo, simulator):
try:
operation = self.operation_service.prepare_operation(
user.id,
project,
simulator_algo,
view_model=simulator,
burst_gid=burst_config.gid,
op_group_id=burst_config.fk_operation_group)
burst_config = self.burst_service.update_simulation_fields(
burst_config, operation.id, simulator.gid)
storage_path = self.storage_interface.get_project_folder(
project.name, str(operation.id))
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
simulator_vm = h5.load_view_model_from_file(simulator_file)
operation = self.operation_service.prepare_operation(
user_id, project, algorithm, view_model=simulator_vm)
self.async_launch_simulation_on_server(operation, zip_folder_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path):
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
finally:
shutil.rmtree(zip_folder_path)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
GROUP_BURST_PENDING[burst_config.id] = True
operations, pse_canceled = self._prepare_operations(
algo_category, burst_config, metric_operation_group,
operation_group, project, range_param1, range_param2,
range_param2_values, session_stored_simulator, simulator_algo,
user)
GROUP_BURST_PENDING[burst_config.id] = False
if pse_canceled:
return
wf_errs = self._launch_operations(operations, burst_config)
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
return operations[0] if len(operations) > 0 else None
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def _launch_operations(self, operations, burst_config):
wf_errs = 0
for operation in operations:
try:
burst_config = dao.get_burst_by_id(burst_config.id)
if burst_config is None or burst_config.status in [
BurstConfiguration.BURST_CANCELED,
BurstConfiguration.BURST_ERROR
]:
self.logger.debug(
"Preparing operations cannot continue. Burst config {}"
.format(burst_config))
return
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
return wf_errs
def _prepare_operations(self, algo_category, burst_config,
metric_operation_group, operation_group, project,
range_param1, range_param2, range_param2_values,
session_stored_simulator, simulator_algo, user):
first_simulator = None
pse_canceled = False
operations = []
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
burst_config = dao.get_burst_by_id(burst_config.id)
if burst_config is None:
self.logger.debug("Burst config was deleted")
pse_canceled = True
break
if burst_config.status in [
BurstConfiguration.BURST_CANCELED,
BurstConfiguration.BURST_ERROR
]:
self.logger.debug(
"Current burst status is {}. Preparing operations cannot continue."
.format(burst_config.status))
pse_canceled = True
break
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(simulator,
range_param1.name,
param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self.operation_service.prepare_operation(
user.id,
project,
simulator_algo,
view_model=simulator,
ranges=ranges,
burst_gid=burst_config.gid,
op_group_id=burst_config.fk_operation_group)
simulator.range_values = ranges
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
storage_path = self.storage_interface.get_project_folder(
project.name, str(operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config, operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=operation.id,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(metrics_datatype_group)
return operations, pse_canceled
@staticmethod
def compute_conn_branch_conditions(is_branch, simulator):
if not is_branch:
return None
conn = load.load_entity_by_gid(simulator.connectivity)
if conn.number_of_regions:
return FilterChain(
fields=[FilterChain.datatype + '.number_of_regions'],
operations=["=="],
values=[conn.number_of_regions])
@staticmethod
def validate_first_fragment(form, project_id, conn_idx):
conn_count = dao.count_datatypes(project_id, conn_idx)
if conn_count == 0:
form.connectivity.errors.append(
"No connectivity in the project! Simulation cannot be started without "
"a connectivity!")
def get_simulation_state_index(self, burst_config,
simulation_history_class):
parent_burst = burst_config.parent_burst_object
simulation_state_index = dao.get_generic_entity(
simulation_history_class, parent_burst.gid, "fk_parent_burst")
if simulation_state_index is None or len(simulation_state_index) < 1:
exc = BurstServiceException(
"Simulation State not found for %s, thus we are unable to branch from "
"it!" % burst_config.name)
self.logger.error(exc)
raise exc
return simulation_state_index
class SimulatorController(BurstBaseController):
KEY_IS_LOAD_AFTER_REDIRECT = "is_load_after_redirect"
COPY_NAME_FORMAT = "copy_of_{}"
BRANCH_NAME_FORMAT = "{}_branch{}"
KEY_KEEP_SAME_SIM_WIZARD = "keep_same_wizard"
def __init__(self):
BurstBaseController.__init__(self)
self.range_parameters = SimulatorRangeParameters()
self.burst_service = BurstService()
self.simulator_service = SimulatorService()
self.cached_simulator_algorithm = self.algorithm_service.get_algorithm_by_module_and_class(
IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS)
self.context = SimulatorContext()
self.monitors_handler = MonitorsWizardHandler()
@expose_json
def cancel_or_remove_burst(self, burst_id):
"""
Cancel or Remove the burst entity given by burst_id (and all linked entities: op, DTs)
:returns True: if the op was successfully.
"""
burst_config = BurstService.load_burst_configuration(int(burst_id))
op_id, is_group = burst_config.operation_info_for_burst_removal
return self.cancel_or_remove_operation(op_id, is_group,
burst_config.is_finished)
def cancel_or_remove_operation(self,
operation_id,
is_group,
remove_after_stop=False):
"""
Stop the operation given by operation_id. If is_group is true stop all the
operations from that group.
"""
# Load before we remove, to have its data in memory here
burst_config = BurstService.get_burst_for_operation_id(
operation_id, is_group)
if burst_config is not None:
self.burst_service.mark_burst_finished(
burst_config,
BurstConfiguration.BURST_CANCELED,
store_h5_file=False)
while GROUP_BURST_PENDING.get(burst_config.id, False):
pass
GROUP_BURST_PENDING.pop(burst_config.id, False)
result = OperationService.stop_operation(operation_id, is_group,
remove_after_stop)
if remove_after_stop:
current_burst = self.context.burst_config
if (current_burst is not None and burst_config is not None
and current_burst.id == burst_config.id and
((current_burst.fk_simulation == operation_id and not is_group)
or (current_burst.fk_operation_group == operation_id
and is_group))):
self.reset_simulator_configuration()
if burst_config is not None:
burst_config = BurstService.load_burst_configuration(
burst_config.id)
if burst_config:
BurstService.remove_burst_configuration(burst_config.id)
return result
@expose_page
@settings
@context_selected
def index(self):
"""Get on burst main page"""
template_specification = dict(
mainContent="burst/main_burst",
title="Simulation Cockpit",
includedResources='project/included_resources')
if not self.context.last_loaded_fragment_url:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_CONNECTIVITY_URL)
self.context.set_burst_config()
_, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if self.context.burst_config.start_time is not None:
is_simulation_load = True
self.context.add_simulator_load_to_session(True)
template_specification['burstConfig'] = self.context.burst_config
template_specification[
'burst_list'] = self.burst_service.get_available_bursts(
self.context.project.id)
form = self.prepare_first_fragment()
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
None,
is_simulation_copy,
is_simulation_load,
last_form_url=self.context.last_loaded_fragment_url,
last_request_type=cherrypy.request.method,
is_first_fragment=True,
is_branch=is_branch)
template_specification.update(**rendering_rules.to_dict())
cherrypy.response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
cherrypy.response.headers['Pragma'] = 'no-cache'
cherrypy.response.headers['Expires'] = '0'
return self.fill_default_attributes(template_specification)
def prepare_first_fragment(self):
self.context.set_simulator()
simulator, _, _, is_branch = self.context.get_common_params()
branch_conditions = self.simulator_service.compute_conn_branch_conditions(
is_branch, simulator)
form = self.algorithm_service.prepare_adapter_form(
form_instance=SimulatorAdapterForm(),
project_id=self.context.project.id,
extra_conditions=branch_conditions)
self.simulator_service.validate_first_fragment(form,
self.context.project.id,
ConnectivityIndex)
form.fill_from_trait(self.context.simulator)
return form
@expose_json
def set_fragment_url(self, **data):
try:
self.context.add_last_loaded_form_url_to_session(data['url'])
except KeyError:
self.logger.error(
"Cannot set last loaded url to session because the required data was not found."
)
@expose_fragment('simulator_fragment')
def set_connectivity(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_COUPLING_PARAMS_URL)
form = SimulatorAdapterForm()
form.fill_from_post(data)
self.simulator_service.reset_at_connectivity_change(
is_simulation_copy, form, session_stored_simulator)
form.fill_trait(session_stored_simulator)
next_form = self.algorithm_service.prepare_adapter_form(
form_instance=get_form_for_coupling(
type(session_stored_simulator.coupling))())
self.range_parameters.coupling_parameters = next_form.get_range_parameters(
)
next_form.fill_from_trait(session_stored_simulator.coupling)
rendering_rules = SimulatorFragmentRenderingRules(
next_form, SimulatorWizzardURLs.SET_COUPLING_PARAMS_URL,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL, is_simulation_copy,
is_simulation_load, self.context.last_loaded_fragment_url,
cherrypy.request.method)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_coupling_params(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_SURFACE_URL)
form = get_form_for_coupling(
type(session_stored_simulator.coupling))()
form.fill_from_post(data)
form.fill_trait(session_stored_simulator.coupling)
surface_fragment = self.algorithm_service.prepare_adapter_form(
form_instance=SimulatorSurfaceFragment(),
project_id=self.context.project.id)
surface_fragment.fill_from_trait(session_stored_simulator.surface)
rendering_rules = SimulatorFragmentRenderingRules(
surface_fragment,
SimulatorWizzardURLs.SET_SURFACE_URL,
SimulatorWizzardURLs.SET_COUPLING_PARAMS_URL,
is_simulation_copy,
is_simulation_load,
self.context.last_loaded_fragment_url,
cherrypy.request.method,
is_branch=is_branch)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_surface(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
rendering_rules = SimulatorFragmentRenderingRules(
previous_form_action_url=SimulatorWizzardURLs.SET_SURFACE_URL,
is_simulation_copy=is_simulation_copy,
is_simulation_readonly_load=is_simulation_load,
last_form_url=self.context.last_loaded_fragment_url,
last_request_type=cherrypy.request.method,
is_branch=is_branch)
if cherrypy.request.method == POST_REQUEST:
form = SimulatorSurfaceFragment()
form.fill_from_post(data)
self.simulator_service.reset_at_surface_change(
is_simulation_copy, form, session_stored_simulator)
form.fill_trait(session_stored_simulator)
if session_stored_simulator.surface:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_CORTEX_URL)
else:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_STIMULUS_URL)
return SimulatorSurfaceFragment.prepare_next_fragment_after_surface(
session_stored_simulator, rendering_rules, self.context.project.id,
SimulatorWizzardURLs.SET_CORTEX_URL,
SimulatorWizzardURLs.SET_STIMULUS_URL)
@expose_fragment('simulator_fragment')
def set_cortex(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_STIMULUS_URL)
rm_fragment = SimulatorRMFragment()
rm_fragment.fill_from_post(data)
rm_fragment.fill_trait(session_stored_simulator.surface)
rendering_rules = SimulatorFragmentRenderingRules(
None, None, SimulatorWizzardURLs.SET_CORTEX_URL,
is_simulation_copy, is_simulation_load,
self.context.last_loaded_fragment_url, cherrypy.request.method)
return SimulatorStimulusFragment.prepare_stimulus_fragment(
session_stored_simulator, rendering_rules, True,
SimulatorWizzardURLs.SET_STIMULUS_URL, self.context.project.id)
@expose_fragment('simulator_fragment')
def set_stimulus(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_MODEL_URL)
stimuli_fragment = SimulatorStimulusFragment(
session_stored_simulator.is_surface_simulation)
stimuli_fragment.fill_from_post(data)
stimuli_fragment.fill_trait(session_stored_simulator)
model_fragment = self.algorithm_service.prepare_adapter_form(
form_instance=SimulatorModelFragment())
model_fragment.fill_from_trait(session_stored_simulator)
rendering_rules = SimulatorFragmentRenderingRules(
model_fragment,
SimulatorWizzardURLs.SET_MODEL_URL,
SimulatorWizzardURLs.SET_STIMULUS_URL,
is_simulation_copy,
is_simulation_load,
self.context.last_loaded_fragment_url,
cherrypy.request.method,
is_model_fragment=True,
is_surface_simulation=session_stored_simulator.
is_surface_simulation,
is_branch=is_branch)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_model(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
set_next_wizard = True
if SimulatorController.KEY_KEEP_SAME_SIM_WIZARD in data:
set_next_wizard = False
if set_next_wizard:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_MODEL_PARAMS_URL)
form = SimulatorModelFragment()
form.fill_from_post(data)
form.fill_trait(session_stored_simulator)
form = self.algorithm_service.prepare_adapter_form(
form_instance=get_form_for_model(
type(session_stored_simulator.model))(is_branch))
self.range_parameters.model_parameters = form.get_range_parameters()
form.fill_from_trait(session_stored_simulator.model)
rendering_rules = SimulatorFragmentRenderingRules(
form, SimulatorWizzardURLs.SET_MODEL_PARAMS_URL,
SimulatorWizzardURLs.SET_MODEL_URL, is_simulation_copy,
is_simulation_load, self.context.last_loaded_fragment_url,
cherrypy.request.method)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_model_params(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_INTEGRATOR_URL)
form = get_form_for_model(type(
session_stored_simulator.model))(is_branch)
if is_branch:
data['variables_of_interest'] = list(
session_stored_simulator.model.variables_of_interest)
form.fill_from_post(data)
form.fill_trait(session_stored_simulator.model)
integrator_fragment = self.algorithm_service.prepare_adapter_form(
form_instance=SimulatorIntegratorFragment())
integrator_fragment.integrator.display_subform = False
integrator_fragment.fill_from_trait(session_stored_simulator)
rendering_rules = SimulatorFragmentRenderingRules(
integrator_fragment,
SimulatorWizzardURLs.SET_INTEGRATOR_URL,
SimulatorWizzardURLs.SET_MODEL_PARAMS_URL,
is_simulation_copy,
is_simulation_load,
self.context.last_loaded_fragment_url,
cherrypy.request.method,
is_branch=is_branch)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_integrator(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_INTEGRATOR_PARAMS_URL)
fragment = SimulatorIntegratorFragment()
fragment.fill_from_post(data)
fragment.fill_trait(session_stored_simulator)
form = self.algorithm_service.prepare_adapter_form(
form_instance=get_form_for_integrator(
type(session_stored_simulator.integrator))(is_branch))
if hasattr(form, 'noise'):
form.noise.display_subform = False
form.fill_from_trait(session_stored_simulator.integrator)
rendering_rules = SimulatorFragmentRenderingRules(
form, SimulatorWizzardURLs.SET_INTEGRATOR_PARAMS_URL,
SimulatorWizzardURLs.SET_INTEGRATOR_URL, is_simulation_copy,
is_simulation_load, self.context.last_loaded_fragment_url,
cherrypy.request.method)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_integrator_params(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
form = get_form_for_integrator(
type(session_stored_simulator.integrator))(is_branch)
if is_branch:
data['dt'] = str(session_stored_simulator.integrator.dt)
form.fill_from_post(data)
form.fill_trait(session_stored_simulator.integrator)
if isinstance(session_stored_simulator.integrator,
IntegratorStochasticViewModel):
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_NOISE_PARAMS_URL)
else:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_MONITORS_URL)
rendering_rules = SimulatorFragmentRenderingRules(
None,
None,
SimulatorWizzardURLs.SET_INTEGRATOR_PARAMS_URL,
is_simulation_copy,
is_simulation_load,
self.context.last_loaded_fragment_url,
cherrypy.request.method,
is_noise_fragment=False)
if not isinstance(session_stored_simulator.integrator,
IntegratorStochasticViewModel):
return self.monitors_handler.prepare_monitor_fragment(
session_stored_simulator, rendering_rules,
SimulatorWizzardURLs.SET_MONITORS_URL)
integrator_noise_fragment = get_form_for_noise(
type(session_stored_simulator.integrator.noise))()
if hasattr(integrator_noise_fragment, 'equation'):
integrator_noise_fragment.equation.display_subform = False
self.range_parameters.integrator_noise_parameters = integrator_noise_fragment.get_range_parameters(
)
integrator_noise_fragment.fill_from_trait(
session_stored_simulator.integrator.noise)
rendering_rules.form = integrator_noise_fragment
rendering_rules.form_action_url = SimulatorWizzardURLs.SET_NOISE_PARAMS_URL
rendering_rules.is_noise_fragment = True
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_noise_params(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
form = get_form_for_noise(
type(session_stored_simulator.integrator.noise))()
form.fill_from_post(data)
form.fill_trait(session_stored_simulator.integrator.noise)
if isinstance(session_stored_simulator.integrator.noise,
AdditiveNoiseViewModel):
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_MONITORS_URL)
else:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_NOISE_EQUATION_PARAMS_URL)
rendering_rules = SimulatorFragmentRenderingRules(
None, None, SimulatorWizzardURLs.SET_NOISE_PARAMS_URL,
is_simulation_copy, is_simulation_load,
self.context.last_loaded_fragment_url, cherrypy.request.method)
return self.monitors_handler.prepare_next_fragment_after_noise(
session_stored_simulator, is_branch, rendering_rules,
SimulatorWizzardURLs.SET_MONITORS_URL,
SimulatorWizzardURLs.SET_NOISE_EQUATION_PARAMS_URL)
@expose_fragment('simulator_fragment')
def set_noise_equation_params(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_MONITORS_URL)
form = get_form_for_equation(
type(session_stored_simulator.integrator.noise.b))()
form.fill_from_post(data)
form.fill_trait(session_stored_simulator.integrator.noise.b)
rendering_rules = SimulatorFragmentRenderingRules(
None, None, SimulatorWizzardURLs.SET_NOISE_EQUATION_PARAMS_URL,
is_simulation_copy, is_simulation_load,
self.context.last_loaded_fragment_url, cherrypy.request.method)
return self.monitors_handler.prepare_monitor_fragment(
session_stored_simulator, rendering_rules,
SimulatorWizzardURLs.SET_MONITORS_URL)
@staticmethod
def build_monitor_url(fragment_url, monitor):
url_regex = '{}/{}'
url = url_regex.format(fragment_url, monitor)
return url
def get_first_monitor_fragment_url(self, simulator, monitors_url):
first_monitor = simulator.first_monitor
if first_monitor is not None:
monitor_vm_name = type(first_monitor).__name__
return self.build_monitor_url(monitors_url, monitor_vm_name)
return SimulatorWizzardURLs.SETUP_PSE_URL
@expose_fragment('simulator_fragment')
def set_monitors(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, is_branch = self.context.get_common_params(
)
if cherrypy.request.method == POST_REQUEST:
fragment = SimulatorMonitorFragment(
is_surface_simulation=session_stored_simulator.
is_surface_simulation)
fragment.fill_from_post(data)
self.monitors_handler.set_monitors_list_on_simulator(
session_stored_simulator, fragment.monitors.value)
last_loaded_fragment_url = self.get_first_monitor_fragment_url(
session_stored_simulator,
SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL)
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
last_loaded_fragment_url)
rendering_rules = SimulatorFragmentRenderingRules(
is_simulation_copy=is_simulation_copy,
is_simulation_readonly_load=is_simulation_load,
last_request_type=cherrypy.request.method,
form_action_url=last_loaded_fragment_url,
previous_form_action_url=SimulatorWizzardURLs.SET_MONITORS_URL)
return self.monitors_handler.get_fragment_after_monitors(
session_stored_simulator, self.context.burst_config,
self.context.project.id, is_branch, rendering_rules,
SimulatorWizzardURLs.SETUP_PSE_URL)
def get_url_after_monitors(self, current_monitor, monitor_name,
next_monitor):
if isinstance(current_monitor, BoldViewModel):
return self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_EQUATION_URL, monitor_name)
if next_monitor is not None:
return self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL,
type(next_monitor).__name__)
return SimulatorWizzardURLs.SETUP_PSE_URL
@staticmethod
def get_url_for_final_fragment(burst_config):
if burst_config.is_pse_burst():
return SimulatorWizzardURLs.LAUNCH_PSE_URL
return SimulatorWizzardURLs.SETUP_PSE_URL
def get_urls_for_next_monitor_fragment(self, next_monitor,
current_monitor):
form_action_url = self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL,
type(next_monitor).__name__)
if_bold_url = self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_EQUATION_URL,
type(current_monitor).__name__)
return form_action_url, if_bold_url
@expose_fragment('simulator_fragment')
def set_monitor_params(self, current_monitor_name, **data):
session_stored_simulator, is_simulation_copy, is_simulator_load, is_branch = self.context.get_common_params(
)
current_monitor, next_monitor = self.monitors_handler.get_current_and_next_monitor_form(
current_monitor_name, session_stored_simulator)
if cherrypy.request.method == POST_REQUEST:
form = get_form_for_monitor(type(current_monitor))(
session_stored_simulator, is_branch)
if is_branch:
data['period'] = str(current_monitor.period)
data['variables_of_interest'] = [
session_stored_simulator.model.variables_of_interest[i]
for i in current_monitor.variables_of_interest
]
form.fill_from_post(data)
form.fill_trait(current_monitor)
last_loaded_form_url = self.get_url_after_monitors(
current_monitor, current_monitor_name, next_monitor)
self.context.add_last_loaded_form_url_to_session(
last_loaded_form_url)
previous_form_action_url = self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL, current_monitor_name)
rendering_rules = SimulatorFragmentRenderingRules(
is_simulation_copy=is_simulation_copy,
is_simulation_readonly_load=is_simulator_load,
last_request_type=cherrypy.request.method,
last_form_url=self.context.last_loaded_fragment_url,
previous_form_action_url=previous_form_action_url)
form_action_url, if_bold_url = self.get_urls_for_next_monitor_fragment(
next_monitor, current_monitor)
self.monitors_handler.update_monitor(current_monitor)
return self.monitors_handler.handle_next_fragment_for_monitors(
self.context, rendering_rules, current_monitor, next_monitor,
False, form_action_url, if_bold_url)
def get_url_after_monitor_equation(self, next_monitor):
if next_monitor is None:
return SimulatorWizzardURLs.SETUP_PSE_URL
last_loaded_fragment_url = self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_PARAMS_URL,
type(next_monitor).__name__)
return last_loaded_fragment_url
@expose_fragment('simulator_fragment')
def set_monitor_equation(self, current_monitor_name, **data):
session_stored_simulator, is_simulation_copy, is_simulator_load, is_branch = self.context.get_common_params(
)
current_monitor, next_monitor = self.monitors_handler.get_current_and_next_monitor_form(
current_monitor_name, session_stored_simulator)
if cherrypy.request.method == POST_REQUEST:
form = get_form_for_equation(type(current_monitor.hrf_kernel))()
form.fill_from_post(data)
form.fill_trait(current_monitor.hrf_kernel)
last_loaded_fragment_url = self.get_url_after_monitor_equation(
next_monitor)
self.context.add_last_loaded_form_url_to_session(
last_loaded_fragment_url)
previous_form_action_url = self.build_monitor_url(
SimulatorWizzardURLs.SET_MONITOR_EQUATION_URL,
current_monitor_name)
rendering_rules = SimulatorFragmentRenderingRules(
None, None, previous_form_action_url, is_simulation_copy,
is_simulator_load, self.context.last_loaded_fragment_url,
cherrypy.request)
form_action_url, if_bold_url = self.get_urls_for_next_monitor_fragment(
next_monitor, current_monitor)
return self.monitors_handler.handle_next_fragment_for_monitors(
self.context, rendering_rules, current_monitor, next_monitor, True,
form_action_url, if_bold_url)
@expose_fragment('simulator_fragment')
def setup_pse(self, **data):
session_stored_simulator, is_simulation_copy, is_simulator_load, _ = self.context.get_common_params(
)
burst_config = self.context.burst_config
all_range_parameters = self.range_parameters.get_all_range_parameters()
next_form = self.algorithm_service.prepare_adapter_form(
form_instance=SimulatorPSEConfigurationFragment(
self.range_parameters.get_all_range_parameters()))
if cherrypy.request.method == POST_REQUEST:
session_stored_simulator.simulation_length = float(
data['simulation_length'])
burst_config.name = data['input_simulation_name_id']
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.SET_PSE_PARAMS_URL)
param1, param2 = self.burst_service.handle_range_params_at_loading(
burst_config, all_range_parameters)
if param1:
param_dict = {'pse_param1': param1.name}
if param2 is not None:
param_dict['pse_param2'] = param2.name
next_form.fill_from_post(param_dict)
rendering_rules = SimulatorFragmentRenderingRules(
next_form, SimulatorWizzardURLs.SET_PSE_PARAMS_URL,
SimulatorWizzardURLs.SETUP_PSE_URL, is_simulation_copy,
is_simulator_load, self.context.last_loaded_fragment_url,
cherrypy.request.method)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def set_pse_params(self, **data):
session_stored_simulator, is_simulation_copy, is_simulation_load, _ = self.context.get_common_params(
)
burst_config = self.context.burst_config
form = SimulatorPSEConfigurationFragment(
self.range_parameters.get_all_range_parameters())
if cherrypy.request.method == POST_REQUEST:
self.context.add_last_loaded_form_url_to_session(
SimulatorWizzardURLs.LAUNCH_PSE_URL)
form.fill_from_post(data)
param1 = form.pse_param1.value
burst_config.range1 = param1.to_json()
param2 = None
if form.pse_param2.value:
param2 = form.pse_param2.value
burst_config.range2 = param2.to_json()
else:
all_range_parameters = self.range_parameters.get_all_range_parameters(
)
param1, param2 = self.burst_service.handle_range_params_at_loading(
burst_config, all_range_parameters)
next_form = self.algorithm_service.prepare_adapter_form(
form_instance=SimulatorPSERangeFragment(param1, param2))
rendering_rules = SimulatorFragmentRenderingRules(
next_form,
SimulatorWizzardURLs.LAUNCH_PSE_URL,
SimulatorWizzardURLs.SET_PSE_PARAMS_URL,
is_simulation_copy,
is_simulation_load,
last_form_url=self.context.last_loaded_fragment_url,
is_launch_pse_fragment=True)
return rendering_rules.to_dict()
@expose_json
def launch_pse(self, **data):
session_stored_simulator = self.context.simulator
all_range_parameters = self.range_parameters.get_all_range_parameters()
range_param1, range_param2 = SimulatorPSERangeFragment.fill_from_post(
all_range_parameters, **data)
burst_config = self.context.burst_config
burst_config.start_time = datetime.now()
burst_config.range1 = range_param1.to_json()
if range_param2:
burst_config.range2 = range_param2.to_json()
burst_config = self.burst_service.prepare_burst_for_pse(burst_config)
session_stored_simulator.operation_group_gid = uuid.UUID(
burst_config.operation_group.gid)
session_stored_simulator.ranges = json.dumps(burst_config.ranges)
try:
thread = threading.Thread(
target=self.simulator_service.async_launch_and_prepare_pse,
kwargs={
'burst_config': burst_config,
'user': self.context.logged_user,
'project': self.context.project,
'simulator_algo': self.cached_simulator_algorithm,
'range_param1': range_param1,
'range_param2': range_param2,
'session_stored_simulator': session_stored_simulator
})
thread.start()
return {'id': burst_config.id}
except BurstServiceException as e:
self.logger.exception("Could not launch burst!")
return {'error': e.message}
@expose_json
def launch_simulation(self, launch_mode, **data):
current_form = SimulatorFinalFragment()
burst_config = self.context.burst_config
burst_config.range1 = None
burst_config.range2 = None
try:
current_form.fill_from_post(data)
except Exception as exc:
self.logger.exception(exc)
return {'error': str(exc)}
burst_name = current_form.simulation_name.value
session_stored_simulator = self.context.simulator
session_stored_simulator.simulation_length = current_form.simulation_length.value
if burst_name != 'none_undefined':
burst_config.name = burst_name
if launch_mode == self.burst_service.LAUNCH_BRANCH:
simulation_state_index = self.simulator_service.get_simulation_state_index(
burst_config, SimulationHistoryIndex)
session_stored_simulator.history_gid = simulation_state_index[
0].gid
burst_config.start_time = datetime.now()
session_burst_config = self.burst_service.store_burst(burst_config)
try:
thread = threading.Thread(target=self.simulator_service.
async_launch_and_prepare_simulation,
kwargs={
'burst_config': session_burst_config,
'user': self.context.logged_user,
'project': self.context.project,
'simulator_algo':
self.cached_simulator_algorithm,
'simulator': session_stored_simulator
})
thread.start()
return {'id': session_burst_config.id}
except BurstServiceException as e:
self.logger.exception('Could not launch burst!')
return {'error': e.message}
@expose_fragment('burst/burst_history')
def load_burst_history(self, initBurst=None):
"""
Load the available burst that are stored in the database at this time.
This is one alternative to 'chrome-back problem'.
"""
bursts = self.burst_service.get_available_bursts(
self.context.project.id)
self.burst_service.populate_burst_disk_usage(bursts)
fromInit = False
if initBurst is not None:
fromInit = True
return {
'burst_list': bursts,
'fromInit': fromInit,
'selectedBurst': self.context.burst_config.id,
'first_fragment_url':
SimulatorFragmentRenderingRules.FIRST_FORM_URL
}
@cherrypy.expose
def get_last_fragment_url(self, burst_config_id):
burst_config = self.burst_service.load_burst_configuration(
burst_config_id)
return self.get_url_for_final_fragment(burst_config)
@expose_fragment('simulator_fragment')
def load_burst_read_only(self, burst_config_id):
try:
burst_config = self.burst_service.load_burst_configuration(
burst_config_id)
storage_path = StorageInterface().get_project_folder(
self.context.project.name, str(burst_config.fk_simulation))
simulator = h5.load_view_model(burst_config.simulator_gid,
storage_path)
last_loaded_form_url = self.get_url_for_final_fragment(
burst_config)
self.context.init_session_at_burst_loading(burst_config, simulator,
last_loaded_form_url)
form = self.prepare_first_fragment()
self.monitors_handler.build_list_of_monitors_from_view_models(
self.context.simulator)
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
is_simulation_readonly_load=True,
is_first_fragment=True)
return rendering_rules.to_dict()
except Exception:
# Most probably Burst was removed. Delete it from session, so that client
# has a good chance to get a good response on refresh
self.logger.exception("Error loading burst")
self.context.remove_burst_config_from_session()
raise
def _prepare_first_fragment_for_burst_copy(self, burst_config_id,
burst_name_format):
simulator, burst_config_copy = self.burst_service.prepare_data_for_burst_copy(
burst_config_id, burst_name_format, self.context.project)
self.monitors_handler.build_list_of_monitors_from_view_models(
simulator)
last_loaded_form_url = self.get_url_for_final_fragment(
burst_config_copy)
self.context.init_session_at_copy_preparation(burst_config_copy,
simulator,
last_loaded_form_url)
return self.prepare_first_fragment()
@expose_fragment('simulator_fragment')
def copy_simulator_configuration(self, burst_config_id):
self.context.add_branch_and_copy_to_session(False, True)
form = self._prepare_first_fragment_for_burst_copy(
burst_config_id, self.COPY_NAME_FORMAT)
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
is_simulation_copy=True,
is_simulation_readonly_load=True,
is_first_fragment=True)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def branch_simulator_configuration(self, burst_config_id):
self.context.add_branch_and_copy_to_session(True, False)
form = self._prepare_first_fragment_for_burst_copy(
burst_config_id, self.BRANCH_NAME_FORMAT)
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
is_simulation_copy=True,
is_simulation_readonly_load=True,
is_first_fragment=True)
return rendering_rules.to_dict()
@expose_fragment('simulator_fragment')
def reset_simulator_configuration(self):
burst_config = BurstConfiguration(self.context.project.id)
self.context.init_session_at_sim_reset(
burst_config, SimulatorWizzardURLs.SET_CONNECTIVITY_URL)
self.monitors_handler.clear_next_monitors_dict()
form = self.prepare_first_fragment()
rendering_rules = SimulatorFragmentRenderingRules(
form,
SimulatorWizzardURLs.SET_CONNECTIVITY_URL,
is_first_fragment=True)
return rendering_rules.to_dict()
@expose_json
def rename_burst(self, burst_id, burst_name):
"""
Rename the burst given by burst_id, setting it's new name to
burst_name.
"""
validation_result = SimulatorFinalFragment.is_burst_name_ok(burst_name)
if validation_result is True:
self.burst_service.rename_burst(burst_id, burst_name)
return {'success': "Simulation successfully renamed!"}
else:
self.logger.exception(validation_result)
return {'error': validation_result}
@expose_json
def get_history_status(self, **data):
"""
For each burst id received, get the status and return it.
"""
return self.burst_service.update_history_status(
json.loads(data['burst_ids']))
@cherrypy.expose
@handle_error(redirect=False)
@check_user
def export(self, burst_id):
export_manager = ExportManager()
export_zip = export_manager.export_simulator_configuration(burst_id)
result_name = "tvb_simulation_" + str(burst_id) + ".zip"
return serve_file(export_zip, "application/x-download", "attachment",
result_name)
@expose_fragment("overlay")
def get_upload_overlay(self):
template_specification = self.fill_overlay_attributes(
None, "Upload", "Simulation ZIP", "burst/upload_burst_overlay",
"dialog-upload")
template_specification[
'first_fragment_url'] = SimulatorWizzardURLs.SET_CONNECTIVITY_URL
return self.fill_default_attributes(template_specification)
@cherrypy.expose
@handle_error(redirect=True)
@check_user
@settings
def load_simulator_configuration_from_zip(self, **data):
"""Upload Simulator from previously exported ZIP file"""
self.logger.debug("Uploading ..." + str(data))
last_loaded_form_url = SimulatorWizzardURLs.SETUP_PSE_URL
try:
upload_param = "uploadedfile"
if upload_param in data and data[upload_param]:
simulator, burst_config, sim_folder = self.burst_service.load_simulation_from_zip(
data[upload_param], self.context.project)
dts_folder = os.path.join(
sim_folder, StorageInterface.EXPORTED_SIMULATION_DTS_DIR)
ImportService().import_project_operations(
self.context.project, dts_folder, False, None)
self.monitors_handler.build_list_of_monitors_from_view_models(
simulator)
if burst_config.is_pse_burst():
last_loaded_form_url = SimulatorWizzardURLs.LAUNCH_PSE_URL
self.context.init_session_at_sim_config_from_zip(
burst_config, simulator, last_loaded_form_url)
except IOError as ioexcep:
self.logger.exception(ioexcep)
self.context.set_warning_message(
"This ZIP does not contain a complete simulator configuration")
except ServicesBaseException as excep:
self.logger.warning(excep.message)
self.context.set_warning_message(excep.message)
raise cherrypy.HTTPRedirect('/burst/')
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.files_helper = FilesHelper()
@transactional
def _prepare_operation(self,
project_id,
user_id,
simulator_id,
simulator_gid,
algo_category,
op_group,
metadata,
ranges=None):
operation_parameters = json.dumps({'gid': simulator_gid.hex})
metadata, user_group = self.operation_service._prepare_metadata(
metadata, algo_category, op_group, {})
meta_str = json.dumps(metadata)
op_group_id = None
if op_group:
op_group_id = op_group.id
operation = Operation(user_id,
project_id,
simulator_id,
operation_parameters,
op_group_id=op_group_id,
meta=meta_str,
range_values=ranges)
self.logger.info("Saving Operation(userId=" + str(user_id) +
", projectId=" + str(project_id) + "," +
str(metadata) + ", algorithmId=" + str(simulator_id) +
", ops_group= " + str(op_group_id) + ", params=" +
str(operation_parameters) + ")")
operation = dao.store_entity(operation)
# TODO: prepare portlets/handle operation groups/no workflows
return operation
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo,
session_stored_simulator,
simulation_state_gid):
try:
metadata = {}
metadata.update({DataTypeMetaData.KEY_BURST: burst_config.id})
simulator_id = simulator_algo.id
algo_category = simulator_algo.algorithm_category
operation = self._prepare_operation(project.id, user.id,
simulator_id,
session_stored_simulator.gid,
algo_category, None, metadata)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
SimulatorSerializer().serialize_simulator(session_stored_simulator,
simulation_state_gid,
storage_path)
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, operation.id, session_stored_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
with SimulatorH5(simulator_file) as simulator_h5:
simulator_gid = simulator_h5.gid.load()
metadata = {}
simulator_id = algorithm.id
algo_category = algorithm.algorithm_category
operation = self._prepare_operation(project.id, user_id, simulator_id,
simulator_gid, algo_category, None,
metadata)
storage_operation_path = self.files_helper.get_project_folder(
project, str(operation.id))
self.async_launch_simulation_on_server(operation, zip_folder_path,
storage_operation_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path,
storage_operation_path):
try:
for file in os.listdir(zip_folder_path):
shutil.move(os.path.join(zip_folder_path, file),
storage_operation_path)
try:
OperationService().launch_operation(operation.id, True)
shutil.rmtree(zip_folder_path)
return operation
except Exception as excep:
self.logger.error(excep)
except Exception as excep:
self.logger.error(excep)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
simulator_id = simulator_algo.id
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
operations = []
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
first_simulator = None
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(
simulator, range_param1.name, param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[
range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self._prepare_operation(
project.id, user.id, simulator_id, simulator.gid,
algo_category, operation_group,
{DataTypeMetaData.KEY_BURST: burst_config.id}, ranges)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
SimulatorSerializer().serialize_simulator(
simulator, None, storage_path)
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
first_operation = operations[0]
storage_path = self.files_helper.get_project_folder(
project, str(first_operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, first_operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=first_operation.id,
fk_parent_burst=burst_config.id,
state=json.loads(
first_operation.meta_data)[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group, fk_parent_burst=burst_config.id)
dao.store_entity(metrics_datatype_group)
wf_errs = 0
for operation in operations:
try:
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def load_from_zip(self, zip_file, project):
import_service = ImportService()
simulator_folder = import_service.import_simulator_configuration_zip(
zip_file)
simulator_h5_filename = DirLoader(
simulator_folder, None).find_file_for_has_traits_type(Simulator)
with SimulatorH5(os.path.join(simulator_folder,
simulator_h5_filename)) as sim_h5:
simulator_gid = sim_h5.gid.load()
simulator = SimulatorSerializer.deserialize_simulator(
simulator_gid, simulator_folder)
burst_config = self.burst_service.load_burst_configuration_from_folder(
simulator_folder, project)
return simulator, burst_config
def run(self):
"""
Get the required data from the operation queue and launch the operation.
"""
# Try to get a spot to launch own operation.
LOCKS_QUEUE.get(True)
operation_id = self.operation_id
run_params = [
TvbProfile.current.PYTHON_INTERPRETER_PATH, '-m',
'tvb.core.operation_async_launcher',
str(operation_id), TvbProfile.CURRENT_PROFILE_NAME
]
# In the exceptional case where the user pressed stop while the Thread startup is done,
# We should no longer launch the operation.
if self.stopped() is False:
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
# anything that was already in $PYTHONPATH should have been reproduced in sys.path
launched_process = Popen(run_params,
stdout=PIPE,
stderr=PIPE,
env=env)
LOGGER.debug(
"Storing pid=%s for operation id=%s launched on local machine."
% (operation_id, launched_process.pid))
op_ident = OperationProcessIdentifier(operation_id,
pid=launched_process.pid)
dao.store_entity(op_ident)
if self.stopped():
# In the exceptional case where the user pressed stop while the Thread startup is done.
# and stop_operation is concurrently asking about OperationProcessIdentity.
self.stop_pid(launched_process.pid)
subprocess_result = launched_process.communicate()
LOGGER.info("Finished with launch of operation %s" % operation_id)
returned = launched_process.wait()
if returned != 0 and not self.stopped():
# Process did not end as expected. (e.g. Segmentation fault)
burst_service = BurstService()
operation = dao.get_operation_by_id(self.operation_id)
LOGGER.error(
"Operation suffered fatal failure! Exit code: %s Exit message: %s"
% (returned, subprocess_result))
burst_service.persist_operation_state(
operation, STATUS_ERROR,
"Operation failed unexpectedly! Please check the log files."
)
burst_entity = dao.get_burst_for_operation_id(
self.operation_id)
if burst_entity:
message = "Error in operation process! Possibly segmentation fault."
burst_service.mark_burst_finished(burst_entity,
error_message=message)
del launched_process
# Give back empty spot now that you finished your operation
CURRENT_ACTIVE_THREADS.remove(self)
LOCKS_QUEUE.put(1)