class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.algorithm_service = AlgorithmService()
self.storage_interface = StorageInterface()
@staticmethod
def _reset_model(session_stored_simulator):
session_stored_simulator.model = type(session_stored_simulator.model)()
vi_indexes = session_stored_simulator.determine_indexes_for_chosen_vars_of_interest(
)
vi_indexes = numpy.array(list(vi_indexes.values()))
for monitor in session_stored_simulator.monitors:
monitor.variables_of_interest = vi_indexes
def reset_at_connectivity_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise
parameters because they might not fit to the new Connectivity's nr of regions.
"""
if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity:
self._reset_model(session_stored_simulator)
if issubclass(type(session_stored_simulator.integrator),
IntegratorStochastic):
session_stored_simulator.integrator.noise = type(
session_stored_simulator.integrator.noise)()
def reset_at_surface_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a surface-simulation and changes the Surface, we want to reset the Model
parameters because they might not fit to the new Surface's nr of vertices.
"""
if is_simulator_copy and (
session_stored_simulator.surface is None and form.surface.value
or session_stored_simulator.surface and form.surface.value !=
session_stored_simulator.surface.surface_gid):
self._reset_model(session_stored_simulator)
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo, simulator):
try:
operation = self.operation_service.prepare_operation(
user.id,
project,
simulator_algo,
view_model=simulator,
burst_gid=burst_config.gid,
op_group_id=burst_config.fk_operation_group)
burst_config = self.burst_service.update_simulation_fields(
burst_config, operation.id, simulator.gid)
storage_path = self.storage_interface.get_project_folder(
project.name, str(operation.id))
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
simulator_vm = h5.load_view_model_from_file(simulator_file)
operation = self.operation_service.prepare_operation(
user_id, project, algorithm, view_model=simulator_vm)
self.async_launch_simulation_on_server(operation, zip_folder_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path):
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
finally:
shutil.rmtree(zip_folder_path)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
GROUP_BURST_PENDING[burst_config.id] = True
operations, pse_canceled = self._prepare_operations(
algo_category, burst_config, metric_operation_group,
operation_group, project, range_param1, range_param2,
range_param2_values, session_stored_simulator, simulator_algo,
user)
GROUP_BURST_PENDING[burst_config.id] = False
if pse_canceled:
return
wf_errs = self._launch_operations(operations, burst_config)
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
return operations[0] if len(operations) > 0 else None
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def _launch_operations(self, operations, burst_config):
wf_errs = 0
for operation in operations:
try:
burst_config = dao.get_burst_by_id(burst_config.id)
if burst_config is None or burst_config.status in [
BurstConfiguration.BURST_CANCELED,
BurstConfiguration.BURST_ERROR
]:
self.logger.debug(
"Preparing operations cannot continue. Burst config {}"
.format(burst_config))
return
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
return wf_errs
def _prepare_operations(self, algo_category, burst_config,
metric_operation_group, operation_group, project,
range_param1, range_param2, range_param2_values,
session_stored_simulator, simulator_algo, user):
first_simulator = None
pse_canceled = False
operations = []
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
burst_config = dao.get_burst_by_id(burst_config.id)
if burst_config is None:
self.logger.debug("Burst config was deleted")
pse_canceled = True
break
if burst_config.status in [
BurstConfiguration.BURST_CANCELED,
BurstConfiguration.BURST_ERROR
]:
self.logger.debug(
"Current burst status is {}. Preparing operations cannot continue."
.format(burst_config.status))
pse_canceled = True
break
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(simulator,
range_param1.name,
param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self.operation_service.prepare_operation(
user.id,
project,
simulator_algo,
view_model=simulator,
ranges=ranges,
burst_gid=burst_config.gid,
op_group_id=burst_config.fk_operation_group)
simulator.range_values = ranges
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
storage_path = self.storage_interface.get_project_folder(
project.name, str(operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config, operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=operation.id,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(metrics_datatype_group)
return operations, pse_canceled
@staticmethod
def compute_conn_branch_conditions(is_branch, simulator):
if not is_branch:
return None
conn = load.load_entity_by_gid(simulator.connectivity)
if conn.number_of_regions:
return FilterChain(
fields=[FilterChain.datatype + '.number_of_regions'],
operations=["=="],
values=[conn.number_of_regions])
@staticmethod
def validate_first_fragment(form, project_id, conn_idx):
conn_count = dao.count_datatypes(project_id, conn_idx)
if conn_count == 0:
form.connectivity.errors.append(
"No connectivity in the project! Simulation cannot be started without "
"a connectivity!")
def get_simulation_state_index(self, burst_config,
simulation_history_class):
parent_burst = burst_config.parent_burst_object
simulation_state_index = dao.get_generic_entity(
simulation_history_class, parent_burst.gid, "fk_parent_burst")
if simulation_state_index is None or len(simulation_state_index) < 1:
exc = BurstServiceException(
"Simulation State not found for %s, thus we are unable to branch from "
"it!" % burst_config.name)
self.logger.error(exc)
raise exc
return simulation_state_index
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.files_helper = FilesHelper()
def _reset_model(self, session_stored_simulator):
session_stored_simulator.model = type(session_stored_simulator.model)()
vi_indexes = MonitorForm.determine_indexes_for_chosen_vars_of_interest(
session_stored_simulator)
vi_indexes = numpy.array(list(vi_indexes.values()))
for monitor in session_stored_simulator.monitors:
monitor.variables_of_interest = vi_indexes
def reset_at_connectivity_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a simulation and changes the Connectivity, we want to reset the Model and Noise
parameters because they might not fit to the new Connectivity's nr of regions.
"""
if is_simulator_copy and form.connectivity.value != session_stored_simulator.connectivity:
self._reset_model(session_stored_simulator)
if issubclass(type(session_stored_simulator.integrator),
IntegratorStochastic):
session_stored_simulator.integrator.noise = type(
session_stored_simulator.integrator.noise)()
def reset_at_surface_change(self, is_simulator_copy, form,
session_stored_simulator):
"""
In case the user copies a surface-simulation and changes the Surface, we want to reset the Model
parameters because they might not fit to the new Surface's nr of vertices.
"""
if is_simulator_copy and (
session_stored_simulator.surface is None and form.surface.value
or session_stored_simulator.surface and form.surface.value !=
session_stored_simulator.surface.surface_gid):
self._reset_model(session_stored_simulator)
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo,
session_stored_simulator):
try:
operation = self.operation_service.prepare_operation(
user.id, project.id, simulator_algo,
session_stored_simulator.gid)
ga = self.operation_service._prepare_metadata(
simulator_algo.algorithm_category, {}, None, burst_config.gid)
session_stored_simulator.generic_attributes = ga
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
h5.store_view_model(session_stored_simulator, storage_path)
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, operation.id, session_stored_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
simulator_vm = h5.load_view_model_from_file(simulator_file)
operation = self.operation_service.prepare_operation(
user_id, project.id, algorithm, simulator_vm.gid)
storage_operation_path = self.files_helper.get_project_folder(
project, str(operation.id))
self.async_launch_simulation_on_server(operation, zip_folder_path,
storage_operation_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path,
storage_operation_path):
try:
for file in os.listdir(zip_folder_path):
shutil.move(os.path.join(zip_folder_path, file),
storage_operation_path)
try:
OperationService().launch_operation(operation.id, True)
shutil.rmtree(zip_folder_path)
return operation
except Exception as excep:
self.logger.error(excep)
except Exception as excep:
self.logger.error(excep)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
operations = []
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
first_simulator = None
ga = self.operation_service._prepare_metadata(
simulator_algo.algorithm_category, {}, operation_group,
burst_config.gid)
session_stored_simulator.generic_attributes = ga
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(
simulator, range_param1.name, param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[
range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self.operation_service.prepare_operation(
user.id, project.id, simulator_algo, simulator.gid,
operation_group, ranges)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
h5.store_view_model(simulator, storage_path)
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
first_operation = operations[0]
storage_path = self.files_helper.get_project_folder(
project, str(first_operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, first_operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=first_operation.id,
fk_parent_burst=burst_config.gid,
state=algo_category.defaultdatastate)
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group, fk_parent_burst=burst_config.gid)
dao.store_entity(metrics_datatype_group)
wf_errs = 0
for operation in operations:
try:
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
return first_operation
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def load_from_zip(self, zip_file, project):
import_service = ImportService()
simulator_folder = import_service.import_simulator_configuration_zip(
zip_file)
simulator_h5_filename = DirLoader(
simulator_folder,
None).find_file_for_has_traits_type(SimulatorAdapterModel)
simulator_h5_filepath = os.path.join(simulator_folder,
simulator_h5_filename)
simulator = h5.load_view_model_from_file(simulator_h5_filepath)
burst_config = self.burst_service.load_burst_configuration_from_folder(
simulator_folder, project)
return simulator, burst_config
class SimulatorService(object):
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.burst_service = BurstService()
self.operation_service = OperationService()
self.files_helper = FilesHelper()
@transactional
def _prepare_operation(self,
project_id,
user_id,
simulator_id,
simulator_gid,
algo_category,
op_group,
metadata,
ranges=None):
operation_parameters = json.dumps({'gid': simulator_gid.hex})
metadata, user_group = self.operation_service._prepare_metadata(
metadata, algo_category, op_group, {})
meta_str = json.dumps(metadata)
op_group_id = None
if op_group:
op_group_id = op_group.id
operation = Operation(user_id,
project_id,
simulator_id,
operation_parameters,
op_group_id=op_group_id,
meta=meta_str,
range_values=ranges)
self.logger.info("Saving Operation(userId=" + str(user_id) +
", projectId=" + str(project_id) + "," +
str(metadata) + ", algorithmId=" + str(simulator_id) +
", ops_group= " + str(op_group_id) + ", params=" +
str(operation_parameters) + ")")
operation = dao.store_entity(operation)
# TODO: prepare portlets/handle operation groups/no workflows
return operation
@staticmethod
def _set_simulator_range_parameter(simulator, range_parameter_name,
range_parameter_value):
range_param_name_list = range_parameter_name.split('.')
current_attr = simulator
for param_name in range_param_name_list[:len(range_param_name_list) -
1]:
current_attr = getattr(current_attr, param_name)
setattr(current_attr, range_param_name_list[-1], range_parameter_value)
def async_launch_and_prepare_simulation(self, burst_config, user, project,
simulator_algo,
session_stored_simulator,
simulation_state_gid):
try:
metadata = {}
metadata.update({DataTypeMetaData.KEY_BURST: burst_config.id})
simulator_id = simulator_algo.id
algo_category = simulator_algo.algorithm_category
operation = self._prepare_operation(project.id, user.id,
simulator_id,
session_stored_simulator.gid,
algo_category, None, metadata)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
SimulatorSerializer().serialize_simulator(session_stored_simulator,
simulation_state_gid,
storage_path)
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, operation.id, session_stored_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
wf_errs = 0
try:
OperationService().launch_operation(operation.id, True)
return operation
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug(
"Finished launching workflow. The operation was launched successfully, "
+ str(wf_errs) + " had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
if burst_config:
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
def prepare_simulation_on_server(self, user_id, project, algorithm,
zip_folder_path, simulator_file):
with SimulatorH5(simulator_file) as simulator_h5:
simulator_gid = simulator_h5.gid.load()
metadata = {}
simulator_id = algorithm.id
algo_category = algorithm.algorithm_category
operation = self._prepare_operation(project.id, user_id, simulator_id,
simulator_gid, algo_category, None,
metadata)
storage_operation_path = self.files_helper.get_project_folder(
project, str(operation.id))
self.async_launch_simulation_on_server(operation, zip_folder_path,
storage_operation_path)
return operation
def async_launch_simulation_on_server(self, operation, zip_folder_path,
storage_operation_path):
try:
for file in os.listdir(zip_folder_path):
shutil.move(os.path.join(zip_folder_path, file),
storage_operation_path)
try:
OperationService().launch_operation(operation.id, True)
shutil.rmtree(zip_folder_path)
return operation
except Exception as excep:
self.logger.error(excep)
except Exception as excep:
self.logger.error(excep)
@staticmethod
def _set_range_param_in_dict(param_value):
if type(param_value) is numpy.ndarray:
return param_value[0]
elif isinstance(param_value, uuid.UUID):
return param_value.hex
else:
return param_value
def async_launch_and_prepare_pse(self, burst_config, user, project,
simulator_algo, range_param1,
range_param2, session_stored_simulator):
try:
simulator_id = simulator_algo.id
algo_category = simulator_algo.algorithm_category
operation_group = burst_config.operation_group
metric_operation_group = burst_config.metric_operation_group
operations = []
range_param2_values = [None]
if range_param2:
range_param2_values = range_param2.get_range_values()
first_simulator = None
for param1_value in range_param1.get_range_values():
for param2_value in range_param2_values:
# Copy, but generate a new GUID for every Simulator in PSE
simulator = copy.deepcopy(session_stored_simulator)
simulator.gid = uuid.uuid4()
self._set_simulator_range_parameter(
simulator, range_param1.name, param1_value)
ranges = {
range_param1.name:
self._set_range_param_in_dict(param1_value)
}
if param2_value is not None:
self._set_simulator_range_parameter(
simulator, range_param2.name, param2_value)
ranges[
range_param2.name] = self._set_range_param_in_dict(
param2_value)
ranges = json.dumps(ranges)
operation = self._prepare_operation(
project.id, user.id, simulator_id, simulator.gid,
algo_category, operation_group,
{DataTypeMetaData.KEY_BURST: burst_config.id}, ranges)
storage_path = self.files_helper.get_project_folder(
project, str(operation.id))
SimulatorSerializer().serialize_simulator(
simulator, None, storage_path)
operations.append(operation)
if first_simulator is None:
first_simulator = simulator
first_operation = operations[0]
storage_path = self.files_helper.get_project_folder(
project, str(first_operation.id))
burst_config = self.burst_service.update_simulation_fields(
burst_config.id, first_operation.id, first_simulator.gid)
self.burst_service.store_burst_configuration(
burst_config, storage_path)
datatype_group = DataTypeGroup(
operation_group,
operation_id=first_operation.id,
fk_parent_burst=burst_config.id,
state=json.loads(
first_operation.meta_data)[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
metrics_datatype_group = DataTypeGroup(
metric_operation_group, fk_parent_burst=burst_config.id)
dao.store_entity(metrics_datatype_group)
wf_errs = 0
for operation in operations:
try:
OperationService().launch_operation(operation.id, True)
except Exception as excep:
self.logger.error(excep)
wf_errs += 1
self.burst_service.mark_burst_finished(
burst_config, error_message=str(excep))
self.logger.debug("Finished launching workflows. " +
str(len(operations) - wf_errs) +
" were launched successfully, " + str(wf_errs) +
" had error on pre-launch steps")
except Exception as excep:
self.logger.error(excep)
self.burst_service.mark_burst_finished(burst_config,
error_message=str(excep))
def load_from_zip(self, zip_file, project):
import_service = ImportService()
simulator_folder = import_service.import_simulator_configuration_zip(
zip_file)
simulator_h5_filename = DirLoader(
simulator_folder, None).find_file_for_has_traits_type(Simulator)
with SimulatorH5(os.path.join(simulator_folder,
simulator_h5_filename)) as sim_h5:
simulator_gid = sim_h5.gid.load()
simulator = SimulatorSerializer.deserialize_simulator(
simulator_gid, simulator_folder)
burst_config = self.burst_service.load_burst_configuration_from_folder(
simulator_folder, project)
return simulator, burst_config
