def launch_pse(self, **data):
# TODO: Split into: set range values and Launch, show message with finished config and nr of simulations
all_range_parameters = self.range_parameters.get_all_range_parameters()
range_param1, range_param2 = SimulatorPSEParamRangeFragment.fill_from_post(
all_range_parameters, **data)
session_stored_simulator = common.get_from_session(
common.KEY_SIMULATOR_CONFIG)
project = common.get_current_project()
user = common.get_logged_user()
burst_config = common.get_from_session(common.KEY_BURST_CONFIG)
burst_config.start_time = datetime.now()
# if burst_name != 'none_undefined':
# burst_config.name = burst_name
# TODO: branch simulation name is different
if burst_config.name is None:
new_id = dao.get_max_burst_id() + 1
burst_config.name = 'simulation_' + str(new_id)
operation_group = OperationGroup(
project.id,
ranges=[range_param1.to_json(),
range_param2.to_json()])
operation_group = dao.store_entity(operation_group)
metric_operation_group = OperationGroup(
project.id,
ranges=[range_param1.to_json(),
range_param2.to_json()])
metric_operation_group = dao.store_entity(metric_operation_group)
burst_config.operation_group = operation_group
burst_config.operation_group_id = operation_group.id
burst_config.metric_operation_group = metric_operation_group
burst_config.metric_operation_group_id = metric_operation_group.id
dao.store_entity(burst_config)
try:
thread = threading.Thread(
target=self.simulator_service.async_launch_and_prepare_pse,
kwargs={
'burst_config': burst_config,
'user': user,
'project': project,
'simulator_algo': self.cached_simulator_algorithm,
'range_param1': range_param1,
'range_param2': range_param2,
'session_stored_simulator': session_stored_simulator
})
thread.start()
except BurstServiceException as e:
self.logger.exception("Could not launch burst!")
return {'error': e.message}
def prepare_burst_for_pse(burst_config):
# type: (BurstConfiguration) -> (BurstConfiguration)
operation_group = OperationGroup(burst_config.fk_project, ranges=burst_config.ranges)
operation_group = dao.store_entity(operation_group)
metric_operation_group = OperationGroup(burst_config.fk_project, ranges=burst_config.ranges)
metric_operation_group = dao.store_entity(metric_operation_group)
burst_config.operation_group = operation_group
burst_config.fk_operation_group = operation_group.id
burst_config.metric_operation_group = metric_operation_group
burst_config.fk_metric_operation_group = metric_operation_group.id
return dao.store_entity(burst_config)
def __init__(self, range1, range2, apriori_data, metrics, datatype_gids):
self.log = get_logger(self.__class__.__name__)
# ranges
all_numbers_range1, self.range1_name, self.range1 = OperationGroup.load_range_numbers(
range1)
all_numbers_range2, self.range2_name, self.range2 = OperationGroup.load_range_numbers(
range2)
# Data from which to interpolate larger 2-D space
self.apriori_x = self._prepare_axes(self.range1, all_numbers_range1)
self.apriori_y = self._prepare_axes(self.range2, all_numbers_range2)
self.apriori_data = apriori_data
self.datatypes_gids = datatype_gids
self.metrics = metrics
def prepare_burst_for_pse(self, burst_config):
# type: (BurstConfiguration) -> None
if burst_config.range2:
ranges = [burst_config.range1, burst_config.range2]
else:
ranges = [burst_config.range1]
operation_group = OperationGroup(burst_config.fk_project, ranges=ranges)
operation_group = dao.store_entity(operation_group)
metric_operation_group = OperationGroup(burst_config.fk_project, ranges=ranges)
metric_operation_group = dao.store_entity(metric_operation_group)
burst_config.operation_group = operation_group
burst_config.fk_operation_group = operation_group.id
burst_config.metric_operation_group = metric_operation_group
burst_config.fk_metric_operation_group = metric_operation_group.id
return dao.store_entity(burst_config)
def _prepare_group(self, project_id, existing_dt_group, kwargs):
"""
Create and store OperationGroup entity, or return None
"""
# Standard ranges as accepted from UI
range1_values = self.get_range_values(kwargs, self._range_name(1))
range2_values = self.get_range_values(kwargs, self._range_name(2))
available_args = self.__expand_arguments([(kwargs, None)],
range1_values,
self._range_name(1))
available_args = self.__expand_arguments(available_args, range2_values,
self._range_name(2))
is_group = False
ranges = []
if self._range_name(1) in kwargs and range1_values is not None:
is_group = True
ranges.append(
json.dumps((kwargs[self._range_name(1)], range1_values)))
if self._range_name(2) in kwargs and range2_values is not None:
is_group = True
ranges.append(
json.dumps((kwargs[self._range_name(2)], range2_values)))
# Now for additional ranges which might be the case for the 'model exploration'
last_range_idx = 3
ranger_name = self._range_name(last_range_idx)
while ranger_name in kwargs:
values_for_range = self.get_range_values(kwargs, ranger_name)
available_args = self.__expand_arguments(available_args,
values_for_range,
ranger_name)
last_range_idx += 1
ranger_name = self._range_name(last_range_idx)
if last_range_idx > 3:
ranges = [
] # Since we only have 3 fields in db for this just hide it
if not is_group:
group = None
elif existing_dt_group is None:
group = OperationGroup(project_id=project_id, ranges=ranges)
group = dao.store_entity(group)
else:
group = existing_dt_group.parent_operation_group
return available_args, group
def _retrieve_operations_in_order(self,
project,
import_path,
importer_operation_id=None):
# type: (Project, str, int) -> list[Operation2ImportData]
retrieved_operations = []
for root, _, files in os.walk(import_path):
if OPERATION_XML in files:
# Previous Operation format for uploading previous versions of projects
operation_file_path = os.path.join(root, OPERATION_XML)
operation, operation_xml_parameters, _ = self.build_operation_from_file(
project, operation_file_path)
operation.import_file = operation_file_path
self.logger.debug("Found operation in old XML format: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(
operation,
root,
info_from_xml=operation_xml_parameters))
else:
# We strive for the new format with ViewModelH5
main_view_model = None
dt_paths = []
all_view_model_files = []
for file in files:
if file.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(root, file)
try:
h5_class = H5File.h5_class_from_file(h5_file)
if h5_class is ViewModelH5:
all_view_model_files.append(h5_file)
if not main_view_model:
view_model = h5.load_view_model_from_file(
h5_file)
if type(
view_model
) in self.view_model2adapter.keys():
main_view_model = view_model
else:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
dt_paths.append(h5_file)
except Exception:
self.logger.warning(
"Unreadable H5 file will be ignored: %s" %
h5_file)
if main_view_model is not None:
alg = self.view_model2adapter[type(main_view_model)]
op_group_id = None
if main_view_model.operation_group_gid:
op_group = dao.get_operationgroup_by_gid(
main_view_model.operation_group_gid.hex)
if not op_group:
op_group = OperationGroup(
project.id,
ranges=json.loads(main_view_model.ranges),
gid=main_view_model.operation_group_gid.hex)
op_group = dao.store_entity(op_group)
op_group_id = op_group.id
operation = Operation(
main_view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
user_group=main_view_model.generic_attributes.
operation_tag,
start_date=datetime.now(),
completion_date=datetime.now(),
op_group_id=op_group_id,
range_values=main_view_model.range_values)
operation.create_date = main_view_model.create_date
operation.visible = main_view_model.generic_attributes.visible
self.logger.debug(
"Found main ViewModel to create operation for it: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, main_view_model,
dt_paths, all_view_model_files))
elif len(dt_paths) > 0:
alg = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
default_adapter = ABCAdapter.build_adapter(alg)
view_model = default_adapter.get_view_model_class()()
view_model.data_file = dt_paths[0]
vm_path = h5.store_view_model(view_model, root)
all_view_model_files.append(vm_path)
operation = Operation(view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
start_date=datetime.now(),
completion_date=datetime.now())
self.logger.debug(
"Found no ViewModel in folder, so we default to " +
str(operation))
if importer_operation_id:
operation.id = importer_operation_id
retrieved_operations.append(
Operation2ImportData(operation, root, view_model,
dt_paths, all_view_model_files,
True))
return sorted(retrieved_operations,
key=lambda op_data: op_data.order_field)
def prepare_operations_for_workflowsteps(self, workflow_step_list,
workflows, user_id, burst_id,
project_id, group,
sim_operations):
"""
Create and store Operation entities from a list of Workflow Steps.
Will be generated workflows x workflow_step_list Operations.
For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created
(in case of PSE).
"""
for step in workflow_step_list:
operation_group = None
if (group is not None) and not isinstance(step, WorkflowStepView):
operation_group = OperationGroup(project_id=project_id,
ranges=group.range_references)
operation_group = dao.store_entity(operation_group)
operation = None
metadata = {DataTypeMetaData.KEY_BURST: burst_id}
algo_category = dao.get_algorithm_by_id(step.fk_algorithm)
if algo_category is not None:
algo_category = algo_category.algorithm_category
for wf_idx, workflow in enumerate(workflows):
cloned_w_step = step.clone()
cloned_w_step.fk_workflow = workflow.id
dynamic_params = cloned_w_step.dynamic_param
op_params = cloned_w_step.static_param
op_params.update(dynamic_params)
range_values = None
group_id = None
if operation_group is not None:
group_id = operation_group.id
range_values = sim_operations[wf_idx].range_values
if not isinstance(step, WorkflowStepView):
## For visualization steps, do not create operations, as those are not really needed.
metadata, user_group = self._prepare_metadata(
metadata, algo_category, operation_group, op_params)
operation = Operation(user_id,
project_id,
step.fk_algorithm,
json.dumps(
op_params,
cls=MapAsJson.MapAsJsonEncoder),
meta=json.dumps(metadata),
op_group_id=group_id,
range_values=range_values,
user_group=user_group)
operation.visible = step.step_visible
operation = dao.store_entity(operation)
cloned_w_step.fk_operation = operation.id
dao.store_entity(cloned_w_step)
if operation_group is not None and operation is not None:
datatype_group = DataTypeGroup(
operation_group,
operation_id=operation.id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)