def build(algorithm=None,
test_user=None,
test_project=None,
operation_status=STATUS_FINISHED,
parameters="test params",
range_values=None):
"""
Create persisted operation.
:param algorithm: When not None, Simulator.
:return: Operation entity after persistence.
"""
if algorithm is None:
algorithm = dao.get_algorithm_by_module(
'tvb.adapters.simulator.simulator_adapter', 'SimulatorAdapter')
if test_user is None:
test_user = user_factory()
if test_project is None:
test_project = project_factory(test_user)
operation = Operation(test_user.id,
test_project.id,
algorithm.id,
parameters,
status=operation_status,
range_values=range_values)
dao.store_entity(operation)
# Make sure lazy attributes are correctly loaded.
return dao.get_operation_by_id(operation.id)
def _prepare_operation(self,
project_id,
user_id,
simulator_id,
simulator_gid,
algo_category,
op_group,
metadata,
ranges=None):
operation_parameters = json.dumps({'gid': simulator_gid.hex})
metadata, user_group = self.operation_service._prepare_metadata(
metadata, algo_category, op_group, {})
meta_str = json.dumps(metadata)
op_group_id = None
if op_group:
op_group_id = op_group.id
operation = Operation(user_id,
project_id,
simulator_id,
operation_parameters,
op_group_id=op_group_id,
meta=meta_str,
range_values=ranges)
self.logger.info("Saving Operation(userId=" + str(user_id) +
", projectId=" + str(project_id) + "," +
str(metadata) + ", algorithmId=" + str(simulator_id) +
", ops_group= " + str(op_group_id) + ", params=" +
str(operation_parameters) + ")")
operation = dao.store_entity(operation)
# TODO: prepare portlets/handle operation groups/no workflows
return operation
def test_adapter_huge_memory_requirement(self, test_adapter_factory):
"""
Test that an MemoryException is raised in case adapter cant launch due to lack of memory.
"""
# Prepare adapter
test_adapter_factory(adapter_class=TestAdapterHugeMemoryRequired)
adapter = TestFactory.create_adapter(
"tvb.tests.framework.adapters.testadapter3",
"TestAdapterHugeMemoryRequired")
# Simulate receiving POST data
form = TestAdapterHugeMemoryRequiredForm()
view_model = form.get_view_model()()
view_model.test = 5
# Prepare operation for launch
operation = Operation(view_model.gid.hex,
self.test_user.id,
self.test_project.id,
adapter.stored_adapter.id,
status=STATUS_STARTED)
operation = dao.store_entity(operation)
# Store ViewModel in H5
parent_folder = FilesHelper().get_project_folder(
self.test_project, str(operation.id))
h5.store_view_model(view_model, parent_folder)
# Launch operation
with pytest.raises(NoMemoryAvailableException):
OperationService().initiate_prelaunch(operation, adapter)
def build(algorithm=None,
test_user=None,
test_project=None,
operation_status=STATUS_FINISHED,
parameters="test params",
meta=None):
"""
Create persisted operation.
:param algorithm: When not None, Simulator.
:return: Operation entity after persistence.
"""
if algorithm is None:
algorithm = dao.get_algorithm_by_module(
'tvb.adapters.simulator.simulator_adapter', 'SimulatorAdapter')
if test_user is None:
test_user = user_factory()
if test_project is None:
test_project = project_factory(test_user)
if meta is None:
meta = {
DataTypeMetaData.KEY_SUBJECT: "John Doe",
DataTypeMetaData.KEY_STATE: "RAW_DATA"
}
operation = Operation(test_user.id,
test_project.id,
algorithm.id,
parameters,
meta=json.dumps(meta),
status=operation_status)
dao.store_entity(operation)
# Make sure lazy attributes are correctly loaded.
return dao.get_operation_by_id(operation.id)
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> Operation
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries)
time_series_index = dao.get_generic_entity(datatype_index,
sim_operation.id,
'fk_from_operation')[0]
ga = self.prepare_metadata(metric_algo.algorithm_category,
time_series_index.fk_parent_burst)
ga.visible = False
view_model = get_class_by_name("{}.{}".format(
MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(ALGORITHMS.keys())
view_model.generic_attributes = ga
parent_burst = dao.get_generic_entity(
BurstConfiguration, time_series_index.fk_parent_burst, 'gid')[0]
metric_op_group = dao.get_operationgroup_by_id(
parent_burst.fk_metric_operation_group)
metric_operation_group_id = parent_burst.fk_metric_operation_group
range_values = sim_operation.range_values
view_model.operation_group_gid = uuid.UUID(metric_op_group.gid)
view_model.ranges = json.dumps(parent_burst.ranges)
view_model.range_values = range_values
view_model.is_metric_operation = True
metric_operation = Operation(view_model.gid.hex,
sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
user_group=ga.operation_tag,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
dao.store_entity(metrics_datatype_group)
self.store_view_model(metric_operation, sim_operation.project,
view_model)
return metric_operation
def _remove_project_node_files(self,
project_id,
gid,
skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
new_op = Operation(
dao.get_system_user().id, links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.parameters,
datatype.parent_operation.meta_data,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project).name
new_op_loaded = dao.get_operation_by_id(new_op.id)
self.structure_helper.write_operation_metadata(
new_op_loaded)
full_path = h5.path_for_stored_index(datatype)
self.structure_helper.move_datatype(
datatype, to_project, str(new_op.id), full_path)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
h5_path = h5.path_for_stored_index(datatype)
self.structure_helper.remove_datatype_file(h5_path)
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException(
"Remove operation failed for unknown reasons.Please contact system administrator."
)
def prepare_operations(self, user_id, project, algorithm, category,
visible=True, existing_dt_group=None, view_model=None, **kwargs):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
"""
operations = []
available_args, group = self._prepare_group(project.id, existing_dt_group, kwargs)
if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER:
raise LaunchException("Too big range specified. You should limit the"
" resulting operations to %d" % TvbProfile.current.MAX_RANGE_NUMBER)
else:
self.logger.debug("Launching a range with %d operations..." % len(available_args))
group_id = None
if group is not None:
group_id = group.id
ga = self._prepare_metadata(category, kwargs, group)
ga.visible = visible
view_model.generic_attributes = ga
self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project.id) +
",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(group_id) + ")")
for (one_set_of_args, range_vals) in available_args:
range_values = json.dumps(range_vals) if range_vals else None
operation = Operation(user_id, project.id, algorithm.id, json.dumps({'gid': view_model.gid.hex}),
op_group_id=group_id, user_group=ga.operation_tag, range_values=range_values)
operation.visible = visible
operations.append(operation)
operations = dao.store_entities(operations)
if group is not None:
if existing_dt_group is None:
datatype_group = DataTypeGroup(group, operation_id=operations[0].id, state=category.defaultdatastate)
dao.store_entity(datatype_group)
else:
# Reset count
existing_dt_group.count_results = None
dao.store_entity(existing_dt_group)
for operation in operations:
self._store_view_model(operation, project, view_model)
return operations, group
def prepare_operation(self, user_id, project_id, algorithm, view_model_gid,
op_group=None, ranges=None, visible=True):
op_group_id = None
if op_group:
op_group_id = op_group.id
if isinstance(view_model_gid, uuid.UUID):
view_model_gid = view_model_gid.hex
operation = Operation(user_id, project_id, algorithm.id, json.dumps({'gid': view_model_gid}),
op_group_id=op_group_id, range_values=ranges)
self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project_id) +
",algorithmId=" + str(algorithm.id) + ", ops_group= " + str(op_group_id) + ")")
operation.visible = visible
operation = dao.store_entity(operation)
return operation
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> Operation
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries)
time_series_index = dao.get_generic_entity(datatype_index,
sim_operation.id,
'fk_from_operation')[0]
view_model = get_class_by_name("{}.{}".format(
MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(choices.values())
range_values = sim_operation.range_values
metadata = {
DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst
}
metadata, user_group = self._prepare_metadata(
metadata, metric_algo.algorithm_category, None, {})
meta_str = json.dumps(metadata)
parent_burst = dao.get_generic_entity(
BurstConfiguration, time_series_index.fk_parent_burst, 'id')[0]
metric_operation_group_id = parent_burst.fk_metric_operation_group
metric_operation = Operation(sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
json.dumps({'gid': view_model.gid.hex}),
meta_str,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
stored_metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
self._store_view_model(stored_metric_operation, sim_operation.project,
view_model)
return stored_metric_operation
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> None
metric_algo = dao.get_algorithm_by_module(
TimeseriesMetricsAdapter.__module__,
TimeseriesMetricsAdapter.__name__)
time_series_index = dao.get_generic_entity(TimeSeriesIndex,
sim_operation.id,
'fk_from_operation')[0]
view_model = TimeseriesMetricsAdapterModel()
view_model.time_series = time_series_index.gid
view_model.algorithms = tuple(choices.values())
range_values = sim_operation.range_values
metadata = {
DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst
}
metadata, user_group = self._prepare_metadata(
metadata, metric_algo.algorithm_category, None, {})
meta_str = json.dumps(metadata)
parent_burst = dao.get_generic_entity(
BurstConfiguration, time_series_index.fk_parent_burst, 'id')[0]
metric_operation_group_id = parent_burst.fk_metric_operation_group
metric_operation = Operation(sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
json.dumps({'gid': view_model.gid.hex}),
meta_str,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
stored_metric_operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
OperationService._store_view_model(stored_metric_operation,
sim_operation.project, view_model)
return stored_metric_operation
def __copy_linked_datatype_before_delete(self, op, datatype, project, fk_to_project):
new_op = Operation(op.view_model_gid,
dao.get_system_user().id,
fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op.visible = datatype.parent_operation.visible
new_op = dao.store_entity(new_op)
to_project = self.find_project(fk_to_project)
to_project_path = self.storage_interface.get_project_folder(to_project.name)
full_path = h5.path_for_stored_index(datatype)
old_folder = self.storage_interface.get_project_folder(project.name, str(op.id))
file_paths = h5.gather_references_of_view_model(op.view_model_gid, old_folder, only_view_models=True)[0]
file_paths.append(full_path)
# The BurstConfiguration h5 file has to be moved only when we handle the time series which has the operation
# folder containing the file
if datatype.is_ts and datatype.fk_parent_burst is not None:
bc_path = h5.path_for(datatype.parent_operation.id, BurstConfigurationH5, datatype.fk_parent_burst,
project.name)
if os.path.exists(bc_path):
file_paths.append(bc_path)
bc = dao.get_burst_for_operation_id(op.id)
bc.fk_simulation = new_op.id
dao.store_entity(bc)
# Move all files to the new operation folder
self.storage_interface.move_datatype_with_sync(to_project, to_project_path, new_op.id, file_paths)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
return new_op
def _prepare_metric_operation(self, sim_operation):
# type: (Operation) -> None
metric_algo = dao.get_algorithm_by_module(
TimeseriesMetricsAdapter.__module__,
TimeseriesMetricsAdapter.__name__)
time_series_index = dao.get_generic_entity(TimeSeriesIndex,
sim_operation.id,
'fk_from_operation')[0]
ts_metrics_adapter_form = TimeseriesMetricsAdapterForm()
ts_metrics_adapter_form.fill_from_trait(
BaseTimeseriesMetricAlgorithm())
ts_metrics_adapter_form.time_series.data = time_series_index.gid
op_params = json.dumps(ts_metrics_adapter_form.get_dict())
range_values = sim_operation.range_values
metadata = {
DataTypeMetaData.KEY_BURST: time_series_index.fk_parent_burst
}
metadata, user_group = self._prepare_metadata(
metadata, metric_algo.algorithm_category, None, op_params)
meta_str = json.dumps(metadata)
parent_burst = dao.get_generic_entity(
BurstConfiguration2, time_series_index.fk_parent_burst, 'id')[0]
metric_operation_group_id = parent_burst.metric_operation_group_id
metric_operation = Operation(sim_operation.fk_launched_by,
sim_operation.fk_launched_in,
metric_algo.id,
op_params,
meta_str,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
operation = dao.store_entity(metric_operation)
metrics_datatype_group = dao.get_generic_entity(
DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0]
if metrics_datatype_group.fk_from_operation is None:
metrics_datatype_group.fk_from_operation = metric_operation.id
return operation
def build(test_user=None,
test_project=None,
is_simulation=False,
store_vm=False,
operation_status=STATUS_FINISHED,
range_values=None,
conn_gid=None):
"""
Create persisted operation with a ViewModel stored
:return: Operation entity after persistence.
"""
if test_user is None:
test_user = user_factory()
if test_project is None:
test_project = project_factory(test_user)
vm_gid = uuid.uuid4()
view_model = None
if is_simulation:
algorithm = dao.get_algorithm_by_module(SIMULATOR_MODULE,
SIMULATOR_CLASS)
if store_vm:
adapter = ABCAdapter.build_adapter(algorithm)
view_model = adapter.get_view_model_class()()
view_model.connectivity = connectivity_factory(
4).gid if conn_gid is None else conn_gid
vm_gid = view_model.gid
else:
algorithm = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
if store_vm:
adapter = ABCAdapter.build_adapter(algorithm)
view_model = adapter.get_view_model_class()()
view_model.data_file = "."
vm_gid = view_model.gid
operation = Operation(vm_gid.hex,
test_user.id,
test_project.id,
algorithm.id,
status=operation_status,
range_values=range_values)
dao.store_entity(operation)
if store_vm:
op_folder = FilesHelper().get_project_folder(
test_project, str(operation.id))
h5.store_view_model(view_model, op_folder)
# Make sure lazy attributes are correctly loaded.
return dao.get_operation_by_id(operation.id)
def prepare_metrics_operation(operation):
# TODO reuse from OperationService and do not duplicate logic here
parent_burst = dao.get_generic_entity(BurstConfiguration,
operation.fk_operation_group,
'fk_operation_group')[0]
metric_operation_group_id = parent_burst.fk_metric_operation_group
range_values = operation.range_values
metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE,
MEASURE_METRICS_CLASS)
metric_operation = Operation(None,
operation.fk_launched_by,
operation.fk_launched_in,
metric_algo.id,
status=STATUS_FINISHED,
op_group_id=metric_operation_group_id,
range_values=range_values)
metric_operation.visible = False
metric_operation = dao.store_entity(metric_operation)
op_dir = StorageInterface().get_project_folder(
operation.project.name, str(metric_operation.id))
return op_dir, metric_operation
def prepare_operation(self,
user_id,
project_id,
algorithm_id,
category,
view_model_gid,
op_group,
metadata,
ranges=None,
visible=True):
operation_parameters = json.dumps({'gid': view_model_gid})
metadata, user_group = self._prepare_metadata(metadata, category,
op_group, {})
meta_str = json.dumps(metadata)
op_group_id = None
if op_group:
op_group_id = op_group.id
operation = Operation(user_id,
project_id,
algorithm_id,
operation_parameters,
op_group_id=op_group_id,
meta=meta_str,
range_values=ranges)
self.logger.debug("Saving Operation(userId=" + str(user_id) +
",projectId=" + str(project_id) + "," +
str(metadata) + ",algorithmId=" + str(algorithm_id) +
", ops_group= " + str(op_group_id) + ")")
visible_operation = visible and category.display is False
operation = dao.store_entity(operation)
operation.visible = visible_operation
return operation
def prepare_operation(self, user_id, project, algorithm, visible=True, view_model=None, ranges=None,
burst_gid=None, op_group_id=None):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
"""
algo_category = dao.get_category_by_id(algorithm.fk_category)
ga = self.prepare_metadata(algo_category, current_ga=view_model.generic_attributes, burst=burst_gid)
ga.visible = visible
view_model.generic_attributes = ga
self.logger.debug("Saving Operation(userId=" + str(user_id) + ",projectId=" + str(project.id) +
",algorithmId=" + str(algorithm.id) + ")")
operation = Operation(view_model.gid.hex, user_id, project.id, algorithm.id, user_group=ga.operation_tag,
op_group_id=op_group_id, range_values=ranges)
operation = dao.store_entity(operation)
self.store_view_model(operation, project, view_model)
return operation
def _remove_project_node_files(self, project_id, gid, skip_validation=False):
"""
Delegate removal of a node in the structure of the project.
In case of a problem will THROW StructureException.
"""
try:
project = self.find_project(project_id)
datatype = dao.get_datatype_by_gid(gid)
links = dao.get_links_for_datatype(datatype.id)
op = dao.get_operation_by_id(datatype.fk_from_operation)
adapter = ABCAdapter.build_adapter(op.algorithm)
if links:
was_link = False
for link in links:
# This means it's only a link and we need to remove it
if link.fk_from_datatype == datatype.id and link.fk_to_project == project.id:
dao.remove_entity(Links, link.id)
was_link = True
if not was_link:
# Create a clone of the operation
# There is no view_model so the view_model_gid is None
new_op = Operation(op.view_model_gid,
dao.get_system_user().id,
links[0].fk_to_project,
datatype.parent_operation.fk_from_algo,
datatype.parent_operation.status,
datatype.parent_operation.start_date,
datatype.parent_operation.completion_date,
datatype.parent_operation.fk_operation_group,
datatype.parent_operation.additional_info,
datatype.parent_operation.user_group,
datatype.parent_operation.range_values)
new_op = dao.store_entity(new_op)
to_project = self.find_project(links[0].fk_to_project)
to_project_path = self.structure_helper.get_project_folder(to_project)
encryption_handler.set_project_active(to_project)
encryption_handler.sync_folders(to_project_path)
to_project_name = to_project.name
full_path = h5.path_for_stored_index(datatype)
self.structure_helper.move_datatype(datatype, to_project_name, str(new_op.id), full_path)
# Move also the ViewModel H5
old_folder = self.structure_helper.get_project_folder(project, str(op.id))
view_model = adapter.load_view_model(op)
vm_full_path = h5.determine_filepath(op.view_model_gid, old_folder)
self.structure_helper.move_datatype(view_model, to_project_name, str(new_op.id), vm_full_path)
encryption_handler.sync_folders(to_project_path)
encryption_handler.set_project_inactive(to_project)
datatype.fk_from_operation = new_op.id
datatype.parent_operation = new_op
dao.store_entity(datatype)
dao.remove_entity(Links, links[0].id)
else:
specific_remover = get_remover(datatype.type)(datatype)
specific_remover.remove_datatype(skip_validation)
h5_path = h5.path_for_stored_index(datatype)
self.structure_helper.remove_datatype_file(h5_path)
encryption_handler.push_folder_to_sync(self.structure_helper.get_project_folder_from_h5(h5_path))
except RemoveDataTypeException:
self.logger.exception("Could not execute operation Node Remove!")
raise
except FileStructureException:
self.logger.exception("Remove operation failed")
raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def build(subject="Datatype Factory User", state="RAW_DATA", project=None):
# TODO This is not real, we miss a ViewModel stored
# there store the name and the (hi, lo, step) value of the range parameters
range_1 = ["row1", [1, 2, 10]]
range_2 = ["row2", [0.1, 0.3, 0.5]]
# there are the actual numbers in the interval
range_values_1 = [1, 3, 5, 7, 9]
range_values_2 = [0.1, 0.4]
user = user_factory()
if project is None:
project = project_factory(user)
algorithm = dao.get_algorithm_by_module(
IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS)
# Create operation
operation = operation_factory(algorithm=algorithm,
test_user=user,
test_project=project)
group = OperationGroup(
project.id, ranges=[json.dumps(range_1),
json.dumps(range_2)])
group = dao.store_entity(group)
group_ms = OperationGroup(
project.id, ranges=[json.dumps(range_1),
json.dumps(range_2)])
group_ms = dao.store_entity(group_ms)
datatype_group = DataTypeGroup(group,
subject=subject,
state=state,
operation_id=operation.id)
datatype_group.no_of_ranges = 2
datatype_group.count_results = 10
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = DataTypeGroup(group_ms,
subject=subject,
state=state,
operation_id=operation.id)
datatype_group.no_of_ranges = 2
datatype_group.count_results = 10
dao.store_entity(dt_group_ms)
# Now create some data types and add them to group
for range_val1 in range_values_1:
for range_val2 in range_values_2:
op = Operation(user.id,
project.id,
algorithm.id,
'test parameters',
status=STATUS_FINISHED,
range_values=json.dumps({
range_1[0]: range_val1,
range_2[0]: range_val2
}))
op.fk_operation_group = group.id
op = dao.store_entity(op)
datatype = time_series_index_factory(op=op)
datatype.number1 = range_val1
datatype.number2 = range_val2
datatype.fk_datatype_group = datatype_group.id
datatype.operation_id = op.id
dao.store_entity(datatype)
op_ms = Operation(user.id,
project.id,
algorithm.id,
'test parameters',
status=STATUS_FINISHED,
range_values=json.dumps({
range_1[0]:
range_val1,
range_2[0]:
range_val2
}))
op_ms.fk_operation_group = group_ms.id
op_ms = dao.store_entity(op_ms)
datatype_measure_factory(datatype, op_ms, dt_group_ms)
return datatype_group
def build(project=None, store_vm=False, use_time_series_region=False, status=STATUS_FINISHED):
# there store the name and the (hi, lo, step) value of the range parameters
range_1 = ["row1", [1, 2, 6]]
range_2 = ["row2", [0.1, 0.3, 0.5]]
# there are the actual numbers in the interval
range_values_1 = [1, 3, 5]
range_values_2 = [0.1, 0.4]
user = user_factory()
if project is None:
project = project_factory(user)
connectivity = connectivity_factory(4)
if use_time_series_region:
operation = operation_factory(test_project=project)
connectivity_index_factory(op=operation, conn=connectivity)
operation2 = operation_factory(test_project=project)
surface = surface_factory()
surface_index_factory(op=operation2, surface=surface)
operation3 = operation_factory(test_project=project)
region_mapping = region_mapping_factory(surface=surface, connectivity=connectivity)
region_mapping_index_factory(op=operation3, conn_gid=connectivity.gid.hex, surface_gid=surface.gid.hex, region_mapping=region_mapping)
algorithm = dao.get_algorithm_by_module(SIMULATOR_MODULE, SIMULATOR_CLASS)
adapter = ABCAdapter.build_adapter(algorithm)
if store_vm:
view_model = adapter.get_view_model_class()()
view_model.connectivity = connectivity.gid
else:
view_model = None
algorithm_ms = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS)
adapter = ABCAdapter.build_adapter(algorithm_ms)
view_model_ms = adapter.get_view_model_class()()
op_group = OperationGroup(project.id, ranges=[json.dumps(range_1), json.dumps(range_2)])
op_group = dao.store_entity(op_group)
op_group_ms = OperationGroup(project.id, ranges=[json.dumps(range_1), json.dumps(range_2)])
op_group_ms = dao.store_entity(op_group_ms)
datatype_group = DataTypeGroup(op_group, state="RAW_DATA")
datatype_group.no_of_ranges = 2
datatype_group.count_results = 6
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = DataTypeGroup(op_group_ms, state="RAW_DATA")
dt_group_ms.no_of_ranges = 2
dt_group_ms.count_results = 6
dao.store_entity(dt_group_ms)
# Now create some data types and add them to group
for range_val1 in range_values_1:
for range_val2 in range_values_2:
view_model_gid = uuid.uuid4()
view_model_ms_gid = uuid.uuid4()
op = Operation(view_model_gid.hex, user.id, project.id, algorithm.id,
status=status, op_group_id=op_group.id,
range_values=json.dumps({range_1[0]: range_val1,
range_2[0]: range_val2}))
op = dao.store_entity(op)
if use_time_series_region:
ts = time_series_region_factory(connectivity=connectivity, region_mapping=region_mapping)
ts_index = time_series_region_index_factory(ts=ts, connectivity=connectivity,
region_mapping=region_mapping, test_user=user,
test_project=project, op=op)
else:
ts = time_series_factory()
ts_index = time_series_index_factory(ts=ts, op=op)
ts_index.fk_datatype_group = datatype_group.id
dao.store_entity(ts_index)
op_ms = Operation(view_model_ms_gid.hex, user.id, project.id, algorithm.id,
status=STATUS_FINISHED, op_group_id=op_group_ms.id,
range_values=json.dumps({range_1[0]: range_val1,
range_2[0]: range_val2}))
op_ms = dao.store_entity(op_ms)
datatype_measure_factory(ts_index, ts, op_ms, dt_group_ms)
if store_vm:
view_model = copy.deepcopy(view_model)
view_model.gid = view_model_gid
op_path = StorageInterface().get_project_folder(project.name, str(op.id))
h5.store_view_model(view_model, op_path)
view_model_ms = copy.deepcopy(view_model_ms)
view_model_ms.gid = view_model_ms_gid
view_model_ms.time_series = ts_index.gid
op_ms_path = StorageInterface().get_project_folder(project.name, str(op_ms.id))
h5.store_view_model(view_model_ms, op_ms_path)
if not datatype_group.fk_from_operation:
# Mark first operation ID
datatype_group.fk_from_operation = op.id
dt_group_ms.fk_from_operation = op_ms.id
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = dao.store_entity(dt_group_ms)
return datatype_group, dt_group_ms
def _retrieve_operations_in_order(self, project, import_path):
# type: (Project, str) -> list[Operation2ImportData]
retrieved_operations = []
for root, _, files in os.walk(import_path):
if OPERATION_XML in files:
# Previous Operation format for uploading previous versions of projects
operation_file_path = os.path.join(root, OPERATION_XML)
operation, operation_xml_parameters = self.__build_operation_from_file(
project, operation_file_path)
operation.import_file = operation_file_path
self.logger.debug("Found operation in old XML format: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(
operation,
root,
info_from_xml=operation_xml_parameters))
else:
# We strive for the new format with ViewModelH5
main_view_model = None
dt_paths = []
all_view_model_files = []
for file in files:
if file.endswith(FilesHelper.TVB_STORAGE_FILE_EXTENSION):
h5_file = os.path.join(root, file)
try:
h5_class = H5File.h5_class_from_file(h5_file)
if h5_class is ViewModelH5:
all_view_model_files.append(h5_file)
if not main_view_model:
view_model = h5.load_view_model_from_file(
h5_file)
if type(view_model
) in VIEW_MODEL2ADAPTER.keys():
main_view_model = view_model
else:
file_update_manager = FilesUpdateManager()
file_update_manager.upgrade_file(h5_file)
dt_paths.append(h5_file)
except Exception:
self.logger.warning(
"Unreadable H5 file will be ignored: %s" %
h5_file)
if main_view_model is not None:
alg = VIEW_MODEL2ADAPTER[type(main_view_model)]
operation = Operation(main_view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
user_group=main_view_model.
generic_attributes.operation_tag,
start_date=datetime.now(),
completion_date=datetime.now())
operation.create_date = main_view_model.create_date
self.logger.debug(
"Found main ViewModel to create operation for it: " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, main_view_model,
dt_paths, all_view_model_files))
elif len(dt_paths) > 0:
alg = dao.get_algorithm_by_module(TVB_IMPORTER_MODULE,
TVB_IMPORTER_CLASS)
default_adapter = ABCAdapter.build_adapter(alg)
view_model = default_adapter.get_view_model_class()()
view_model.data_file = dt_paths[0]
vm_path = h5.store_view_model(view_model, root)
all_view_model_files.append(vm_path)
operation = Operation(view_model.gid.hex,
project.fk_admin,
project.id,
alg.id,
status=STATUS_FINISHED,
start_date=datetime.now(),
completion_date=datetime.now())
self.logger.debug(
"Found no ViewModel in folder, so we default to " +
str(operation))
retrieved_operations.append(
Operation2ImportData(operation, root, view_model,
dt_paths, all_view_model_files,
True))
return sorted(retrieved_operations,
key=lambda op_data: op_data.order_field)
def build(subject="Datatype Factory User", state="RAW_DATA", project=None):
range_1 = ["row1", [1, 2, 3]]
range_2 = ["row2", [0.1, 0.3, 0.5]]
user = user_factory()
if project is None:
project = project_factory(user)
# Create an algorithm
alg_category = AlgorithmCategory('one', True)
dao.store_entity(alg_category)
ad = Algorithm(IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS, alg_category.id)
algorithm = dao.get_algorithm_by_module(
IntrospectionRegistry.SIMULATOR_MODULE,
IntrospectionRegistry.SIMULATOR_CLASS)
if algorithm is None:
algorithm = dao.store_entity(ad)
# Create meta
meta = {
DataTypeMetaData.KEY_SUBJECT: "Datatype Factory User",
DataTypeMetaData.KEY_STATE: "RAW_DATA"
}
# Create operation
operation = operation_factory(algorithm=algorithm,
test_user=user,
test_project=project,
meta=meta)
group = OperationGroup(
project.id, ranges=[json.dumps(range_1),
json.dumps(range_2)])
group = dao.store_entity(group)
group_ms = OperationGroup(
project.id, ranges=[json.dumps(range_1),
json.dumps(range_2)])
group_ms = dao.store_entity(group_ms)
datatype_group = DataTypeGroup(group,
subject=subject,
state=state,
operation_id=operation.id)
datatype_group = dao.store_entity(datatype_group)
dt_group_ms = DataTypeGroup(group_ms,
subject=subject,
state=state,
operation_id=operation.id)
dao.store_entity(dt_group_ms)
# Now create some data types and add them to group
for range_val1 in range_1[1]:
for range_val2 in range_2[1]:
op = Operation(user.id,
project.id,
algorithm.id,
'test parameters',
meta=json.dumps(meta),
status=STATUS_FINISHED,
range_values=json.dumps({
range_1[0]: range_val1,
range_2[0]: range_val2
}))
op.fk_operation_group = group.id
op = dao.store_entity(op)
datatype = time_series_index_factory(op=op)
datatype.number1 = range_val1
datatype.number2 = range_val2
datatype.fk_datatype_group = datatype_group.id
datatype.operation_id = op.id
dao.store_entity(datatype)
op_ms = Operation(user.id,
project.id,
algorithm.id,
'test parameters',
meta=json.dumps(meta),
status=STATUS_FINISHED,
range_values=json.dumps({
range_1[0]:
range_val1,
range_2[0]:
range_val2
}))
op_ms.fk_operation_group = group_ms.id
op_ms = dao.store_entity(op_ms)
datatype_measure_factory(datatype)
return datatype_group
def prepare_operations(self,
user_id,
project_id,
algorithm,
category,
metadata,
visible=True,
existing_dt_group=None,
**kwargs):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
:param metadata: Initial MetaData with potential Burst identification inside.
"""
operations = []
available_args, group = self._prepare_group(project_id,
existing_dt_group, kwargs)
if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER:
raise LaunchException(
"Too big range specified. You should limit the"
" resulting operations to %d" %
TvbProfile.current.MAX_RANGE_NUMBER)
else:
self.logger.debug("Launching a range with %d operations..." %
len(available_args))
group_id = None
if group is not None:
group_id = group.id
metadata, user_group = self._prepare_metadata(metadata, category,
group, kwargs)
self.logger.debug("Saving Operation(userId=" + str(user_id) +
",projectId=" + str(project_id) + "," +
str(metadata) + ",algorithmId=" + str(algorithm.id) +
", ops_group= " + str(group_id) + ")")
visible_operation = visible and category.display is False
meta_str = json.dumps(metadata)
for (one_set_of_args, range_vals) in available_args:
range_values = json.dumps(range_vals) if range_vals else None
operation = Operation(user_id,
project_id,
algorithm.id,
json.dumps(one_set_of_args),
meta_str,
op_group_id=group_id,
user_group=user_group,
range_values=range_values)
operation.visible = visible_operation
operations.append(operation)
operations = dao.store_entities(operations)
if group is not None:
burst_id = None
if DataTypeMetaData.KEY_BURST in metadata:
burst_id = metadata[DataTypeMetaData.KEY_BURST]
if existing_dt_group is None:
datatype_group = DataTypeGroup(
group,
operation_id=operations[0].id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
else:
# Reset count
existing_dt_group.count_results = None
dao.store_entity(existing_dt_group)
return operations, group
def launch_sim(self, user_id, project, algorithm, zip_folder_path,
simulator_file):
return Operation('', '', '', {})
def prepare_operations_for_workflowsteps(self, workflow_step_list,
workflows, user_id, burst_id,
project_id, group,
sim_operations):
"""
Create and store Operation entities from a list of Workflow Steps.
Will be generated workflows x workflow_step_list Operations.
For every step in workflow_step_list one OperationGroup and one DataTypeGroup will be created
(in case of PSE).
"""
for step in workflow_step_list:
operation_group = None
if (group is not None) and not isinstance(step, WorkflowStepView):
operation_group = OperationGroup(project_id=project_id,
ranges=group.range_references)
operation_group = dao.store_entity(operation_group)
operation = None
metadata = {DataTypeMetaData.KEY_BURST: burst_id}
algo_category = dao.get_algorithm_by_id(step.fk_algorithm)
if algo_category is not None:
algo_category = algo_category.algorithm_category
for wf_idx, workflow in enumerate(workflows):
cloned_w_step = step.clone()
cloned_w_step.fk_workflow = workflow.id
dynamic_params = cloned_w_step.dynamic_param
op_params = cloned_w_step.static_param
op_params.update(dynamic_params)
range_values = None
group_id = None
if operation_group is not None:
group_id = operation_group.id
range_values = sim_operations[wf_idx].range_values
if not isinstance(step, WorkflowStepView):
## For visualization steps, do not create operations, as those are not really needed.
metadata, user_group = self._prepare_metadata(
metadata, algo_category, operation_group, op_params)
operation = Operation(user_id,
project_id,
step.fk_algorithm,
json.dumps(
op_params,
cls=MapAsJson.MapAsJsonEncoder),
meta=json.dumps(metadata),
op_group_id=group_id,
range_values=range_values,
user_group=user_group)
operation.visible = step.step_visible
operation = dao.store_entity(operation)
cloned_w_step.fk_operation = operation.id
dao.store_entity(cloned_w_step)
if operation_group is not None and operation is not None:
datatype_group = DataTypeGroup(
operation_group,
operation_id=operation.id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
