def _check_datatype_group_removed(self, datatype_group_id, operation_groupp_id): """ Checks if the DataTypeGroup and OperationGroup was removed. """ try: dao.get_generic_entity(DataTypeGroup, datatype_group_id) raise AssertionError("The DataTypeGroup entity was not removed.") except Exception: pass try: dao.get_operationgroup_by_id(operation_groupp_id) raise AssertionError("The OperationGroup entity was not removed.") except Exception: pass
def _check_datatype_group_removed(self, datatype_group_id, operation_groupp_id): """ Checks if the DataTypeGroup and OperationGroup was removed. """ try: dao.get_generic_entity(model.DataTypeGroup, datatype_group_id) self.fail("The DataTypeGroup entity was not removed.") except Exception: pass try: dao.get_operationgroup_by_id(operation_groupp_id) self.fail("The OperationGroup entity was not removed.") except Exception: pass
def upgrade(migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ meta.bind = migrate_engine table = meta.tables['DATA_TYPES_GROUPS'] create_column(COL_RANGES_1, table) create_column(COL_RANGES_2, table) try: ## Iterate DataTypeGroups from previous code-versions and try to update value for the new column. previous_groups = dao.get_generic_entity(model.DataTypeGroup, "0", "no_of_ranges") for group in previous_groups: operation_group = dao.get_operationgroup_by_id(group.fk_operation_group) #group.only_numeric_ranges = operation_group.has_only_numeric_ranges if operation_group.range3 is not None: group.no_of_ranges = 3 elif operation_group.range2 is not None: group.no_of_ranges = 2 elif operation_group.range1 is not None: group.no_of_ranges = 1 else: group.no_of_ranges = 0 dao.store_entity(group) except Exception, excep: ## we can live with a column only having default value. We will not stop the startup. logger = get_logger(__name__) logger.exception(excep)
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. :param datatype_group_gid: the group id for the `DataType` to be visualised :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :param size_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :returns: `ContextDiscretePSE` :raises Exception: when `datatype_group_id` is invalid (not in database) """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception("Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one.") operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) range1_name, range1_values, range1_labels = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range1) range2_name, range2_values, range2_labels = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page) pse_context.setRanges(range1_name, range1_values, range1_labels, range2_name, range2_values, range2_labels) final_dict = {} operations = dao.get_operations_in_group(operation_group.id) for operation_ in operations: if not operation_.has_finished: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = range_values[range1_name] key_2 = model.RANGE_MISSING_STRING if range2_name is not None: key_2 = range_values[range2_name] datatype = None if operation_.status == model.STATUS_FINISHED: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = {} final_dict[key_1][key_2] = pse_context.build_node_info(operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} return pse_context
def import_operation(operation_entity, migration=False): """ Store a Operation entity. """ do_merge = False if operation_entity.id: do_merge = True operation_entity = dao.store_entity(operation_entity, merge=do_merge) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) if datatype_group is None and migration is False: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id( operation_group_id) datatype_group = DataTypeGroup( operation_group, operation_id=operation_entity.id) datatype_group.state = UploadAlgorithmCategoryConfig.defaultdatastate datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def create_view_model(self, operation_entity, operation_data, new_op_folder, generic_attributes=None, add_params=None): view_model = self._get_new_form_view_model( operation_entity, operation_data.info_from_xml) if add_params is not None: for element in add_params: key_attr = getattr(view_model, element[0]) setattr(key_attr, element[1], element[2]) view_model.range_values = operation_entity.range_values op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: view_model.operation_group_gid = uuid.UUID(op_group.gid) view_model.ranges = json.dumps(op_group.range_references) view_model.is_metric_operation = 'DatatypeMeasure' in op_group.name if generic_attributes is not None: view_model.generic_attributes = generic_attributes view_model.generic_attributes.operation_tag = operation_entity.user_group h5.store_view_model(view_model, new_op_folder) view_model_disk_size = StorageInterface.compute_recursive_h5_disk_usage( new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size operation_entity.view_model_gid = view_model.gid.hex dao.store_entity(operation_entity) return view_model
def prepare_indexes_for_simulation_results(self, operation, result_filenames, burst): indexes = list() self.logger.debug("Preparing indexes for simulation results in operation {}...".format(operation.id)) for filename in result_filenames: try: self.logger.debug("Preparing index for filename: {}".format(filename)) index = h5.index_for_h5_file(filename)() h5_class = h5.REGISTRY.get_h5file_for_index(type(index)) with h5_class(filename) as index_h5: index.fill_from_h5(index_h5) index.fill_from_generic_attributes(index_h5.load_generic_attributes()) index.fk_parent_burst = burst.gid index.fk_from_operation = operation.id if operation.fk_operation_group: datatype_group = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group) self.logger.debug( "Found DatatypeGroup with id {} for operation {}".format(datatype_group.id, operation.id)) index.fk_datatype_group = datatype_group.id # Update the operation group name operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group) operation_group.fill_operationgroup_name("TimeSeriesRegionIndex") dao.store_entity(operation_group) self.logger.debug( "Prepared index {} for file {} in operation {}".format(index.summary_info, filename, operation.id)) indexes.append(index) except Exception as e: self.logger.debug("Skip preparing index {} because there was an error.".format(filename)) self.logger.error(e) self.logger.debug("Prepared {} indexes for results in operation {}...".format(len(indexes), operation.id)) return indexes
def update_db_with_results(operation, sim_h5_filenames, metric_operation, metric_h5_filename): # type: (Operation, list, Operation, str) -> (str, int) """ Generate corresponding Index entities for the resulted H5 files and insert them in DB. """ burst_service = BurstService() index_list = [] is_group = operation.fk_operation_group is not None burst_config = burst_service.get_burst_for_operation_id(operation.id) if is_group: burst_config = burst_service.get_burst_for_operation_id( operation.fk_operation_group, True) all_indexes = burst_service.prepare_indexes_for_simulation_results( operation, sim_h5_filenames, burst_config) if is_group: # Update the operation group name operation_group = dao.get_operationgroup_by_id( metric_operation.fk_operation_group) operation_group.fill_operationgroup_name("DatatypeMeasureIndex") dao.store_entity(operation_group) metric_index = burst_service.prepare_index_for_metric_result( metric_operation, metric_h5_filename, burst_config) all_indexes.append(metric_index) for index in all_indexes: index = dao.store_entity(index) index_list.append(index) burst_service.update_burst_status(burst_config)
def _prepare_metric_operation(self, sim_operation): # type: (Operation) -> Operation metric_algo = dao.get_algorithm_by_module(MEASURE_METRICS_MODULE, MEASURE_METRICS_CLASS) datatype_index = h5.REGISTRY.get_index_for_datatype(TimeSeries) time_series_index = dao.get_generic_entity(datatype_index, sim_operation.id, 'fk_from_operation')[0] ga = self.prepare_metadata(metric_algo.algorithm_category, time_series_index.fk_parent_burst) ga.visible = False view_model = get_class_by_name("{}.{}".format(MEASURE_METRICS_MODULE, MEASURE_METRICS_MODEL_CLASS))() view_model.time_series = time_series_index.gid view_model.algorithms = tuple(ALGORITHMS.keys()) view_model.generic_attributes = ga parent_burst = dao.get_generic_entity(BurstConfiguration, time_series_index.fk_parent_burst, 'gid')[0] metric_op_group = dao.get_operationgroup_by_id(parent_burst.fk_metric_operation_group) metric_operation_group_id = parent_burst.fk_metric_operation_group range_values = sim_operation.range_values view_model.operation_group_gid = uuid.UUID(metric_op_group.gid) view_model.ranges = json.dumps(parent_burst.ranges) view_model.range_values = range_values view_model.is_metric_operation = True metric_operation = Operation(view_model.gid.hex, sim_operation.fk_launched_by, sim_operation.fk_launched_in, metric_algo.id, user_group=ga.operation_tag, op_group_id=metric_operation_group_id, range_values=range_values) metric_operation.visible = False metric_operation = dao.store_entity(metric_operation) metrics_datatype_group = dao.get_generic_entity(DataTypeGroup, metric_operation_group_id, 'fk_operation_group')[0] if metrics_datatype_group.fk_from_operation is None: metrics_datatype_group.fk_from_operation = metric_operation.id dao.store_entity(metrics_datatype_group) self.store_view_model(metric_operation, sim_operation.project, view_model) return metric_operation
def upgrade(migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ meta.bind = migrate_engine table = meta.tables['DATA_TYPES_GROUPS'] create_column(COL_RANGES_1, table) create_column(COL_RANGES_2, table) try: ## Iterate DataTypeGroups from previous code-versions and try to update value for the new column. previous_groups = dao.get_generic_entity(model.DataTypeGroup, "0", "no_of_ranges") for group in previous_groups: operation_group = dao.get_operationgroup_by_id(group.fk_operation_group) #group.only_numeric_ranges = operation_group.has_only_numeric_ranges if operation_group.range3 is not None: group.no_of_ranges = 3 elif operation_group.range2 is not None: group.no_of_ranges = 2 elif operation_group.range1 is not None: group.no_of_ranges = 1 else: group.no_of_ranges = 0 dao.store_entity(group) except Exception as excep: ## we can live with a column only having default value. We will not stop the startup. logger = get_logger(__name__) logger.exception(excep) session = SA_SESSIONMAKER() session.execute(text("""UPDATE "OPERATIONS" SET status = CASE WHEN status = 'FINISHED' THEN '4-FINISHED' WHEN status = 'STARTED' THEN '3-STARTED' WHEN status = 'CANCELED' THEN '2-CANCELED' ELSE '1-ERROR' END WHERE status IN ('FINISHED', 'CANCELED', 'STARTED', 'ERROR');""")) session.commit() session.close() try: session = SA_SESSIONMAKER() for sim_state in session.query(SimulationState).filter(SimulationState.fk_datatype_group is not None).all(): session.delete(sim_state) session.commit() session.close() except Exception as excep: ## It might happen that SimulationState table is not yet created, e.g. if user has version 1.0.2 logger = get_logger(__name__) logger.exception(excep)
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) _, range1_name, range1_labels = operation_group.load_range_numbers( operation_group.range1) has_range2, range2_name, range2_labels = operation_group.load_range_numbers( operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, range1_labels, range2_labels, color_metric, size_metric, back_page) final_dict = dict() operations = dao.get_operations_in_group(operation_group.id) for operation_ in operations: if operation_.status == model.STATUS_STARTED: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = range_values[range1_name] key_2 = model.RANGE_MISSING_STRING if has_range2 is not None: key_2 = range_values[range2_name] datatype = None if operation_.status == model.STATUS_FINISHED: datatype = dao.get_results_for_operation(operation_.id)[0] measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = { key_2: pse_context.build_node_info(operation_, datatype) } else: final_dict[key_1][key_2] = pse_context.build_node_info( operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} return pse_context
def launch(self, datatype_group, **kwargs): """ Also overwrite launch from ABCDisplayer, since we want to handle a list of figures, instead of only one Matplotlib figure. :raises LaunchException: when called before all operations in the group are finished """ if self.PARAM_FIGURE_SIZE in kwargs: figsize = kwargs[self.PARAM_FIGURE_SIZE] figsize = ((figsize[0]) / 80, (figsize[1]) / 80) del kwargs[self.PARAM_FIGURE_SIZE] else: figsize = (15, 7) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) _, range1_name, self.range1 = operation_group.load_range_numbers( operation_group.range1) _, range2_name, self.range2 = operation_group.load_range_numbers( operation_group.range2) # Get the computed measures on this DataTypeGroup first_op = dao.get_operations_in_group(operation_group.id)[0] if first_op.status != model.STATUS_FINISHED: raise LaunchException( "Not all operations from this range are finished. Cannot generate data until then." ) datatype = dao.get_results_for_operation(first_op.id)[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.id)[0] else: dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') if dt_measure: dt_measure = dt_measure[0] figure_nrs = {} metrics = dt_measure.metrics if dt_measure else [] for metric in metrics: # Separate plot for each metric. self._create_plot(metric, figsize, operation_group, range1_name, range2_name, figure_nrs) parameters = dict(title=self._ui_name, showFullToolbar=True, serverIp=config.SERVER_IP, serverPort=config.MPLH5_SERVER_PORT, figureNumbers=figure_nrs, metrics=metrics, figuresJSON=json.dumps(figure_nrs)) return self.build_display_result("pse_isocline/view", parameters)
def _capture_operation_results(self, result, user_tag=None): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ results_to_store = [] data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id # All entities will have the same subject and state subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT] state = self.meta_data[DataTypeMetaData.KEY_STATE] burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] perpetuated_identifier = None if DataTypeMetaData.KEY_TAG_1 in self.meta_data: perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1] for res in result: if res is None: continue res.subject = str(subject) res.state = state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data if not res.user_tag_1: res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier else: res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier res.fk_datatype_group = data_type_group_id ## Compute size-on disk, in case file-storage is used if hasattr(res, 'storage_path') and hasattr(res, 'get_storage_file_name'): associated_file = os.path.join(res.storage_path, res.get_storage_file_name()) res.close_file() res.disk_size = self.file_handler.compute_size_on_disk(associated_file) res = dao.store_entity(res) # Write metaData res.persist_full_metadata() results_to_store.append(res) del result[0:len(result)] result.extend(results_to_store) if len(result) and self._is_group_launch(): ## Update the operation group name operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group) operation_group.fill_operationgroup_name(result[0].type) dao.store_entity(operation_group) return 'Operation ' + str(self.operation_id) + ' has finished.', len(results_to_store)
def from_db(cls, operation_group_id): """ Collects from db the information about the operation group that is required by the isocline view. """ operations = dao.get_operations_in_group(operation_group_id) operation_group = dao.get_operationgroup_by_id(operation_group_id) self = cls(operation_group.range1, operation_group.range2, {}, PseIsoModel._find_metrics(operations), None) self._fill_apriori_data(operations) return self
def _remove_project_node_files(self, project_id, gid, links, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) if links: op = dao.get_operation_by_id(datatype.fk_from_operation) # Instead of deleting, we copy the datatype to the linked project # We also clone the operation new_operation = self.__copy_linked_datatype_before_delete(op, datatype, project, links[0].fk_to_project) # If there is a datatype group and operation group and they were not moved yet to the linked project, # then do it if datatype.fk_datatype_group is not None: dt_group_op = dao.get_operation_by_id(datatype.fk_from_operation) op_group = dao.get_operationgroup_by_id(dt_group_op.fk_operation_group) op_group.fk_launched_in = links[0].fk_to_project dao.store_entity(op_group) burst = dao.get_burst_for_operation_id(op.id) if burst is not None: burst.fk_project = links[0].fk_to_project dao.store_entity(burst) dt_group = dao.get_datatypegroup_by_op_group_id(op_group.id) dt_group.parent_operation = new_operation dt_group.fk_from_operation = new_operation.id dao.store_entity(dt_group) else: # There is no link for this datatype so it has to be deleted specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) # Remove burst if dt has one and it still exists if datatype.fk_parent_burst is not None and datatype.is_ts: burst = dao.get_burst_for_operation_id(datatype.fk_from_operation) if burst is not None: dao.remove_entity(BurstConfiguration, burst.id) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def test_set_op_group_visibility(self): """ Tests if the visibility for an operation group is set correct. """ _, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") list_of_operations = dao.get_operations_in_group(group_id) for operation in list_of_operations: self.assertTrue(operation.visible, "The operation should be visible.") op_group = dao.get_operationgroup_by_id(group_id) self.project_service.set_operation_and_group_visibility(op_group.gid, False, True) operations = dao.get_operations_in_group(group_id) for operation in operations: self.assertFalse(operation.visible, "The operation should not be visible.")
def test_set_op_group_visibility(self): """ Tests if the visibility for an operation group is set correct. """ _, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") list_of_operations = dao.get_operations_in_group(group_id) for operation in list_of_operations: assert operation.visible, "The operation should be visible." op_group = dao.get_operationgroup_by_id(group_id) self.project_service.set_operation_and_group_visibility(op_group.gid, False, True) operations = dao.get_operations_in_group(group_id) for operation in operations: assert not operation.visible, "The operation should not be visible."
def _capture_operation_results(self, result): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] count_stored = 0 group_type = None # In case of a group, the first not-none type is sufficient to memorize here for res in result: if res is None: continue res.subject = self.generic_attributes.subject res.state = self.generic_attributes.state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data res.user_tag_1 = self.generic_attributes.user_tag_1 res.user_tag_2 = self.generic_attributes.user_tag_2 res.fk_datatype_group = data_type_group_id # Compute size-on disk, in case file-storage is used associated_file = h5.path_for_stored_index(res) if os.path.exists(associated_file): res.disk_size = self.file_handler.compute_size_on_disk( associated_file) with H5File.from_file(associated_file) as f: f.store_generic_attributes(self.generic_attributes) dao.store_entity(res) group_type = res.type count_stored += 1 if count_stored > 0 and self._is_group_launch(): # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(group_type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', count_stored
def _capture_operation_results(self, result): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id count_stored = 0 if result is None: return "", count_stored group_type = None # In case of a group, the first not-none type is sufficient to memorize here for res in result: if res is None: continue if not res.fixed_generic_attributes: res.fill_from_generic_attributes(self.generic_attributes) res.fk_from_operation = self.operation_id res.fk_datatype_group = data_type_group_id associated_file = h5.path_for_stored_index(res) if os.path.exists(associated_file): if not res.fixed_generic_attributes: with H5File.from_file(associated_file) as f: f.store_generic_attributes(self.generic_attributes) # Compute size-on disk, in case file-storage is used res.disk_size = self.storage_interface.compute_size_on_disk( associated_file) dao.store_entity(res) res.after_store() group_type = res.type count_stored += 1 if count_stored > 0 and self._is_group_launch(): # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(group_type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', count_stored
def test_set_op_group_visibility(self, datatype_group_factory): """ Tests if the visibility for an operation group is set correct. """ group = datatype_group_factory() list_of_operations = dao.get_operations_in_group(group.id) for operation in list_of_operations: assert operation.visible, "The operation should be visible." op_group = dao.get_operationgroup_by_id(group.id) self.project_service.set_operation_and_group_visibility( op_group.gid, False, True) operations = dao.get_operations_in_group(group.id) for operation in operations: assert not operation.visible, "The operation should not be visible."
def launch(self, datatype_group, **kwargs): """ Also overwrite launch from ABCDisplayer, since we want to handle a list of figures, instead of only one Matplotlib figure. :raises LaunchException: when called before all operations in the group are finished """ if self.PARAM_FIGURE_SIZE in kwargs: figsize = kwargs[self.PARAM_FIGURE_SIZE] figsize = ((figsize[0]) / 80, (figsize[1]) / 80) del kwargs[self.PARAM_FIGURE_SIZE] else: figsize = (15, 7) operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) _, range1_name, self.range1 = operation_group.load_range_numbers(operation_group.range1) _, range2_name, self.range2 = operation_group.load_range_numbers(operation_group.range2) for operation in dao.get_operations_in_group(operation_group.id): if operation.status == model.STATUS_STARTED: raise LaunchException("Can not display until all operations from this range are finished!") op_results = dao.get_results_for_operation(operation.id) if len(op_results): datatype = op_results[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.id)[0] else: dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') if dt_measure: dt_measure = dt_measure[0] break else: dt_measure = None figure_nrs = {} metrics = dt_measure.metrics if dt_measure else {} if metrics: for metric in metrics: # Separate plot for each metric. self._create_plot(metric, figsize, operation_group, range1_name, range2_name, figure_nrs) else: raise LaunchException("No datatypes were generated due to simulation errors. Nothing to display.") parameters = dict(title=self._ui_name, showFullToolbar=True, serverIp=config.SERVER_IP, serverPort=config.MPLH5_SERVER_PORT, figureNumbers=figure_nrs, metrics=metrics, figuresJSON=json.dumps(figure_nrs)) return self.build_display_result("pse_isocline/view", parameters)
def __init__(self, datatype_group_gid): self.datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if self.datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) self.operation_group = dao.get_operationgroup_by_id( self.datatype_group.fk_operation_group) self.operations = dao.get_operations_in_group(self.operation_group.id) self.pse_model_list = self.parse_pse_data_for_display() self.all_metrics = dict() self._prepare_ranges_data()
def set_operation_and_group_visibility(entity_gid, is_visible, is_operation_group=False): """ Sets the operation visibility. If 'is_operation_group' is True than this method will change the visibility for all the operation from the OperationGroup with the GID field equal to 'entity_gid'. """ if not is_operation_group: #we assure that if the operation belongs to a group than the visibility will be changed for the entire group operation = dao.get_operation_by_gid(entity_gid) if operation.fk_operation_group is not None: op_group = dao.get_operationgroup_by_id(operation.fk_operation_group) entity_gid = op_group.gid is_operation_group = True dao.set_operation_and_group_visibility(entity_gid, is_visible, is_operation_group)
def load_datatype_from_file(self, current_file, op_id, datatype_group=None, current_project_id=None): # type: (str, int, DataTypeGroup, int) -> HasTraitsIndex """ Creates an instance of datatype from storage / H5 file :returns: DatatypeIndex """ self.logger.debug("Loading DataType from file: %s" % current_file) h5_class = H5File.h5_class_from_file(current_file) if h5_class is BurstConfigurationH5: if current_project_id is None: op_entity = dao.get_operationgroup_by_id(op_id) current_project_id = op_entity.fk_launched_in h5_file = BurstConfigurationH5(current_file) burst = BurstConfiguration(current_project_id) burst.fk_simulation = op_id h5_file.load_into(burst) result = burst else: datatype, generic_attributes = h5.load_with_links(current_file) index_class = h5.REGISTRY.get_index_for_datatype( datatype.__class__) datatype_index = index_class() datatype_index.fill_from_has_traits(datatype) datatype_index.fill_from_generic_attributes(generic_attributes) # Add all the required attributes if datatype_group: datatype_index.fk_datatype_group = datatype_group.id if len(datatype_group.subject) == 0: datatype_group.subject = datatype_index.subject dao.store_entity(datatype_group) datatype_index.fk_from_operation = op_id associated_file = h5.path_for_stored_index(datatype_index) if os.path.exists(associated_file): datatype_index.disk_size = FilesHelper.compute_size_on_disk( associated_file) result = datatype_index return result
def prepare_node_data(datatype_group): if datatype_group is None: raise Exception("Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one.") operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) operations = dao.get_operations_in_group(operation_group.id) node_info_dict = dict() for operation_ in operations: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] node_info_dict[datatype.gid] = dict(operation_id=operation_.id, datatype_gid=datatype.gid, datatype_type=datatype.type, datatype_subject=datatype.subject, datatype_invalid=datatype.invalid) return node_info_dict
def __import_operation(operation_entity): """ Store a Operation entity. """ operation_entity = dao.store_entity(operation_entity) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: try: datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) except SQLAlchemyError: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id(operation_group_id) datatype_group = DataTypeGroup(operation_group, operation_id=operation_entity.id) datatype_group.state = UploadAlgorithmCategoryConfig.defaultdatastate datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def __import_operation(operation_entity): """ Store a Operation entity. """ operation_entity = dao.store_entity(operation_entity) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: try: datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) except SQLAlchemyError: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id(operation_group_id) datatype_group = model.DataTypeGroup(operation_group, operation_id=operation_entity.id) datatype_group.state = ADAPTERS['Upload']['defaultdatastate'] datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def prepare_node_data(datatype_group): if datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) operations = dao.get_operations_in_group(operation_group.id) node_info_dict = dict() for operation_ in operations: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] node_info_dict[datatype.gid] = dict( operation_id=operation_.id, datatype_gid=datatype.gid, datatype_type=datatype.type, datatype_subject=datatype.subject, datatype_invalid=datatype.invalid) return node_info_dict
def get_operation_group_by_id(operation_group_id): """ Loads OperationGroup from DB""" return dao.get_operationgroup_by_id(operation_group_id)
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. :param datatype_group_gid: the group id for the `DataType` to be visualised :param back_page: Page where back button will direct :param color_metric: String referring to metric to apply on colors :param size_metric: String referring to metric to apply on sizes :returns: `ContextDiscretePSE` :raises Exception: when `datatype_group_id` is invalid (not in database) """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception("Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one.") operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) name1, values1, labels1, only_numbers1 = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range1) name2, values2, labels2, only_numbers2 = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page) pse_context.setRanges(name1, values1, labels1, name2, values2, labels2, only_numbers1 and only_numbers2) final_dict = {} operations = dao.get_operations_in_group(operation_group.id) fake_numbers1 = dict(zip(values1, range(len(list(values1))))) fake_numbers2 = dict(zip(values2, range(len(list(values2))))) for operation_ in operations: if not operation_.has_finished: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = DiscretePSEAdapter.get_value_on_axe(range_values, only_numbers1, name1, fake_numbers1) key_2 = DiscretePSEAdapter.get_value_on_axe(range_values, only_numbers2, name2, fake_numbers2) datatype = None if operation_.status == model.STATUS_FINISHED: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = {} final_dict[key_1][key_2] = pse_context.build_node_info(operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} if not only_numbers1: pse_context.values_x = range(len(list(values1))) if not only_numbers2: pse_context.values_y = range(len(list(values2))) return pse_context
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. :param datatype_group_gid: the group id for the `DataType` to be visualised :param back_page: Page where back button will direct :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :param size_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :returns: `ContextDiscretePSE` :raises Exception: when `datatype_group_id` is invalid (not in database) """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) name1, values1, labels1, only_numbers1 = DiscretePSEAdapter.prepare_range_labels( operation_group, operation_group.range1) name2, values2, labels2, only_numbers2 = DiscretePSEAdapter.prepare_range_labels( operation_group, operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page) pse_context.setRanges(name1, values1, labels1, name2, values2, labels2, only_numbers1 and only_numbers2) final_dict = {} operations = dao.get_operations_in_group(operation_group.id) fake_numbers1 = dict(zip(values1, range(len(list(values1))))) fake_numbers2 = dict(zip(values2, range(len(list(values2))))) for operation_ in operations: if not operation_.has_finished: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = DiscretePSEAdapter.get_value_on_axe( range_values, only_numbers1, name1, fake_numbers1) key_2 = DiscretePSEAdapter.get_value_on_axe( range_values, only_numbers2, name2, fake_numbers2) datatype = None if operation_.status == model.STATUS_FINISHED: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. measures = dao.get_generic_entity( DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity( DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = {} final_dict[key_1][key_2] = pse_context.build_node_info( operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} if not only_numbers1: pse_context.values_x = range(len(list(values1))) if not only_numbers2: pse_context.values_y = range(len(list(values2))) return pse_context
def upgrade(migrate_engine): """ Upgrade operations go here. Don't create your own engine; bind migrate_engine to your metadata. """ meta.bind = migrate_engine table = meta.tables['DATA_TYPES_GROUPS'] create_column(COL_RANGES_1, table) create_column(COL_RANGES_2, table) try: ## Iterate DataTypeGroups from previous code-versions and try to update value for the new column. previous_groups = dao.get_generic_entity(model.DataTypeGroup, "0", "no_of_ranges") for group in previous_groups: operation_group = dao.get_operationgroup_by_id( group.fk_operation_group) #group.only_numeric_ranges = operation_group.has_only_numeric_ranges if operation_group.range3 is not None: group.no_of_ranges = 3 elif operation_group.range2 is not None: group.no_of_ranges = 2 elif operation_group.range1 is not None: group.no_of_ranges = 1 else: group.no_of_ranges = 0 dao.store_entity(group) except Exception as excep: ## we can live with a column only having default value. We will not stop the startup. logger = get_logger(__name__) logger.exception(excep) session = SA_SESSIONMAKER() session.execute( text("""UPDATE "OPERATIONS" SET status = CASE WHEN status = 'FINISHED' THEN '4-FINISHED' WHEN status = 'STARTED' THEN '3-STARTED' WHEN status = 'CANCELED' THEN '2-CANCELED' ELSE '1-ERROR' END WHERE status IN ('FINISHED', 'CANCELED', 'STARTED', 'ERROR');""" )) session.commit() session.close() try: session = SA_SESSIONMAKER() # TODO: fix me # for sim_state in session.query(SimulationState).filter(SimulationState.fk_datatype_group is not None).all(): # session.delete(sim_state) session.commit() session.close() except Exception as excep: ## It might happen that SimulationState table is not yet created, e.g. if user has version 1.0.2 logger = get_logger(__name__) logger.exception(excep)
def import_list_of_operations(self, project, import_path, is_group=False, importer_operation_id=None): """ This method scans provided folder and identify all operations that needs to be imported """ all_dts_count = 0 all_stored_dts_count = 0 imported_operations = [] ordered_operations = self._retrieve_operations_in_order( project, import_path, None if is_group else importer_operation_id) if is_group and len(ordered_operations) > 0: first_op = dao.get_operation_by_id(importer_operation_id) vm_path = h5.determine_filepath(first_op.view_model_gid, os.path.dirname(import_path)) os.remove(vm_path) ordered_operations[0].operation.id = importer_operation_id for operation_data in ordered_operations: if operation_data.is_old_form: operation_entity, datatype_group = self.import_operation( operation_data.operation) new_op_folder = self.storage_interface.get_project_folder( project.name, str(operation_entity.id)) try: operation_datatypes = self._load_datatypes_from_operation_folder( operation_data.operation_folder, operation_entity, datatype_group) # Create and store view_model from operation self.create_view_model(operation_entity, operation_data, new_op_folder) self._store_imported_datatypes_in_db( project, operation_datatypes) imported_operations.append(operation_entity) except MissingReferenceException: operation_entity.status = STATUS_ERROR dao.store_entity(operation_entity) elif operation_data.main_view_model is not None: operation_data.operation.create_date = datetime.now() operation_data.operation.start_date = datetime.now() operation_data.operation.completion_date = datetime.now() do_merge = False if importer_operation_id: do_merge = True operation_entity = dao.store_entity(operation_data.operation, merge=do_merge) dt_group = None op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: dt_group = dao.get_datatypegroup_by_op_group_id( op_group.id) if not dt_group: first_op = dao.get_operations_in_group( op_group.id, only_first_operation=True) dt_group = DataTypeGroup( op_group, operation_id=first_op.id, state=DEFAULTDATASTATE_INTERMEDIATE) dt_group = dao.store_entity(dt_group) # Store the DataTypes in db dts = {} all_dts_count += len(operation_data.dt_paths) for dt_path in operation_data.dt_paths: dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id) if isinstance(dt, BurstConfiguration): if op_group: dt.fk_operation_group = op_group.id all_stored_dts_count += self._store_or_link_burst_config( dt, dt_path, project.id) else: dts[dt_path] = dt if op_group: op_group.fill_operationgroup_name(dt.type) dao.store_entity(op_group) try: stored_dts_count = self._store_imported_datatypes_in_db( project, dts) all_stored_dts_count += stored_dts_count if operation_data.main_view_model.is_metric_operation: self._update_burst_metric(operation_entity) imported_operations.append(operation_entity) new_op_folder = self.storage_interface.get_project_folder( project.name, str(operation_entity.id)) view_model_disk_size = 0 for h5_file in operation_data.all_view_model_files: view_model_disk_size += StorageInterface.compute_size_on_disk( h5_file) shutil.move(h5_file, new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size dao.store_entity(operation_entity) except MissingReferenceException as excep: self.storage_interface.remove_operation_data( project.name, operation_entity.id) operation_entity.fk_operation_group = None dao.store_entity(operation_entity) dao.remove_entity(DataTypeGroup, dt_group.id) raise excep else: self.logger.warning( "Folder %s will be ignored, as we could not find a serialized " "operation or DTs inside!" % operation_data.operation_folder) # We want importer_operation_id to be kept just for the first operation (the first iteration) if is_group: importer_operation_id = None self._update_dt_groups(project.id) self._update_burst_configurations(project.id) return imported_operations, all_dts_count, all_stored_dts_count
def _capture_operation_results(self, result, user_tag=None): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ results_to_store = [] data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id # All entities will have the same subject and state subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT] state = self.meta_data[DataTypeMetaData.KEY_STATE] burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] perpetuated_identifier = None if DataTypeMetaData.KEY_TAG_1 in self.meta_data: perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1] for res in result: if res is None: continue res.subject = str(subject) res.state = state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data if not res.user_tag_1: res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier else: res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier res.fk_datatype_group = data_type_group_id ## Compute size-on disk, in case file-storage is used if hasattr(res, 'storage_path') and hasattr( res, 'get_storage_file_name'): associated_file = os.path.join(res.storage_path, res.get_storage_file_name()) res.close_file() res.disk_size = self.file_handler.compute_size_on_disk( associated_file) res = dao.store_entity(res) # Write metaData res.persist_full_metadata() results_to_store.append(res) del result[0:len(result)] result.extend(results_to_store) if len(result) and self._is_group_launch(): ## Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(result[0].type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', len(results_to_store)
def load_datatype_from_file(self, current_file, op_id, datatype_group=None, current_project_id=None): # type: (str, int, DataTypeGroup, int) -> HasTraitsIndex """ Creates an instance of datatype from storage / H5 file :returns: DatatypeIndex """ self.logger.debug("Loading DataType from file: %s" % current_file) h5_class = H5File.h5_class_from_file(current_file) if h5_class is BurstConfigurationH5: if current_project_id is None: op_entity = dao.get_operationgroup_by_id(op_id) current_project_id = op_entity.fk_launched_in h5_file = BurstConfigurationH5(current_file) burst = BurstConfiguration(current_project_id) burst.fk_simulation = op_id h5_file.load_into(burst) result = burst else: datatype, generic_attributes = h5.load_with_links(current_file) already_existing_datatype = h5.load_entity_by_gid(datatype.gid) if datatype_group is not None and already_existing_datatype is not None: raise DatatypeGroupImportException( "The datatype group that you are trying to import" " already exists!") index_class = h5.REGISTRY.get_index_for_datatype( datatype.__class__) datatype_index = index_class() datatype_index.fill_from_has_traits(datatype) datatype_index.fill_from_generic_attributes(generic_attributes) if datatype_group is not None and hasattr(datatype_index, 'fk_source_gid') and \ datatype_index.fk_source_gid is not None: ts = h5.load_entity_by_gid(datatype_index.fk_source_gid) if ts is None: op = dao.get_operations_in_group( datatype_group.fk_operation_group, only_first_operation=True) op.fk_operation_group = None dao.store_entity(op) dao.remove_entity(OperationGroup, datatype_group.fk_operation_group) dao.remove_entity(DataTypeGroup, datatype_group.id) raise DatatypeGroupImportException( "Please import the time series group before importing the" " datatype measure group!") # Add all the required attributes if datatype_group: datatype_index.fk_datatype_group = datatype_group.id if len(datatype_group.subject) == 0: datatype_group.subject = datatype_index.subject dao.store_entity(datatype_group) datatype_index.fk_from_operation = op_id associated_file = h5.path_for_stored_index(datatype_index) if os.path.exists(associated_file): datatype_index.disk_size = StorageInterface.compute_size_on_disk( associated_file) result = datatype_index return result
def import_project_operations(self, project, import_path, is_group=False, importer_operation_id=None): """ This method scans provided folder and identify all operations that needs to be imported """ all_dts_count = 0 all_stored_dts_count = 0 imported_operations = [] ordered_operations = self._retrieve_operations_in_order( project, import_path, importer_operation_id) for operation_data in ordered_operations: if operation_data.is_old_form: operation_entity, datatype_group = self.import_operation( operation_data.operation) new_op_folder = self.files_helper.get_project_folder( project, str(operation_entity.id)) try: operation_datatypes = self._load_datatypes_from_operation_folder( operation_data.operation_folder, operation_entity, datatype_group) # Create and store view_model from operation self.create_view_model(operation_entity, operation_data, new_op_folder) self._store_imported_datatypes_in_db( project, operation_datatypes) imported_operations.append(operation_entity) except MissingReferenceException: operation_entity.status = STATUS_ERROR dao.store_entity(operation_entity) elif operation_data.main_view_model is not None: do_merge = False if importer_operation_id: do_merge = True operation_entity = dao.store_entity(operation_data.operation, merge=do_merge) dt_group = None op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: dt_group = dao.get_datatypegroup_by_op_group_id( op_group.id) if not dt_group: first_op = dao.get_operations_in_group( op_group.id, only_first_operation=True) dt_group = DataTypeGroup( op_group, operation_id=first_op.id, state=DEFAULTDATASTATE_INTERMEDIATE) dt_group = dao.store_entity(dt_group) # Store the DataTypes in db dts = {} all_dts_count += len(operation_data.dt_paths) for dt_path in operation_data.dt_paths: dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id) if isinstance(dt, BurstConfiguration): if op_group: dt.fk_operation_group = op_group.id all_stored_dts_count += self._store_or_link_burst_config( dt, dt_path, project.id) else: dts[dt_path] = dt if op_group: op_group.fill_operationgroup_name(dt.type) dao.store_entity(op_group) try: stored_dts_count = self._store_imported_datatypes_in_db( project, dts) all_stored_dts_count += stored_dts_count if operation_data.main_view_model.is_metric_operation: self._update_burst_metric(operation_entity) #TODO: TVB-2849 to reveiw these flags and simplify condition if stored_dts_count > 0 or ( not operation_data.is_self_generated and not is_group) or importer_operation_id is not None: imported_operations.append(operation_entity) new_op_folder = self.files_helper.get_project_folder( project, str(operation_entity.id)) view_model_disk_size = 0 for h5_file in operation_data.all_view_model_files: view_model_disk_size += FilesHelper.compute_size_on_disk( h5_file) shutil.move(h5_file, new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size dao.store_entity(operation_entity) else: # In case all Dts under the current operation were Links and the ViewModel is dummy, # don't keep the Operation empty in DB dao.remove_entity(Operation, operation_entity.id) self.files_helper.remove_operation_data( project.name, operation_entity.id) except MissingReferenceException as excep: dao.remove_entity(Operation, operation_entity.id) self.files_helper.remove_operation_data( project.name, operation_entity.id) raise excep else: self.logger.warning( "Folder %s will be ignored, as we could not find a serialized " "operation or DTs inside!" % operation_data.operation_folder) self._update_dt_groups(project.id) self._update_burst_configurations(project.id) return imported_operations, all_dts_count, all_stored_dts_count