def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" adapter_instance = TestFactory.create_adapter('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 1, "Expected one operation group" assert all_operations[0][2] == 2, "Expected 2 operations in group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group("tvb.tests.framework.adapters.testadapter3", "TestAdapter3") group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: "param_5", "param_5": [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def load_burst(self, burst_id): """ :param burst_id: the id of the burst that should be loaded Having this input the method should: - load the entity from the DB - get all the workflow steps for the saved burst id - go trough the visualization workflow steps to create the tab configuration of the burst using the tab_index and index_in_tab fields saved on each workflow_step """ burst = dao.get_burst_by_id(burst_id) burst.prepare_after_load() burst.reset_tabs() burst_workflows = dao.get_workflows_for_burst(burst.id) group_gid = None if len(burst_workflows) == 1: # A simple burst with no range parameters burst = self.__populate_tabs_from_workflow(burst, burst_workflows[0]) elif len(burst_workflows) > 1: # A burst workflow with a range of values, created multiple workflows and need # to launch parameter space exploration with the resulted group self.__populate_tabs_from_workflow(burst, burst_workflows[0]) executed_steps = dao.get_workflow_steps(burst_workflows[0].id) operation = dao.get_operation_by_id(executed_steps[0].fk_operation) if operation.operation_group: workflow_group = dao.get_datatypegroup_by_op_group_id( operation.operation_group.id) group_gid = workflow_group.gid return burst, group_gid
def load_burst(self, burst_id): """ :param burst_id: the id of the burst that should be loaded Having this input the method should: - load the entity from the DB - get all the workflow steps for the saved burst id - go trough the visualization workflow steps to create the tab configuration of the burst using the tab_index and index_in_tab fields saved on each workflow_step """ burst = dao.get_burst_by_id(burst_id) burst.prepare_after_load() burst.reset_tabs() burst_workflows = dao.get_workflows_for_burst(burst.id) group_gid = None if len(burst_workflows) == 1: # A simple burst with no range parameters burst = self.__populate_tabs_from_workflow(burst, burst_workflows[0]) elif len(burst_workflows) > 1: # A burst workflow with a range of values, created multiple workflows and need # to launch parameter space exploration with the resulted group self.__populate_tabs_from_workflow(burst, burst_workflows[0]) executed_steps = dao.get_workflow_steps(burst_workflows[0].id) operation = dao.get_operation_by_id(executed_steps[0].fk_operation) if operation.operation_group: workflow_group = dao.get_datatypegroup_by_op_group_id(operation.operation_group.id) group_gid = workflow_group.gid return burst, group_gid
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 0, "There should be no operation") algogroup = dao.find_group('tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') group, _ = flow_service.prepare_adapter(self.test_project.id, algogroup) adapter_instance = flow_service.build_adapter_instance(group) data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) self.assertEqual(len(all_operations), 1, "Expected one operation group") self.assertEqual(all_operations[0][2], 2, "Expected 2 operations in group") operation_group_id = all_operations[0][3] self.assertNotEquals(operation_group_id, None, "The operation should be part of a group.") self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) self.assertTrue(len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes))) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) self.assertEqual(dt.fk_datatype_group, datatype_group.id, "DataTypeGroup is incorrect")
def import_operation(operation_entity, migration=False): """ Store a Operation entity. """ do_merge = False if operation_entity.id: do_merge = True operation_entity = dao.store_entity(operation_entity, merge=do_merge) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) if datatype_group is None and migration is False: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id( operation_group_id) datatype_group = DataTypeGroup( operation_group, operation_id=operation_entity.id) datatype_group.state = UploadAlgorithmCategoryConfig.defaultdatastate datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def prepare_indexes_for_simulation_results(self, operation, result_filenames, burst): indexes = list() self.logger.debug("Preparing indexes for simulation results in operation {}...".format(operation.id)) for filename in result_filenames: try: self.logger.debug("Preparing index for filename: {}".format(filename)) index = h5.index_for_h5_file(filename)() h5_class = h5.REGISTRY.get_h5file_for_index(type(index)) with h5_class(filename) as index_h5: index.fill_from_h5(index_h5) index.fill_from_generic_attributes(index_h5.load_generic_attributes()) index.fk_parent_burst = burst.gid index.fk_from_operation = operation.id if operation.fk_operation_group: datatype_group = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group) self.logger.debug( "Found DatatypeGroup with id {} for operation {}".format(datatype_group.id, operation.id)) index.fk_datatype_group = datatype_group.id # Update the operation group name operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group) operation_group.fill_operationgroup_name("TimeSeriesRegionIndex") dao.store_entity(operation_group) self.logger.debug( "Prepared index {} for file {} in operation {}".format(index.summary_info, filename, operation.id)) indexes.append(index) except Exception as e: self.logger.debug("Skip preparing index {} because there was an error.".format(filename)) self.logger.error(e) self.logger.debug("Prepared {} indexes for results in operation {}...".format(len(indexes), operation.id)) return indexes
def test_datatypes_groups(self, test_adapter_factory): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ # TODO: re-write this to use groups correctly all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" algo = test_adapter_factory(TestAdapter3) adapter_instance = ABCAdapter.build_adapter(algo) data = {model_burst.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations FlowService().fire_operation(adapter_instance, self.test_user, self.test_project.id) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 1, "Expected one operation group" assert all_operations[0][2] == 2, "Expected 2 operations in group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group(operation_group_id=operation_group_id) assert len(resulted_datatypes) >= 2, "Expected at least 2, but: " + str(len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def prepare_indexes_for_simulation_results(self, operation, result_filenames, burst): indexes = list() self.logger.debug( "Preparing indexes for simulation results in operation {}...". format(operation.id)) for filename in result_filenames: index = h5.index_for_h5_file(filename)() # TODO: don't load full TS in memory and make this read nicer datatype, ga = h5.load_with_references(filename) index.fill_from_has_traits(datatype) index.fill_from_generic_attributes(ga) index.fk_parent_burst = burst.gid index.fk_from_operation = operation.id if operation.fk_operation_group: datatype_group = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group) self.logger.debug( "Found DatatypeGroup with id {} for operation {}".format( datatype_group.id, operation.id)) index.fk_datatype_group = datatype_group.id self.logger.debug( "Prepared index {} for file {} in operation {}".format( index.summary_info, filename, operation.id)) indexes.append(index) self.logger.debug( "Prepared {} indexes for results in operation {}...".format( len(indexes), operation.id)) return indexes
def test_get_visualizers_for_group(self): _, op_group_id = TestFactory.create_group(self.test_user, self.test_project) dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id) result = self.flow_service.get_visualizers_for_group(dt_group.gid) # Only the discreet is expected self.assertEqual(1, len(result)) self.assertEqual(DISCRETE_PSE_ADAPTER_CLASS, result[0].classname)
def test_get_visualizers_for_group(self): _, op_group_id = TestFactory.create_group(self.test_user, self.test_project) dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id) result = self.flow_service.get_visualizers_for_group(dt_group.gid) # Only the discreet is expected assert 1 == len(result) assert IntrospectionRegistry.DISCRETE_PSE_ADAPTER_CLASS == result[0].classname
def test_get_visualizers_for_group(self): _, op_group_id = TestFactory.create_group(self.test_user, self.test_project) dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id) result = self.flow_service.get_visualizers_for_group(dt_group.gid) # Only the discreet is expected self.assertEqual(1, len(result)) self.assertEqual(DISCRETE_PSE_ADAPTER_CLASS, result[0].classname)
def test_get_visualizers_for_group(self, datatype_group_factory): group = datatype_group_factory() dt_group = dao.get_datatypegroup_by_op_group_id( group.fk_from_operation) result = self.flow_service.get_visualizers_for_group(dt_group.gid) # Only the discreet is expected assert 1 == len(result) assert IntrospectionRegistry.DISCRETE_PSE_ADAPTER_CLASS == result[ 0].classname
def test_get_visualizers_for_group(self, datatype_group_factory): group = datatype_group_factory() dt_group = dao.get_datatypegroup_by_op_group_id(group.fk_from_operation) result = self.flow_service.get_visualizers_for_group(dt_group.gid) # Both discrete and isocline are expected due to the 2 ranges set in the factory assert 2 == len(result) result_classnames = [res.classname for res in result] assert IntrospectionRegistry.ISOCLINE_PSE_ADAPTER_CLASS in result_classnames assert IntrospectionRegistry.DISCRETE_PSE_ADAPTER_CLASS in result_classnames
def _capture_operation_results(self, result, user_tag=None): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ results_to_store = [] data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string(datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group).id # All entities will have the same subject and state subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT] state = self.meta_data[DataTypeMetaData.KEY_STATE] burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] perpetuated_identifier = None if DataTypeMetaData.KEY_TAG_1 in self.meta_data: perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1] for res in result: if res is None: continue res.subject = str(subject) res.state = state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data if not res.user_tag_1: res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier else: res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier res.fk_datatype_group = data_type_group_id ## Compute size-on disk, in case file-storage is used if hasattr(res, 'storage_path') and hasattr(res, 'get_storage_file_name'): associated_file = os.path.join(res.storage_path, res.get_storage_file_name()) res.close_file() res.disk_size = self.file_handler.compute_size_on_disk(associated_file) res = dao.store_entity(res) # Write metaData res.persist_full_metadata() results_to_store.append(res) del result[0:len(result)] result.extend(results_to_store) if len(result) and self._is_group_launch(): ## Update the operation group name operation_group = dao.get_operationgroup_by_id(operation.fk_operation_group) operation_group.fill_operationgroup_name(result[0].type) dao.store_entity(operation_group) return 'Operation ' + str(self.operation_id) + ' has finished.', len(results_to_store)
def _remove_project_node_files(self, project_id, gid, links, skip_validation=False): """ Delegate removal of a node in the structure of the project. In case of a problem will THROW StructureException. """ try: project = self.find_project(project_id) datatype = dao.get_datatype_by_gid(gid) if links: op = dao.get_operation_by_id(datatype.fk_from_operation) # Instead of deleting, we copy the datatype to the linked project # We also clone the operation new_operation = self.__copy_linked_datatype_before_delete(op, datatype, project, links[0].fk_to_project) # If there is a datatype group and operation group and they were not moved yet to the linked project, # then do it if datatype.fk_datatype_group is not None: dt_group_op = dao.get_operation_by_id(datatype.fk_from_operation) op_group = dao.get_operationgroup_by_id(dt_group_op.fk_operation_group) op_group.fk_launched_in = links[0].fk_to_project dao.store_entity(op_group) burst = dao.get_burst_for_operation_id(op.id) if burst is not None: burst.fk_project = links[0].fk_to_project dao.store_entity(burst) dt_group = dao.get_datatypegroup_by_op_group_id(op_group.id) dt_group.parent_operation = new_operation dt_group.fk_from_operation = new_operation.id dao.store_entity(dt_group) else: # There is no link for this datatype so it has to be deleted specific_remover = get_remover(datatype.type)(datatype) specific_remover.remove_datatype(skip_validation) # Remove burst if dt has one and it still exists if datatype.fk_parent_burst is not None and datatype.is_ts: burst = dao.get_burst_for_operation_id(datatype.fk_from_operation) if burst is not None: dao.remove_entity(BurstConfiguration, burst.id) except RemoveDataTypeException: self.logger.exception("Could not execute operation Node Remove!") raise except FileStructureException: self.logger.exception("Remove operation failed") raise StructureException("Remove operation failed for unknown reasons.Please contact system administrator.")
def _create_datatype_group(self): """ Creates a project, one DataTypeGroup with 2 DataTypes into the new group. """ test_project = TestFactory.create_project(self.test_user, "NewProject") all_operations = dao.get_filtered_operations(test_project.id, None, is_count=True) self.assertEqual(0, all_operations, "There should be no operation.") datatypes, op_group_id = TestFactory.create_group(self.test_user, test_project) dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id) return test_project, dt_group.id, datatypes[0], datatypes[1]
def _create_datatype_group(self): """ Creates a project, one DataTypeGroup with 2 DataTypes into the new group. """ test_project = TestFactory.create_project(self.test_user, "NewProject") all_operations = dao.get_filtered_operations(test_project.id, None, is_count=True) assert 0 == all_operations, "There should be no operation." datatypes, op_group_id = TestFactory.create_group(self.test_user, test_project) dt_group = dao.get_datatypegroup_by_op_group_id(op_group_id) return test_project, dt_group.id, datatypes[0], datatypes[1]
def _capture_operation_results(self, result): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] count_stored = 0 group_type = None # In case of a group, the first not-none type is sufficient to memorize here for res in result: if res is None: continue res.subject = self.generic_attributes.subject res.state = self.generic_attributes.state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data res.user_tag_1 = self.generic_attributes.user_tag_1 res.user_tag_2 = self.generic_attributes.user_tag_2 res.fk_datatype_group = data_type_group_id # Compute size-on disk, in case file-storage is used associated_file = h5.path_for_stored_index(res) if os.path.exists(associated_file): res.disk_size = self.file_handler.compute_size_on_disk( associated_file) with H5File.from_file(associated_file) as f: f.store_generic_attributes(self.generic_attributes) dao.store_entity(res) group_type = res.type count_stored += 1 if count_stored > 0 and self._is_group_launch(): # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(group_type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', count_stored
def _capture_operation_results(self, result): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id count_stored = 0 if result is None: return "", count_stored group_type = None # In case of a group, the first not-none type is sufficient to memorize here for res in result: if res is None: continue if not res.fixed_generic_attributes: res.fill_from_generic_attributes(self.generic_attributes) res.fk_from_operation = self.operation_id res.fk_datatype_group = data_type_group_id associated_file = h5.path_for_stored_index(res) if os.path.exists(associated_file): if not res.fixed_generic_attributes: with H5File.from_file(associated_file) as f: f.store_generic_attributes(self.generic_attributes) # Compute size-on disk, in case file-storage is used res.disk_size = self.storage_interface.compute_size_on_disk( associated_file) dao.store_entity(res) res.after_store() group_type = res.type count_stored += 1 if count_stored > 0 and self._is_group_launch(): # Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(group_type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', count_stored
def prepare_index_for_metric_result(self, operation, result_filename, burst): self.logger.debug("Preparing index for metric result in operation {}...".format(operation.id)) index = h5.index_for_h5_file(result_filename)() with DatatypeMeasureH5(result_filename) as dti_h5: index.gid = dti_h5.gid.load().hex index.metrics = json.dumps(dti_h5.metrics.load()) index.fk_source_gid = dti_h5.analyzed_datatype.load().hex index.fk_from_operation = operation.id index.fk_parent_burst = burst.gid datatype_group = dao.get_datatypegroup_by_op_group_id(operation.fk_operation_group) self.logger.debug("Found DatatypeGroup with id {} for operation {}".format(datatype_group.id, operation.id)) index.fk_datatype_group = datatype_group.id self.logger.debug("Prepared index {} for results in operation {}...".format(index.summary_info, operation.id)) return index
def test_datatypes_groups(self, test_adapter_factory, datatype_group_factory): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" dt_group = datatype_group_factory(project=self.test_project) model = TestModel() test_adapter_factory() adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter1", "TestAdapter1") operations = dao.get_operations_in_group(dt_group.id) for op in operations: model.gid = uuid.uuid4() op_path = StorageInterface().get_project_folder( self.test_project.name, str(op.id)) op.view_model_gid = model.gid.hex op.algorithm = adapter.stored_adapter h5.store_view_model(model, op_path) dao.store_entity(op) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 2, "Expected two operation groups" assert all_operations[0][2] == 6, "Expected 6 operations in one group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[1][0]) self.operation_service.stop_operation(all_operations[1][1]) # Make sure operations are executed self.operation_service.launch_operation(all_operations[1][0], False) self.operation_service.launch_operation(all_operations[1][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) assert len( resulted_datatypes) >= 2, "Expected at least 2, but: " + str( len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def __import_operation(operation_entity): """ Store a Operation entity. """ operation_entity = dao.store_entity(operation_entity) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: try: datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) except SQLAlchemyError: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id(operation_group_id) datatype_group = model.DataTypeGroup(operation_group, operation_id=operation_entity.id) datatype_group.state = ADAPTERS['Upload']['defaultdatastate'] datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def _create_operations_with_inputs(self, datatype_group, is_group_parent=False): """ Method used for creating a complex tree of operations. If 'if_group_parent' is True then a new group will be created and one of its entries it will be used as input for the returned operations. """ group_dts = dao.get_datatypes_from_datatype_group(datatype_group.id) if is_group_parent: datatype_gid = group_dts[0].gid else: datatype_gid = self._create_value_wrapper(self.test_user, self.test_project)[1] parameters = json.dumps({"param_name": datatype_gid}) ops = [] for i in range(4): ops.append( TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)) if i in [1, 3]: ops[i].visible = False ops[i].parameters = parameters ops[i] = dao.store_entity(ops[i]) # groups ops_group = dao.get_operations_in_group( datatype_group.fk_from_operation) assert 9 == len(ops_group) ops_group[0].parameters = parameters ops_group[0] = dao.store_entity(ops_group[0]) ops_group[1].visible = False ops_group[1].parameters = parameters ops_group[1] = dao.store_entity(ops_group[1]) ops.extend(ops_group) if is_group_parent: dt_group = dao.get_datatypegroup_by_op_group_id(datatype_group.id) return ops, dt_group.id return ops, datatype_gid
def __import_operation(operation_entity): """ Store a Operation entity. """ operation_entity = dao.store_entity(operation_entity) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: try: datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) except SQLAlchemyError: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id(operation_group_id) datatype_group = model.DataTypeGroup(operation_group, operation_id=operation_entity.id) datatype_group.state = ADAPTERS['Upload']['defaultdatastate'] datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def __import_operation(operation_entity): """ Store a Operation entity. """ operation_entity = dao.store_entity(operation_entity) operation_group_id = operation_entity.fk_operation_group datatype_group = None if operation_group_id is not None: try: datatype_group = dao.get_datatypegroup_by_op_group_id(operation_group_id) except SQLAlchemyError: # If no dataType group present for current op. group, create it. operation_group = dao.get_operationgroup_by_id(operation_group_id) datatype_group = DataTypeGroup(operation_group, operation_id=operation_entity.id) datatype_group.state = UploadAlgorithmCategoryConfig.defaultdatastate datatype_group = dao.store_entity(datatype_group) return operation_entity, datatype_group
def prepare_datatypes_for_export(data): """ Method used for exporting data type groups. This method returns a list of all datatype indexes needed to be exported and a dictionary where keys are operation folder names and the values are lists containing the paths that belong to one particular operation folder. """ all_datatypes = ProjectService.get_all_datatypes_from_data(data) first_datatype = all_datatypes[0] # We are exporting a group of datatype measures so we need to find the group of time series if hasattr(first_datatype, 'fk_source_gid'): ts = h5.load_entity_by_gid(first_datatype.fk_source_gid) dt_metric_group = dao.get_datatypegroup_by_op_group_id( ts.parent_operation.fk_operation_group) datatype_measure_list = ProjectService.get_all_datatypes_from_data( dt_metric_group) all_datatypes = datatype_measure_list + all_datatypes else: ts_group = dao.get_datatype_measure_group_from_ts_from_pse( first_datatype.gid, DatatypeMeasureIndex) time_series_list = ProjectService.get_all_datatypes_from_data( ts_group) all_datatypes = all_datatypes + time_series_list if all_datatypes is None or len(all_datatypes) == 0: raise ExportException( "Could not export a data type group with no data!") op_file_dict = dict() for dt in all_datatypes: h5_path = h5.path_for_stored_index(dt) op_folder = os.path.dirname(h5_path) op_file_dict[op_folder] = [h5_path] op = dao.get_operation_by_id(dt.fk_from_operation) vms = h5.gather_references_of_view_model(op.view_model_gid, os.path.dirname(h5_path), only_view_models=True) op_file_dict[op_folder].extend(vms[0]) return all_datatypes, op_file_dict
def test_datatypes_groups(self): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ flow_service = FlowService() all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" adapter_instance = TestFactory.create_adapter( 'tvb.tests.framework.adapters.testadapter3', 'TestAdapter3') data = {model.RANGE_PARAMETER_1: 'param_5', 'param_5': [1, 2]} ## Create Group of operations flow_service.fire_operation(adapter_instance, self.test_user, self.test_project.id, **data) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 1, "Expected one operation group" assert all_operations[0][2] == 2, "Expected 2 operations in group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[0][0]) self.operation_service.stop_operation(all_operations[0][1]) ## Make sure operations are executed self.operation_service.launch_operation(all_operations[0][0], False) self.operation_service.launch_operation(all_operations[0][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) assert len( resulted_datatypes) >= 2, "Expected at least 2, but: " + str( len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def _create_operations_with_inputs(self, is_group_parent=False): """ Method used for creating a complex tree of operations. If 'if_group_parent' is True then a new group will be created and one of its entries it will be used as input for the returned operations. """ group_dts, root_op_group_id = TestFactory.create_group(self.test_user, self.test_project) if is_group_parent: datatype_gid = group_dts[0].gid else: datatype_gid = ProjectServiceTest._create_value_wrapper(self.test_user, self.test_project)[1] parameters = json.dumps({"param_name": datatype_gid}) ops = [] for i in range(4): ops.append(TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)) if i in [1, 3]: ops[i].visible = False ops[i].parameters = parameters ops[i] = dao.store_entity(ops[i]) #groups _, ops_group = TestFactory.create_group(self.test_user, self.test_project) ops_group = dao.get_operations_in_group(ops_group) self.assertEqual(2, len(ops_group)) ops_group[0].parameters = parameters ops_group[0] = dao.store_entity(ops_group[0]) ops_group[1].visible = False ops_group[1].parameters = parameters ops_group[1] = dao.store_entity(ops_group[1]) ops.extend(ops_group) if is_group_parent: dt_group = dao.get_datatypegroup_by_op_group_id(root_op_group_id) return ops, dt_group.id return ops, datatype_gid
def get_datatypegroup_by_op_group_id(operation_group_id): """ Returns the DataTypeGroup with the specified id. """ return dao.get_datatypegroup_by_op_group_id(operation_group_id)
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations(project_id, applied_filters) pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE) if current_ops is None: return selected_project, 0, [], 0 operations = [] view_categ_id = dao.get_visualisers_categories()[0].id for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[13] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid ## Filter only viewers for current DataTypeGroup entity: result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid) except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) is str: result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) is str: result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) is str: result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [] for dt in datatype_results: dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid) if dt_loaded: result['results'].append(dt_loaded) else: self.logger.warning("Could not retrieve datatype %s" % str(dt)) operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: ## We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no
def import_list_of_operations(self, project, import_path, is_group=False, importer_operation_id=None): """ This method scans provided folder and identify all operations that needs to be imported """ all_dts_count = 0 all_stored_dts_count = 0 imported_operations = [] ordered_operations = self._retrieve_operations_in_order( project, import_path, None if is_group else importer_operation_id) if is_group and len(ordered_operations) > 0: first_op = dao.get_operation_by_id(importer_operation_id) vm_path = h5.determine_filepath(first_op.view_model_gid, os.path.dirname(import_path)) os.remove(vm_path) ordered_operations[0].operation.id = importer_operation_id for operation_data in ordered_operations: if operation_data.is_old_form: operation_entity, datatype_group = self.import_operation( operation_data.operation) new_op_folder = self.storage_interface.get_project_folder( project.name, str(operation_entity.id)) try: operation_datatypes = self._load_datatypes_from_operation_folder( operation_data.operation_folder, operation_entity, datatype_group) # Create and store view_model from operation self.create_view_model(operation_entity, operation_data, new_op_folder) self._store_imported_datatypes_in_db( project, operation_datatypes) imported_operations.append(operation_entity) except MissingReferenceException: operation_entity.status = STATUS_ERROR dao.store_entity(operation_entity) elif operation_data.main_view_model is not None: operation_data.operation.create_date = datetime.now() operation_data.operation.start_date = datetime.now() operation_data.operation.completion_date = datetime.now() do_merge = False if importer_operation_id: do_merge = True operation_entity = dao.store_entity(operation_data.operation, merge=do_merge) dt_group = None op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: dt_group = dao.get_datatypegroup_by_op_group_id( op_group.id) if not dt_group: first_op = dao.get_operations_in_group( op_group.id, only_first_operation=True) dt_group = DataTypeGroup( op_group, operation_id=first_op.id, state=DEFAULTDATASTATE_INTERMEDIATE) dt_group = dao.store_entity(dt_group) # Store the DataTypes in db dts = {} all_dts_count += len(operation_data.dt_paths) for dt_path in operation_data.dt_paths: dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id) if isinstance(dt, BurstConfiguration): if op_group: dt.fk_operation_group = op_group.id all_stored_dts_count += self._store_or_link_burst_config( dt, dt_path, project.id) else: dts[dt_path] = dt if op_group: op_group.fill_operationgroup_name(dt.type) dao.store_entity(op_group) try: stored_dts_count = self._store_imported_datatypes_in_db( project, dts) all_stored_dts_count += stored_dts_count if operation_data.main_view_model.is_metric_operation: self._update_burst_metric(operation_entity) imported_operations.append(operation_entity) new_op_folder = self.storage_interface.get_project_folder( project.name, str(operation_entity.id)) view_model_disk_size = 0 for h5_file in operation_data.all_view_model_files: view_model_disk_size += StorageInterface.compute_size_on_disk( h5_file) shutil.move(h5_file, new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size dao.store_entity(operation_entity) except MissingReferenceException as excep: self.storage_interface.remove_operation_data( project.name, operation_entity.id) operation_entity.fk_operation_group = None dao.store_entity(operation_entity) dao.remove_entity(DataTypeGroup, dt_group.id) raise excep else: self.logger.warning( "Folder %s will be ignored, as we could not find a serialized " "operation or DTs inside!" % operation_data.operation_folder) # We want importer_operation_id to be kept just for the first operation (the first iteration) if is_group: importer_operation_id = None self._update_dt_groups(project.id) self._update_burst_configurations(project.id) return imported_operations, all_dts_count, all_stored_dts_count
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations(project_id, applied_filters) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) if total_filtered >= start_idx + OPERATIONS_PAGE_SIZE: end_idx = OPERATIONS_PAGE_SIZE else: end_idx = total_filtered - start_idx pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, end_idx) started_ops = 0 if current_ops is None: return selected_project, [], 0 operations = [] for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[14] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) datatype = dao.get_datatype_by_id(datatype_group.id) result["datatype_group_gid"] = datatype.gid result["gid"] = operation_group.gid all_categs = dao.get_algorithm_categories() view_categ = dao.get_visualisers_categories()[0] excludes = [categ.id for categ in all_categs if categ.id != view_categ.id] algo = self.retrieve_launchers("DataTypeGroup", datatype.gid, exclude_categories=excludes).values()[0] view_groups = [] for algo in algo.values(): url = '/flow/' + str(algo['category']) + '/' + str(algo['id']) if algo['part_of_group']: url = '/flow/prepare_group_launch/' + datatype.gid + '/' + \ str(algo['category']) + '/' + str(algo['id']) view_groups.append(dict(name=algo["displayName"], url=url, param_name=algo['children'][0]['param_name'], part_of_group=algo['part_of_group'])) result["view_groups"] = view_groups except Exception, excep: self.logger.error(excep) self.logger.warning("Will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["method"] = one_op[5] result["user"] = dao.get_user_by_id(one_op[6]) if type(one_op[7]) in (str, unicode): result["create"] = string2date(str(one_op[7])) else: result["create"] = one_op[7] if type(one_op[8]) in (str, unicode): result["start"] = string2date(str(one_op[8])) else: result["start"] = one_op[8] if type(one_op[9]) in (str, unicode): result["complete"] = string2date(str(one_op[9])) else: result["complete"] = one_op[9] if result["complete"] is not None and result["start"] is not None: result["duration"] = timedelta2string(result["complete"] - result["start"]) result["status"] = one_op[10] if result["status"] == model.STATUS_STARTED: started_ops += 1 result["additional"] = one_op[11] result["visible"] = True if one_op[12] > 0 else False result['operation_tag'] = one_op[13] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type, dt.gid, 'gid')[0] for dt in datatype_results] operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name, figure.operation.id) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result)
def remove_datatype(self, project_id, datatype_gid, skip_validation=False, existing_dt_links=None): """ Method used for removing a dataType. If the given dataType is a DatatypeGroup or a dataType from a DataTypeGroup than this method will remove the entire group. The operation(s) used for creating the dataType(s) will also be removed. """ datatype = dao.get_datatype_by_gid(datatype_gid) if datatype is None: self.logger.warning("Attempt to delete DT[%s] which no longer exists." % datatype_gid) return if datatype.parent_operation.fk_launched_in != int(project_id): self.logger.warning("Datatype with GUID [%s] has been moved to another project and does " "not need to be deleted anymore." % datatype_gid) return is_datatype_group = False datatype_group = None new_dt_links = [] # Datatype Groups were already handled when the first DatatypeMeasureIndex has been found if dao.is_datatype_group(datatype_gid): is_datatype_group = True datatype_group = datatype # Found the first DatatypeMeasureIndex from a group elif datatype.fk_datatype_group is not None: is_datatype_group = True # We load it this way to make sure we have the 'fk_operation_group' in every case datatype_group_gid = dao.get_datatype_by_id(datatype.fk_datatype_group).gid datatype_group = h5.load_entity_by_gid(datatype_group_gid) operations_set = [datatype.fk_from_operation] correct = True if is_datatype_group: operations_set = [datatype_group.fk_from_operation] self.logger.debug("Removing datatype group %s" % datatype_group) datatypes = self.get_all_datatypes_from_data(datatype_group) first_datatype = datatypes[0] if hasattr(first_datatype, 'fk_source_gid'): ts = h5.load_entity_by_gid(first_datatype.fk_source_gid) ts_group = dao.get_datatypegroup_by_op_group_id(ts.parent_operation.fk_operation_group) dm_group = datatype_group else: dt_measure_index = get_class_by_name("{}.{}".format(DATATYPE_MEASURE_INDEX_MODULE, DATATYPE_MEASURE_INDEX_CLASS)) dm_group = dao.get_datatype_measure_group_from_ts_from_pse(first_datatype.gid, dt_measure_index) ts_group = datatype_group links = [] if ts_group: links.extend(dao.get_links_for_datatype(ts_group.id)) correct = correct and self._remove_operation_group(ts_group.fk_operation_group, project_id, skip_validation, operations_set, links) if dm_group: links.extend(dao.get_links_for_datatype(dm_group.id)) correct = correct and self._remove_operation_group(dm_group.fk_operation_group, project_id, skip_validation, operations_set, links) if len(links) > 0: # We want to get the links for the first TSIndex directly # This code works for all cases datatypes = dao.get_datatype_in_group(ts_group.id) ts = datatypes[0] new_dt_links = self._add_links_for_datatype_references(ts, links[0].fk_to_project, links[0].id, existing_dt_links) else: self.logger.debug("Removing datatype %s" % datatype) links = dao.get_links_for_datatype(datatype.id) if len(links) > 0: new_dt_links = self._add_links_for_datatype_references(datatype, links[0].fk_to_project, links[0].id, existing_dt_links) self._remove_project_node_files(project_id, datatype.gid, links, skip_validation) # Remove Operation entity in case no other DataType needs them. project = dao.get_project_by_id(project_id) for operation_id in operations_set: dependent_dt = dao.get_generic_entity(DataType, operation_id, "fk_from_operation") if len(dependent_dt) > 0: # Do not remove Operation in case DataType still exist referring it. continue correct = correct and dao.remove_entity(Operation, operation_id) # Make sure Operation folder is removed self.storage_interface.remove_operation_data(project.name, operation_id) self.storage_interface.push_folder_to_sync(project.name) if not correct: raise RemoveDataTypeException("Could not remove DataType " + str(datatype_gid)) return new_dt_links
def import_project_operations(self, project, import_path, is_group=False, importer_operation_id=None): """ This method scans provided folder and identify all operations that needs to be imported """ all_dts_count = 0 all_stored_dts_count = 0 imported_operations = [] ordered_operations = self._retrieve_operations_in_order( project, import_path, importer_operation_id) for operation_data in ordered_operations: if operation_data.is_old_form: operation_entity, datatype_group = self.import_operation( operation_data.operation) new_op_folder = self.files_helper.get_project_folder( project, str(operation_entity.id)) try: operation_datatypes = self._load_datatypes_from_operation_folder( operation_data.operation_folder, operation_entity, datatype_group) # Create and store view_model from operation self.create_view_model(operation_entity, operation_data, new_op_folder) self._store_imported_datatypes_in_db( project, operation_datatypes) imported_operations.append(operation_entity) except MissingReferenceException: operation_entity.status = STATUS_ERROR dao.store_entity(operation_entity) elif operation_data.main_view_model is not None: do_merge = False if importer_operation_id: do_merge = True operation_entity = dao.store_entity(operation_data.operation, merge=do_merge) dt_group = None op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: dt_group = dao.get_datatypegroup_by_op_group_id( op_group.id) if not dt_group: first_op = dao.get_operations_in_group( op_group.id, only_first_operation=True) dt_group = DataTypeGroup( op_group, operation_id=first_op.id, state=DEFAULTDATASTATE_INTERMEDIATE) dt_group = dao.store_entity(dt_group) # Store the DataTypes in db dts = {} all_dts_count += len(operation_data.dt_paths) for dt_path in operation_data.dt_paths: dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id) if isinstance(dt, BurstConfiguration): if op_group: dt.fk_operation_group = op_group.id all_stored_dts_count += self._store_or_link_burst_config( dt, dt_path, project.id) else: dts[dt_path] = dt if op_group: op_group.fill_operationgroup_name(dt.type) dao.store_entity(op_group) try: stored_dts_count = self._store_imported_datatypes_in_db( project, dts) all_stored_dts_count += stored_dts_count if operation_data.main_view_model.is_metric_operation: self._update_burst_metric(operation_entity) #TODO: TVB-2849 to reveiw these flags and simplify condition if stored_dts_count > 0 or ( not operation_data.is_self_generated and not is_group) or importer_operation_id is not None: imported_operations.append(operation_entity) new_op_folder = self.files_helper.get_project_folder( project, str(operation_entity.id)) view_model_disk_size = 0 for h5_file in operation_data.all_view_model_files: view_model_disk_size += FilesHelper.compute_size_on_disk( h5_file) shutil.move(h5_file, new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size dao.store_entity(operation_entity) else: # In case all Dts under the current operation were Links and the ViewModel is dummy, # don't keep the Operation empty in DB dao.remove_entity(Operation, operation_entity.id) self.files_helper.remove_operation_data( project.name, operation_entity.id) except MissingReferenceException as excep: dao.remove_entity(Operation, operation_entity.id) self.files_helper.remove_operation_data( project.name, operation_entity.id) raise excep else: self.logger.warning( "Folder %s will be ignored, as we could not find a serialized " "operation or DTs inside!" % operation_data.operation_folder) self._update_dt_groups(project.id) self._update_burst_configurations(project.id) return imported_operations, all_dts_count, all_stored_dts_count
def get_datatypegroup_by_op_group_id(operation_group_id): """ Returns the DataTypeGroup with the specified id. """ return dao.get_datatypegroup_by_op_group_id(operation_group_id)
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations(project_id, applied_filters) pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE) if current_ops is None: return selected_project, 0, [], 0 operations = [] view_categ_id = dao.get_visualisers_categories()[0].id for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[14] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid ## Filter only viewers for current DataTypeGroup entity: launcher = self.retrieve_launchers(datatype_group.gid, include_categories=[view_categ_id]).values()[0] view_groups = [] for launcher in launcher.values(): url = '/flow/' + str(launcher['category']) + '/' + str(launcher['id']) if launcher['part_of_group']: url = '/flow/prepare_group_launch/' + datatype_group.gid + '/' + \ str(launcher['category']) + '/' + str(launcher['id']) view_groups.append(dict(name=launcher["displayName"], url=url, param_name=launcher['children'][0]['param_name'], part_of_group=launcher['part_of_group'])) result["view_groups"] = view_groups except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["method"] = one_op[5] result["user"] = dao.get_user_by_id(one_op[6]) if type(one_op[7]) in (str, unicode): result["create"] = string2date(str(one_op[7])) else: result["create"] = one_op[7] if type(one_op[8]) in (str, unicode): result["start"] = string2date(str(one_op[8])) else: result["start"] = one_op[8] if type(one_op[9]) in (str, unicode): result["complete"] = string2date(str(one_op[9])) else: result["complete"] = one_op[9] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[10] result["additional"] = one_op[11] result["visible"] = True if one_op[12] > 0 else False result['operation_tag'] = one_op[13] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type, dt.gid, 'gid')[0] for dt in datatype_results] operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: ## We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no
def _capture_operation_results(self, result, user_tag=None): """ After an operation was finished, make sure the results are stored in DB storage and the correct meta-data,IDs are set. """ results_to_store = [] data_type_group_id = None operation = dao.get_operation_by_id(self.operation_id) if operation.user_group is None or len(operation.user_group) == 0: operation.user_group = date2string( datetime.now(), date_format=LESS_COMPLEX_TIME_FORMAT) operation = dao.store_entity(operation) if self._is_group_launch(): data_type_group_id = dao.get_datatypegroup_by_op_group_id( operation.fk_operation_group).id # All entities will have the same subject and state subject = self.meta_data[DataTypeMetaData.KEY_SUBJECT] state = self.meta_data[DataTypeMetaData.KEY_STATE] burst_reference = None if DataTypeMetaData.KEY_BURST in self.meta_data: burst_reference = self.meta_data[DataTypeMetaData.KEY_BURST] perpetuated_identifier = None if DataTypeMetaData.KEY_TAG_1 in self.meta_data: perpetuated_identifier = self.meta_data[DataTypeMetaData.KEY_TAG_1] for res in result: if res is None: continue res.subject = str(subject) res.state = state res.fk_parent_burst = burst_reference res.fk_from_operation = self.operation_id res.framework_metadata = self.meta_data if not res.user_tag_1: res.user_tag_1 = user_tag if user_tag is not None else perpetuated_identifier else: res.user_tag_2 = user_tag if user_tag is not None else perpetuated_identifier res.fk_datatype_group = data_type_group_id ## Compute size-on disk, in case file-storage is used if hasattr(res, 'storage_path') and hasattr( res, 'get_storage_file_name'): associated_file = os.path.join(res.storage_path, res.get_storage_file_name()) res.close_file() res.disk_size = self.file_handler.compute_size_on_disk( associated_file) res = dao.store_entity(res) # Write metaData res.persist_full_metadata() results_to_store.append(res) del result[0:len(result)] result.extend(results_to_store) if len(result) and self._is_group_launch(): ## Update the operation group name operation_group = dao.get_operationgroup_by_id( operation.fk_operation_group) operation_group.fill_operationgroup_name(result[0].type) dao.store_entity(operation_group) return 'Operation ' + str( self.operation_id) + ' has finished.', len(results_to_store)
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations( project_id, applied_filters) pages_no = total_filtered // OPERATIONS_PAGE_SIZE + ( 1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE) if current_ops is None: return selected_project, 0, [], 0 operations = [] for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[13] operation_group_id = one_op[3] if operation_group_id is not None and operation_group_id: try: operation_group = dao.get_generic_entity( OperationGroup, operation_group_id)[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) result[ "datatype_group_gid"] = datatype_group.gid if datatype_group is not None else None result["gid"] = operation_group.gid # Filter only viewers for current DataTypeGroup entity: if datatype_group is None: view_groups = None else: view_groups = AlgorithmService( ).get_visualizers_for_group(datatype_group.gid) result["view_groups"] = view_groups except Exception: self.logger.exception( "We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) is str: result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) is str: result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) is str: result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result[ "start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] if not result['group']: result['results'] = dao.get_results_for_operation( result['id']) else: result['results'] = None operations.append(result) except Exception: # We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception( "Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no