def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) ## Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) username = dao.get_user_by_id(operation.fk_launched_by).username burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = ProjectService._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, username, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) ## Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details
def _update_pse_burst_status(self, burst_config): operations_in_group = dao.get_operations_in_group( burst_config.fk_operation_group) if burst_config.fk_metric_operation_group: operations_in_group.extend( dao.get_operations_in_group( burst_config.fk_metric_operation_group)) operation_statuses = list() for operation in operations_in_group: if not has_finished(operation.status): self.logger.debug( 'Operation {} in group {} is not finished, burst status will not be updated' .format(operation.id, operation.fk_operation_group)) return operation_statuses.append(operation.status) self.logger.debug( 'All operations in burst {} have finished. Will update burst status' .format(burst_config.id)) if STATUS_ERROR in operation_statuses: self.mark_burst_finished( burst_config, BurstConfiguration.BURST_ERROR, 'Some operations in PSE have finished with errors') elif STATUS_CANCELED in operation_statuses: self.mark_burst_finished(burst_config, BurstConfiguration.BURST_CANCELED) else: self.mark_burst_finished(burst_config)
def get_operation_details(self, operation_gid, is_group): """ :returns: an entity OperationOverlayDetails filled with all information for current operation details. """ if is_group: operation_group = self.get_operation_group_by_gid(operation_gid) operation = dao.get_operations_in_group(operation_group.id, False, True) # Reload, to make sure all attributes lazy are populated as well. operation = dao.get_operation_by_gid(operation.gid) no_of_op_in_group = dao.get_operations_in_group(operation_group.id, is_count=True) datatype_group = self.get_datatypegroup_by_op_group_id(operation_group.id) count_result = dao.count_datatypes_in_group(datatype_group.id) else: operation = dao.get_operation_by_gid(operation_gid) if operation is None: return None no_of_op_in_group = 1 count_result = dao.count_resulted_datatypes(operation.id) user_display_name = dao.get_user_by_id(operation.fk_launched_by).display_name burst = dao.get_burst_for_operation_id(operation.id) datatypes_param, all_special_params = self._review_operation_inputs(operation.gid) op_pid = dao.get_operation_process_for_operation(operation.id) op_details = OperationOverlayDetails(operation, user_display_name, len(datatypes_param), count_result, burst, no_of_op_in_group, op_pid) # Add all parameter which are set differently by the user on this Operation. if all_special_params is not None: op_details.add_scientific_fields(all_special_params) return op_details
def do_operation_launch(operation_id): """ Event attached to the local queue for executing an operation, when we will have resources available. """ log = get_logger('tvb.core.operation_async_launcher') burst_service = BurstService2() try: log.debug("Loading operation with id=%s" % operation_id) curent_operation = dao.get_operation_by_id(operation_id) stored_adapter = curent_operation.algorithm log.debug("Importing Algorithm: " + str(stored_adapter.classname) + " for Operation:" + str(curent_operation.id)) params = parse_json_parameters(curent_operation.parameters) adapter_instance = ABCAdapter.build_adapter(stored_adapter) # These should go once we have a common place for it if not isinstance(adapter_instance, SimulatorAdapter): adapter_form = adapter_instance.get_form()( project_id=curent_operation.fk_launched_in) adapter_form.fill_from_post(params) adapter_instance.submit_form(adapter_form) # Un-comment bellow for profiling an operation: # import cherrypy.lib.profiler as profiler # p = profiler.Profiler("/Users/lia.domide/TVB/profiler/") # p.run(OperationService().initiate_prelaunch, curent_operation, adapter_instance, {}, **PARAMS) OperationService().initiate_prelaunch(curent_operation, adapter_instance, **params) if curent_operation.fk_operation_group: parent_burst = dao.get_generic_entity( BurstConfiguration2, curent_operation.fk_operation_group, 'operation_group_id')[0] operations_in_group = dao.get_operations_in_group( curent_operation.fk_operation_group) if parent_burst.metric_operation_group_id: operations_in_group.extend( dao.get_operations_in_group( parent_burst.metric_operation_group_id)) for operation in operations_in_group: if not has_finished(operation.status): break if parent_burst is not None: burst_service.mark_burst_finished(parent_burst) else: parent_burst = burst_service.get_burst_for_operation_id( operation_id) if parent_burst is not None: burst_service.mark_burst_finished(parent_burst) log.debug("Successfully finished operation " + str(operation_id)) except Exception as excep: log.error("Could not execute operation " + str(sys.argv[1])) log.exception(excep) parent_burst = burst_service.get_burst_for_operation_id(operation_id) if parent_burst is not None: burst_service.mark_burst_finished(parent_burst, error_message=str(excep))
def do_operation_launch(operation_id): """ Event attached to the local queue for executing an operation, when we will have resources available. """ log = get_logger('tvb.core.operation_async_launcher') burst_service = BurstService() try: log.debug("Loading operation with id=%s" % operation_id) curent_operation = dao.get_operation_by_id(operation_id) stored_adapter = curent_operation.algorithm log.debug("Importing Algorithm: " + str(stored_adapter.classname) + " for Operation:" + str(curent_operation.id)) adapter_instance = ABCAdapter.build_adapter(stored_adapter) # Un-comment bellow for profiling an operation: # import cherrypy.lib.profiler as profiler # p = profiler.Profiler("/Users/lia.domide/TVB/profiler/") # p.run(OperationService().initiate_prelaunch, curent_operation, adapter_instance, {}, **PARAMS) OperationService().initiate_prelaunch(curent_operation, adapter_instance) if curent_operation.fk_operation_group: parent_burst = dao.get_generic_entity( BurstConfiguration, curent_operation.fk_operation_group, 'fk_operation_group')[0] operations_in_group = dao.get_operations_in_group( curent_operation.fk_operation_group) if parent_burst.fk_metric_operation_group: operations_in_group.extend( dao.get_operations_in_group( parent_burst.fk_metric_operation_group)) burst_finished = True for operation in operations_in_group: if not has_finished(operation.status): burst_finished = False break if burst_finished and parent_burst is not None and parent_burst.status != BurstConfiguration.BURST_ERROR: burst_service.mark_burst_finished(parent_burst) else: parent_burst = burst_service.get_burst_for_operation_id( operation_id) if parent_burst is not None: burst_service.mark_burst_finished(parent_burst) log.debug("Successfully finished operation " + str(operation_id)) except Exception as excep: log.error("Could not execute operation " + str(operation_id)) log.exception(excep) parent_burst = burst_service.get_burst_for_operation_id(operation_id) if parent_burst is not None: burst_service.mark_burst_finished(parent_burst, error_message=str(excep))
def test_set_op_group_visibility(self): """ Tests if the visibility for an operation group is set correct. """ _, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") list_of_operations = dao.get_operations_in_group(group_id) for operation in list_of_operations: self.assertTrue(operation.visible, "The operation should be visible.") op_group = dao.get_operationgroup_by_id(group_id) self.project_service.set_operation_and_group_visibility(op_group.gid, False, True) operations = dao.get_operations_in_group(group_id) for operation in operations: self.assertFalse(operation.visible, "The operation should not be visible.")
def test_set_op_group_visibility(self): """ Tests if the visibility for an operation group is set correct. """ _, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") list_of_operations = dao.get_operations_in_group(group_id) for operation in list_of_operations: assert operation.visible, "The operation should be visible." op_group = dao.get_operationgroup_by_id(group_id) self.project_service.set_operation_and_group_visibility(op_group.gid, False, True) operations = dao.get_operations_in_group(group_id) for operation in operations: assert not operation.visible, "The operation should not be visible."
def test_set_op_and_group_visibility(self): """ When changing the visibility for an operation that belongs to an operation group, we should also change the visibility for the entire group of operations. """ _, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") list_of_operations = dao.get_operations_in_group(group_id) for operation in list_of_operations: assert operation.visible, "The operation should be visible." self.project_service.set_operation_and_group_visibility(list_of_operations[0].gid, False) operations = dao.get_operations_in_group(group_id) for operation in operations: assert not operation.visible, "The operation should not be visible."
def test_set_op_and_group_visibility(self): """ When changing the visibility for an operation that belongs to an operation group, we should also change the visibility for the entire group of operations. """ _, group_id = TestFactory.create_group(self.test_user, subject="test-subject-1") list_of_operations = dao.get_operations_in_group(group_id) for operation in list_of_operations: self.assertTrue(operation.visible, "The operation should be visible.") self.project_service.set_operation_and_group_visibility(list_of_operations[0].gid, False) operations = dao.get_operations_in_group(group_id) for operation in operations: self.assertFalse(operation.visible, "The operation should not be visible.")
def test_set_op_group_visibility(self, datatype_group_factory): """ Tests if the visibility for an operation group is set correct. """ group = datatype_group_factory() list_of_operations = dao.get_operations_in_group(group.id) for operation in list_of_operations: assert operation.visible, "The operation should be visible." op_group = dao.get_operationgroup_by_id(group.id) self.project_service.set_operation_and_group_visibility( op_group.gid, False, True) operations = dao.get_operations_in_group(group.id) for operation in operations: assert not operation.visible, "The operation should not be visible."
def test_set_op_and_group_visibility(self, datatype_group_factory): """ When changing the visibility for an operation that belongs to an operation group, we should also change the visibility for the entire group of operations. """ group = datatype_group_factory() list_of_operations = dao.get_operations_in_group(group.id) for operation in list_of_operations: assert operation.visible, "The operation should be visible." self.project_service.set_operation_and_group_visibility( list_of_operations[0].gid, False) operations = dao.get_operations_in_group(group.id) for operation in operations: assert not operation.visible, "The operation should not be visible."
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. :param datatype_group_gid: the group id for the `DataType` to be visualised :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :param size_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :returns: `ContextDiscretePSE` :raises Exception: when `datatype_group_id` is invalid (not in database) """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception("Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one.") operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) range1_name, range1_values, range1_labels = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range1) range2_name, range2_values, range2_labels = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page) pse_context.setRanges(range1_name, range1_values, range1_labels, range2_name, range2_values, range2_labels) final_dict = {} operations = dao.get_operations_in_group(operation_group.id) for operation_ in operations: if not operation_.has_finished: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = range_values[range1_name] key_2 = model.RANGE_MISSING_STRING if range2_name is not None: key_2 = range_values[range2_name] datatype = None if operation_.status == model.STATUS_FINISHED: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = {} final_dict[key_1][key_2] = pse_context.build_node_info(operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} return pse_context
def test_remove_burst_operation_group(self, simulation_launch): first_op = simulation_launch(self.test_user, self.test_project, 1000, True) operations_group_id = first_op.fk_operation_group assert not first_op.has_finished self.flow_c.cancel_or_remove_operation(operations_group_id, 1, True) operations = dao.get_operations_in_group(operations_group_id) for operation in operations: operation = dao.try_get_operation_by_id(operation.id) assert operation is None
def build_structure_for_operation_group(operation_group_gid): group = dao.get_operationgroup_by_gid(operation_group_gid) operation = dao.get_operations_in_group(group.id, only_first_operation=True) algo = dao.get_algorithm_by_id(operation.fk_from_algo) structure = NodeStructure(operation_group_gid, algo.name) structure.data = NodeData.build_node_for_operation(operation, group.id) return structure
def build_structure_for_operation_group(operation_group_gid): group = dao.get_operationgroup_by_gid(operation_group_gid) operation = dao.get_operations_in_group(group.id, only_first_operation=True) algo = dao.get_algorithm_by_id(operation.fk_from_algo) structure = NodeStructure(operation_group_gid, algo.displayname) structure.data = NodeData.build_node_for_operation(operation, group.id) return structure
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) _, range1_name, range1_labels = operation_group.load_range_numbers( operation_group.range1) has_range2, range2_name, range2_labels = operation_group.load_range_numbers( operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, range1_labels, range2_labels, color_metric, size_metric, back_page) final_dict = dict() operations = dao.get_operations_in_group(operation_group.id) for operation_ in operations: if operation_.status == model.STATUS_STARTED: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = range_values[range1_name] key_2 = model.RANGE_MISSING_STRING if has_range2 is not None: key_2 = range_values[range2_name] datatype = None if operation_.status == model.STATUS_FINISHED: datatype = dao.get_results_for_operation(operation_.id)[0] measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = { key_2: pse_context.build_node_info(operation_, datatype) } else: final_dict[key_1][key_2] = pse_context.build_node_info( operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} return pse_context
def test_stop_burst_operation_group(self, simulation_launch): first_op = simulation_launch(self.test_user, self.test_project, 1000, True) operations_group_id = first_op.fk_operation_group assert not first_op.has_finished sleep(5) self.flow_c.cancel_or_remove_operation(operations_group_id, 1, False) operations = dao.get_operations_in_group(operations_group_id) for operation in operations: operation = dao.get_operation_by_id(operation.id) assert operation.status == STATUS_CANCELED
def launch(self, datatype_group, **kwargs): """ Also overwrite launch from ABCDisplayer, since we want to handle a list of figures, instead of only one Matplotlib figure. :raises LaunchException: when called before all operations in the group are finished """ if self.PARAM_FIGURE_SIZE in kwargs: figsize = kwargs[self.PARAM_FIGURE_SIZE] figsize = ((figsize[0]) / 80, (figsize[1]) / 80) del kwargs[self.PARAM_FIGURE_SIZE] else: figsize = (15, 7) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) _, range1_name, self.range1 = operation_group.load_range_numbers( operation_group.range1) _, range2_name, self.range2 = operation_group.load_range_numbers( operation_group.range2) # Get the computed measures on this DataTypeGroup first_op = dao.get_operations_in_group(operation_group.id)[0] if first_op.status != model.STATUS_FINISHED: raise LaunchException( "Not all operations from this range are finished. Cannot generate data until then." ) datatype = dao.get_results_for_operation(first_op.id)[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.id)[0] else: dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') if dt_measure: dt_measure = dt_measure[0] figure_nrs = {} metrics = dt_measure.metrics if dt_measure else [] for metric in metrics: # Separate plot for each metric. self._create_plot(metric, figsize, operation_group, range1_name, range2_name, figure_nrs) parameters = dict(title=self._ui_name, showFullToolbar=True, serverIp=config.SERVER_IP, serverPort=config.MPLH5_SERVER_PORT, figureNumbers=figure_nrs, metrics=metrics, figuresJSON=json.dumps(figure_nrs)) return self.build_display_result("pse_isocline/view", parameters)
def __init__(self, operation_group_gid): NodeStructure.__init__(self, operation_group_gid, "") group = dao.get_operationgroup_by_gid(operation_group_gid) operation = dao.get_operations_in_group(group.id, only_first_operation=True) algo = dao.get_algorithm_by_id(operation.fk_from_algo) node_data = NodeData(MAX_SHAPE_SIZE, OPERATION_GROUP_SHAPE_COLOR, OPERATION_GROUP_SHAPE, NODE_OPERATION_GROUP_TYPE, group.id, str(operation.start_date)) self.name = algo.name self.data = node_data
def from_db(cls, operation_group_id): """ Collects from db the information about the operation group that is required by the isocline view. """ operations = dao.get_operations_in_group(operation_group_id) operation_group = dao.get_operationgroup_by_id(operation_group_id) self = cls(operation_group.range1, operation_group.range2, {}, PseIsoModel._find_metrics(operations), None) self._fill_apriori_data(operations) return self
def test_stop_operations_group(self, test_adapter_factory, datatype_group_factory): group = datatype_group_factory(status=STATUS_STARTED, store_vm=True) operations = dao.get_operations_in_group(group.fk_from_operation) operation_group_id = 0 for operation in operations: operation = dao.get_operation_by_id(operation.id) assert not operation.has_finished operation_group_id = operation.fk_operation_group self.flow_c.cancel_or_remove_operation(operation_group_id, 1, False) for operation in operations: operation = dao.get_operation_by_id(operation.id) assert operation.status == STATUS_CANCELED
def get_datatypes_inputs_for_operation_group(self, group_id, selected_filter): """ Returns the dataType inputs for an operation group. If more dataTypes are part of the same dataType group then only the dataType group will be returned instead of them. """ operations_gids = dao.get_operations_in_group(group_id, only_gids=True) op_group_inputs = dict() for gid in operations_gids: op_inputs = self.get_datatype_and_datatypegroup_inputs_for_operation(gid[0], selected_filter) for datatype in op_inputs: op_group_inputs[datatype.id] = datatype return list(op_group_inputs.values())
def get_datatypes_inputs_for_operation_group(group_id, selected_filter): """ Returns the dataType inputs for an operation group. If more dataTypes are part of the same dataType group then only the dataType group will be returned instead of them. """ operations_gids = dao.get_operations_in_group(group_id, only_gids=True) op_group_inputs = dict() for gid in operations_gids: op_inputs = ProjectService.get_datatype_and_datatypegroup_inputs_for_operation(gid[0], selected_filter) for datatype in op_inputs: op_group_inputs[datatype.id] = datatype return op_group_inputs.values()
def test_datatypes_groups(self, test_adapter_factory, datatype_group_factory): """ Tests if the dataType group is set correct on the dataTypes resulted from the same operation group. """ all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 0, "There should be no operation" dt_group = datatype_group_factory(project=self.test_project) model = TestModel() test_adapter_factory() adapter = TestFactory.create_adapter( "tvb.tests.framework.adapters.testadapter1", "TestAdapter1") operations = dao.get_operations_in_group(dt_group.id) for op in operations: model.gid = uuid.uuid4() op_path = StorageInterface().get_project_folder( self.test_project.name, str(op.id)) op.view_model_gid = model.gid.hex op.algorithm = adapter.stored_adapter h5.store_view_model(model, op_path) dao.store_entity(op) all_operations = dao.get_filtered_operations(self.test_project.id, None) assert len(all_operations) == 2, "Expected two operation groups" assert all_operations[0][2] == 6, "Expected 6 operations in one group" operation_group_id = all_operations[0][3] assert operation_group_id != None, "The operation should be part of a group." self.operation_service.stop_operation(all_operations[1][0]) self.operation_service.stop_operation(all_operations[1][1]) # Make sure operations are executed self.operation_service.launch_operation(all_operations[1][0], False) self.operation_service.launch_operation(all_operations[1][1], False) resulted_datatypes = dao.get_datatype_in_group( operation_group_id=operation_group_id) assert len( resulted_datatypes) >= 2, "Expected at least 2, but: " + str( len(resulted_datatypes)) dt = dao.get_datatype_by_id(resulted_datatypes[0].id) datatype_group = dao.get_datatypegroup_by_op_group_id( operation_group_id) assert dt.fk_datatype_group == datatype_group.id, "DataTypeGroup is incorrect"
def launch(self, datatype_group, **kwargs): """ Also overwrite launch from ABCDisplayer, since we want to handle a list of figures, instead of only one Matplotlib figure. :raises LaunchException: when called before all operations in the group are finished """ if self.PARAM_FIGURE_SIZE in kwargs: figsize = kwargs[self.PARAM_FIGURE_SIZE] figsize = ((figsize[0]) / 80, (figsize[1]) / 80) del kwargs[self.PARAM_FIGURE_SIZE] else: figsize = (15, 7) operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) _, range1_name, self.range1 = operation_group.load_range_numbers(operation_group.range1) _, range2_name, self.range2 = operation_group.load_range_numbers(operation_group.range2) for operation in dao.get_operations_in_group(operation_group.id): if operation.status == model.STATUS_STARTED: raise LaunchException("Can not display until all operations from this range are finished!") op_results = dao.get_results_for_operation(operation.id) if len(op_results): datatype = op_results[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.id)[0] else: dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') if dt_measure: dt_measure = dt_measure[0] break else: dt_measure = None figure_nrs = {} metrics = dt_measure.metrics if dt_measure else {} if metrics: for metric in metrics: # Separate plot for each metric. self._create_plot(metric, figsize, operation_group, range1_name, range2_name, figure_nrs) else: raise LaunchException("No datatypes were generated due to simulation errors. Nothing to display.") parameters = dict(title=self._ui_name, showFullToolbar=True, serverIp=config.SERVER_IP, serverPort=config.MPLH5_SERVER_PORT, figureNumbers=figure_nrs, metrics=metrics, figuresJSON=json.dumps(figure_nrs)) return self.build_display_result("pse_isocline/view", parameters)
def __init__(self, datatype_group_gid): self.datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if self.datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) self.operation_group = dao.get_operationgroup_by_id( self.datatype_group.fk_operation_group) self.operations = dao.get_operations_in_group(self.operation_group.id) self.pse_model_list = self.parse_pse_data_for_display() self.all_metrics = dict() self._prepare_ranges_data()
def prepare_node_data(datatype_group): if datatype_group is None: raise Exception("Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one.") operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) operations = dao.get_operations_in_group(operation_group.id) node_info_dict = dict() for operation_ in operations: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] node_info_dict[datatype.gid] = dict(operation_id=operation_.id, datatype_gid=datatype.gid, datatype_type=datatype.type, datatype_subject=datatype.subject, datatype_invalid=datatype.invalid) return node_info_dict
def _create_operations_with_inputs(self, datatype_group, is_group_parent=False): """ Method used for creating a complex tree of operations. If 'if_group_parent' is True then a new group will be created and one of its entries it will be used as input for the returned operations. """ group_dts = dao.get_datatypes_from_datatype_group(datatype_group.id) if is_group_parent: datatype_gid = group_dts[0].gid else: datatype_gid = self._create_value_wrapper(self.test_user, self.test_project)[1] parameters = json.dumps({"param_name": datatype_gid}) ops = [] for i in range(4): ops.append( TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)) if i in [1, 3]: ops[i].visible = False ops[i].parameters = parameters ops[i] = dao.store_entity(ops[i]) # groups ops_group = dao.get_operations_in_group( datatype_group.fk_from_operation) assert 9 == len(ops_group) ops_group[0].parameters = parameters ops_group[0] = dao.store_entity(ops_group[0]) ops_group[1].visible = False ops_group[1].parameters = parameters ops_group[1] = dao.store_entity(ops_group[1]) ops.extend(ops_group) if is_group_parent: dt_group = dao.get_datatypegroup_by_op_group_id(datatype_group.id) return ops, dt_group.id return ops, datatype_gid
def prepare_node_data(datatype_group): if datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) operations = dao.get_operations_in_group(operation_group.id) node_info_dict = dict() for operation_ in operations: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] node_info_dict[datatype.gid] = dict( operation_id=operation_.id, datatype_gid=datatype.gid, datatype_type=datatype.type, datatype_subject=datatype.subject, datatype_invalid=datatype.invalid) return node_info_dict
def _create_operations_with_inputs(self, is_group_parent=False): """ Method used for creating a complex tree of operations. If 'if_group_parent' is True then a new group will be created and one of its entries it will be used as input for the returned operations. """ group_dts, root_op_group_id = TestFactory.create_group(self.test_user, self.test_project) if is_group_parent: datatype_gid = group_dts[0].gid else: datatype_gid = ProjectServiceTest._create_value_wrapper(self.test_user, self.test_project)[1] parameters = json.dumps({"param_name": datatype_gid}) ops = [] for i in range(4): ops.append(TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project)) if i in [1, 3]: ops[i].visible = False ops[i].parameters = parameters ops[i] = dao.store_entity(ops[i]) #groups _, ops_group = TestFactory.create_group(self.test_user, self.test_project) ops_group = dao.get_operations_in_group(ops_group) self.assertEqual(2, len(ops_group)) ops_group[0].parameters = parameters ops_group[0] = dao.store_entity(ops_group[0]) ops_group[1].visible = False ops_group[1].parameters = parameters ops_group[1] = dao.store_entity(ops_group[1]) ops.extend(ops_group) if is_group_parent: dt_group = dao.get_datatypegroup_by_op_group_id(root_op_group_id) return ops, dt_group.id return ops, datatype_gid
def import_list_of_operations(self, project, import_path, is_group=False, importer_operation_id=None): """ This method scans provided folder and identify all operations that needs to be imported """ all_dts_count = 0 all_stored_dts_count = 0 imported_operations = [] ordered_operations = self._retrieve_operations_in_order( project, import_path, None if is_group else importer_operation_id) if is_group and len(ordered_operations) > 0: first_op = dao.get_operation_by_id(importer_operation_id) vm_path = h5.determine_filepath(first_op.view_model_gid, os.path.dirname(import_path)) os.remove(vm_path) ordered_operations[0].operation.id = importer_operation_id for operation_data in ordered_operations: if operation_data.is_old_form: operation_entity, datatype_group = self.import_operation( operation_data.operation) new_op_folder = self.storage_interface.get_project_folder( project.name, str(operation_entity.id)) try: operation_datatypes = self._load_datatypes_from_operation_folder( operation_data.operation_folder, operation_entity, datatype_group) # Create and store view_model from operation self.create_view_model(operation_entity, operation_data, new_op_folder) self._store_imported_datatypes_in_db( project, operation_datatypes) imported_operations.append(operation_entity) except MissingReferenceException: operation_entity.status = STATUS_ERROR dao.store_entity(operation_entity) elif operation_data.main_view_model is not None: operation_data.operation.create_date = datetime.now() operation_data.operation.start_date = datetime.now() operation_data.operation.completion_date = datetime.now() do_merge = False if importer_operation_id: do_merge = True operation_entity = dao.store_entity(operation_data.operation, merge=do_merge) dt_group = None op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: dt_group = dao.get_datatypegroup_by_op_group_id( op_group.id) if not dt_group: first_op = dao.get_operations_in_group( op_group.id, only_first_operation=True) dt_group = DataTypeGroup( op_group, operation_id=first_op.id, state=DEFAULTDATASTATE_INTERMEDIATE) dt_group = dao.store_entity(dt_group) # Store the DataTypes in db dts = {} all_dts_count += len(operation_data.dt_paths) for dt_path in operation_data.dt_paths: dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id) if isinstance(dt, BurstConfiguration): if op_group: dt.fk_operation_group = op_group.id all_stored_dts_count += self._store_or_link_burst_config( dt, dt_path, project.id) else: dts[dt_path] = dt if op_group: op_group.fill_operationgroup_name(dt.type) dao.store_entity(op_group) try: stored_dts_count = self._store_imported_datatypes_in_db( project, dts) all_stored_dts_count += stored_dts_count if operation_data.main_view_model.is_metric_operation: self._update_burst_metric(operation_entity) imported_operations.append(operation_entity) new_op_folder = self.storage_interface.get_project_folder( project.name, str(operation_entity.id)) view_model_disk_size = 0 for h5_file in operation_data.all_view_model_files: view_model_disk_size += StorageInterface.compute_size_on_disk( h5_file) shutil.move(h5_file, new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size dao.store_entity(operation_entity) except MissingReferenceException as excep: self.storage_interface.remove_operation_data( project.name, operation_entity.id) operation_entity.fk_operation_group = None dao.store_entity(operation_entity) dao.remove_entity(DataTypeGroup, dt_group.id) raise excep else: self.logger.warning( "Folder %s will be ignored, as we could not find a serialized " "operation or DTs inside!" % operation_data.operation_folder) # We want importer_operation_id to be kept just for the first operation (the first iteration) if is_group: importer_operation_id = None self._update_dt_groups(project.id) self._update_burst_configurations(project.id) return imported_operations, all_dts_count, all_stored_dts_count
def set_group_descendants_visibility(operation_group_id): ops_in_group = dao.get_operations_in_group(operation_group_id) for group_op in ops_in_group: set_visibility(group_op)
def get_operations_in_group(operation_group): """ Return all the operations from an operation group. """ return dao.get_operations_in_group(operation_group.id)
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. :param datatype_group_gid: the group id for the `DataType` to be visualised :param back_page: Page where back button will direct :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :param size_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid` :returns: `ContextDiscretePSE` :raises Exception: when `datatype_group_id` is invalid (not in database) """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception( "Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one." ) operation_group = dao.get_operationgroup_by_id( datatype_group.fk_operation_group) name1, values1, labels1, only_numbers1 = DiscretePSEAdapter.prepare_range_labels( operation_group, operation_group.range1) name2, values2, labels2, only_numbers2 = DiscretePSEAdapter.prepare_range_labels( operation_group, operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page) pse_context.setRanges(name1, values1, labels1, name2, values2, labels2, only_numbers1 and only_numbers2) final_dict = {} operations = dao.get_operations_in_group(operation_group.id) fake_numbers1 = dict(zip(values1, range(len(list(values1))))) fake_numbers2 = dict(zip(values2, range(len(list(values2))))) for operation_ in operations: if not operation_.has_finished: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = DiscretePSEAdapter.get_value_on_axe( range_values, only_numbers1, name1, fake_numbers1) key_2 = DiscretePSEAdapter.get_value_on_axe( range_values, only_numbers2, name2, fake_numbers2) datatype = None if operation_.status == model.STATUS_FINISHED: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. measures = dao.get_generic_entity( DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity( DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = {} final_dict[key_1][key_2] = pse_context.build_node_info( operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} if not only_numbers1: pse_context.values_x = range(len(list(values1))) if not only_numbers2: pse_context.values_y = range(len(list(values2))) return pse_context
def load_datatype_from_file(self, current_file, op_id, datatype_group=None, current_project_id=None): # type: (str, int, DataTypeGroup, int) -> HasTraitsIndex """ Creates an instance of datatype from storage / H5 file :returns: DatatypeIndex """ self.logger.debug("Loading DataType from file: %s" % current_file) h5_class = H5File.h5_class_from_file(current_file) if h5_class is BurstConfigurationH5: if current_project_id is None: op_entity = dao.get_operationgroup_by_id(op_id) current_project_id = op_entity.fk_launched_in h5_file = BurstConfigurationH5(current_file) burst = BurstConfiguration(current_project_id) burst.fk_simulation = op_id h5_file.load_into(burst) result = burst else: datatype, generic_attributes = h5.load_with_links(current_file) already_existing_datatype = h5.load_entity_by_gid(datatype.gid) if datatype_group is not None and already_existing_datatype is not None: raise DatatypeGroupImportException( "The datatype group that you are trying to import" " already exists!") index_class = h5.REGISTRY.get_index_for_datatype( datatype.__class__) datatype_index = index_class() datatype_index.fill_from_has_traits(datatype) datatype_index.fill_from_generic_attributes(generic_attributes) if datatype_group is not None and hasattr(datatype_index, 'fk_source_gid') and \ datatype_index.fk_source_gid is not None: ts = h5.load_entity_by_gid(datatype_index.fk_source_gid) if ts is None: op = dao.get_operations_in_group( datatype_group.fk_operation_group, only_first_operation=True) op.fk_operation_group = None dao.store_entity(op) dao.remove_entity(OperationGroup, datatype_group.fk_operation_group) dao.remove_entity(DataTypeGroup, datatype_group.id) raise DatatypeGroupImportException( "Please import the time series group before importing the" " datatype measure group!") # Add all the required attributes if datatype_group: datatype_index.fk_datatype_group = datatype_group.id if len(datatype_group.subject) == 0: datatype_group.subject = datatype_index.subject dao.store_entity(datatype_group) datatype_index.fk_from_operation = op_id associated_file = h5.path_for_stored_index(datatype_index) if os.path.exists(associated_file): datatype_index.disk_size = StorageInterface.compute_size_on_disk( associated_file) result = datatype_index return result
def plot(self, figure, operation_group, metric, range1_name, range2_name): """ Do the plot for the given figure. Also need operation group, metric and ranges in order to compute the data to be plotted. """ operations = dao.get_operations_in_group(operation_group.id) # Data from which to interpolate larger 2-D space apriori_x = numpy.array(self.range1) apriori_y = numpy.array(self.range2) apriori_data = numpy.zeros((apriori_x.size, apriori_y.size)) # An 2D array of GIDs which is used later to launch overlay for a DataType datatypes_gids = [[None for _ in self.range2] for _ in self.range1] for operation_ in operations: range_values = eval(operation_.range_values) key_1 = range_values[range1_name] index_x = self.range1.index(key_1) key_2 = range_values[range2_name] index_y = self.range2.index(key_2) if operation_.status != model.STATUS_FINISHED: raise LaunchException("Not all operations from this range are complete. Cannot view until then.") datatype = dao.get_results_for_operation(operation_.id)[0] datatypes_gids[index_x][index_y] = datatype.gid if datatype.type == "DatatypeMeasure": measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') if measures: apriori_data[index_x][index_y] = measures[0].metrics[metric] else: apriori_data[index_x][index_y] = 0 # Attempt order-3 interpolation. kx = ky = 3 if len(self.range1) <= 3 or len(self.range2) <= 3: # Number of points is too small, just do linear interpolation kx = ky = 1 s = interpolate.RectBivariateSpline(apriori_x, apriori_y, apriori_data, kx=kx, ky=ky) # Get data of higher resolution that we'll plot later on posteriori_x = numpy.arange(self.range1[0], self.range1[-1], (self.range1[-1] - self.range1[0]) / RESOLUTION[0]) posteriori_y = numpy.arange(self.range2[0], self.range2[-1], (self.range2[-1] - self.range2[0]) / RESOLUTION[1]) posteriori_data = numpy.rot90(s(posteriori_x, posteriori_y)) self.interp_models[figure.number] = s # Do actual plot. axes = figure.gca() img = axes.imshow(posteriori_data, extent=(min(self.range1), max(self.range1), min(self.range2), max(self.range2)), aspect='auto', interpolation='nearest') axes.set_title("Interpolated values for metric %s" % (metric,)) figure.colorbar(img) axes.set_xlabel(range1_name) axes.set_ylabel(range2_name) def format_coord(x, y): return 'x=%1.4f, y=%1.4f' % (x, y) axes.format_coord = format_coord return datatypes_gids
def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None): """ We suppose that there are max 2 ranges and from each operation results exactly one dataType. :param datatype_group_gid: the group id for the `DataType` to be visualised :param back_page: Page where back button will direct :param color_metric: String referring to metric to apply on colors :param size_metric: String referring to metric to apply on sizes :returns: `ContextDiscretePSE` :raises Exception: when `datatype_group_id` is invalid (not in database) """ datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid) if datatype_group is None: raise Exception("Selected DataTypeGroup is no longer present in the database. " "It might have been remove or the specified id is not the correct one.") operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group) name1, values1, labels1, only_numbers1 = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range1) name2, values2, labels2, only_numbers2 = DiscretePSEAdapter.prepare_range_labels(operation_group, operation_group.range2) pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page) pse_context.setRanges(name1, values1, labels1, name2, values2, labels2, only_numbers1 and only_numbers2) final_dict = {} operations = dao.get_operations_in_group(operation_group.id) fake_numbers1 = dict(zip(values1, range(len(list(values1))))) fake_numbers2 = dict(zip(values2, range(len(list(values2))))) for operation_ in operations: if not operation_.has_finished: pse_context.has_started_ops = True range_values = eval(operation_.range_values) key_1 = DiscretePSEAdapter.get_value_on_axe(range_values, only_numbers1, name1, fake_numbers1) key_2 = DiscretePSEAdapter.get_value_on_axe(range_values, only_numbers2, name2, fake_numbers2) datatype = None if operation_.status == model.STATUS_FINISHED: datatypes = dao.get_results_for_operation(operation_.id) if len(datatypes) > 0: datatype = datatypes[0] if datatype.type == "DatatypeMeasure": ## Load proper entity class from DB. measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') pse_context.prepare_metrics_datatype(measures, datatype) if key_1 not in final_dict: final_dict[key_1] = {} final_dict[key_1][key_2] = pse_context.build_node_info(operation_, datatype) pse_context.fill_object(final_dict) ## datatypes_dict is not actually used in the drawing of the PSE and actually ## causes problems in case of NaN values, so just remove it before creating the json pse_context.datatypes_dict = {} if not only_numbers1: pse_context.values_x = range(len(list(values1))) if not only_numbers2: pse_context.values_y = range(len(list(values2))) return pse_context
def import_project_operations(self, project, import_path, is_group=False, importer_operation_id=None): """ This method scans provided folder and identify all operations that needs to be imported """ all_dts_count = 0 all_stored_dts_count = 0 imported_operations = [] ordered_operations = self._retrieve_operations_in_order( project, import_path, importer_operation_id) for operation_data in ordered_operations: if operation_data.is_old_form: operation_entity, datatype_group = self.import_operation( operation_data.operation) new_op_folder = self.files_helper.get_project_folder( project, str(operation_entity.id)) try: operation_datatypes = self._load_datatypes_from_operation_folder( operation_data.operation_folder, operation_entity, datatype_group) # Create and store view_model from operation self.create_view_model(operation_entity, operation_data, new_op_folder) self._store_imported_datatypes_in_db( project, operation_datatypes) imported_operations.append(operation_entity) except MissingReferenceException: operation_entity.status = STATUS_ERROR dao.store_entity(operation_entity) elif operation_data.main_view_model is not None: do_merge = False if importer_operation_id: do_merge = True operation_entity = dao.store_entity(operation_data.operation, merge=do_merge) dt_group = None op_group = dao.get_operationgroup_by_id( operation_entity.fk_operation_group) if op_group: dt_group = dao.get_datatypegroup_by_op_group_id( op_group.id) if not dt_group: first_op = dao.get_operations_in_group( op_group.id, only_first_operation=True) dt_group = DataTypeGroup( op_group, operation_id=first_op.id, state=DEFAULTDATASTATE_INTERMEDIATE) dt_group = dao.store_entity(dt_group) # Store the DataTypes in db dts = {} all_dts_count += len(operation_data.dt_paths) for dt_path in operation_data.dt_paths: dt = self.load_datatype_from_file(dt_path, operation_entity.id, dt_group, project.id) if isinstance(dt, BurstConfiguration): if op_group: dt.fk_operation_group = op_group.id all_stored_dts_count += self._store_or_link_burst_config( dt, dt_path, project.id) else: dts[dt_path] = dt if op_group: op_group.fill_operationgroup_name(dt.type) dao.store_entity(op_group) try: stored_dts_count = self._store_imported_datatypes_in_db( project, dts) all_stored_dts_count += stored_dts_count if operation_data.main_view_model.is_metric_operation: self._update_burst_metric(operation_entity) #TODO: TVB-2849 to reveiw these flags and simplify condition if stored_dts_count > 0 or ( not operation_data.is_self_generated and not is_group) or importer_operation_id is not None: imported_operations.append(operation_entity) new_op_folder = self.files_helper.get_project_folder( project, str(operation_entity.id)) view_model_disk_size = 0 for h5_file in operation_data.all_view_model_files: view_model_disk_size += FilesHelper.compute_size_on_disk( h5_file) shutil.move(h5_file, new_op_folder) operation_entity.view_model_disk_size = view_model_disk_size dao.store_entity(operation_entity) else: # In case all Dts under the current operation were Links and the ViewModel is dummy, # don't keep the Operation empty in DB dao.remove_entity(Operation, operation_entity.id) self.files_helper.remove_operation_data( project.name, operation_entity.id) except MissingReferenceException as excep: dao.remove_entity(Operation, operation_entity.id) self.files_helper.remove_operation_data( project.name, operation_entity.id) raise excep else: self.logger.warning( "Folder %s will be ignored, as we could not find a serialized " "operation or DTs inside!" % operation_data.operation_folder) self._update_dt_groups(project.id) self._update_burst_configurations(project.id) return imported_operations, all_dts_count, all_stored_dts_count
def plot(self, figure, operation_group, metric, range1_name, range2_name): """ Do the plot for the given figure. Also need operation group, metric and ranges in order to compute the data to be plotted. """ operations = dao.get_operations_in_group(operation_group.id) # Data from which to interpolate larger 2-D space apriori_x = numpy.array(self.range1) apriori_y = numpy.array(self.range2) apriori_data = numpy.zeros((apriori_x.size, apriori_y.size)) # An 2D array of GIDs which is used later to launch overlay for a DataType datatypes_gids = [[None for _ in self.range2] for _ in self.range1] for operation_ in operations: range_values = eval(operation_.range_values) key_1 = range_values[range1_name] index_x = self.range1.index(key_1) key_2 = range_values[range2_name] index_y = self.range2.index(key_2) if operation_.status == model.STATUS_STARTED: raise LaunchException("Not all operations from this range are complete. Cannot view until then.") operation_results = dao.get_results_for_operation(operation_.id) if operation_results: datatype = operation_results[0] datatypes_gids[index_x][index_y] = datatype.gid if datatype.type == "DatatypeMeasure": measures = dao.get_generic_entity(DatatypeMeasure, datatype.id) else: measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype') else: datatypes_gids[index_x][index_y] = None measures = None if measures: apriori_data[index_x][index_y] = measures[0].metrics[metric] else: apriori_data[index_x][index_y] = numpy.NaN # Convert array to 0 but keep track of nan values so we can replace after interpolation # since interpolating with nan values will just break the whole process nan_indices = numpy.isnan(apriori_data) self.nan_indices[figure.number] = nan_indices apriori_data = numpy.nan_to_num(apriori_data) # Attempt order-3 interpolation. kx = ky = 3 if len(self.range1) <= 3 or len(self.range2) <= 3: # Number of points is too small, just do linear interpolation kx = ky = 1 s = interpolate.RectBivariateSpline(apriori_x, apriori_y, apriori_data, kx=kx, ky=ky) # Get data of higher resolution that we'll plot later on posteriori_x = numpy.arange(self.range1[0], self.range1[-1], (self.range1[-1] - self.range1[0]) / RESOLUTION[0]) posteriori_y = numpy.arange(self.range2[0], self.range2[-1], (self.range2[-1] - self.range2[0]) / RESOLUTION[1]) posteriori_data = s(posteriori_x, posteriori_y) x_granularity = RESOLUTION[0] / len(self.range1) y_granularity = RESOLUTION[1] / len(self.range2) for idx, row in enumerate(nan_indices): for idy, was_nan in enumerate(row): if was_nan: # Now we want to set back all the values that were NaN before interpolation # and keep track of the change in granularity. For this reason for each nan # value we had before, we will now have a matrix of the shape [x_granularity x y_granularity] # full of NaN values start_x = idx * x_granularity end_x = (idx + 1) * x_granularity start_y = idy * y_granularity end_y = (idy + 1) * y_granularity for x_scaled in xrange(start_x, end_x, 1): for y_scaled in xrange(start_y, end_y, 1): posteriori_data[x_scaled, y_scaled] = numpy.NaN # Rotate to get good plot posteriori_data = numpy.rot90(posteriori_data) self.interp_models[figure.number] = s # Do actual plot. axes = figure.gca() img = axes.imshow(posteriori_data, extent=(min(self.range1), max(self.range1), min(self.range2), max(self.range2)), aspect='auto', interpolation='nearest') axes.set_title("Interpolated values for metric %s" % (metric,)) figure.colorbar(img) axes.set_xlabel(range1_name) axes.set_ylabel(range2_name) def format_coord(x, y): return 'x=%1.4f, y=%1.4f' % (x, y) axes.format_coord = format_coord return datatypes_gids