Ejemplo n.º 1
0
    def _find_metrics(operations):
        """ Search for an operation with results. Then get the metrics of the generated data type"""
        dt_measure = None

        for operation in operations:

            if not operation.has_finished:
                raise LaunchException("Can not display until all operations from this range are finished!")

            op_results = dao.get_results_for_operation(operation.id)
            if len(op_results):
                datatype = op_results[0]
                if datatype.type == "DatatypeMeasure":
                    ## Load proper entity class from DB.
                    dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.id)[0]
                else:
                    dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
                    if dt_measure:
                        dt_measure = dt_measure[0]
                break

        if dt_measure:
            return dt_measure.metrics
        else:
            raise LaunchException("No datatypes were generated due to simulation errors. Nothing to display.")
Ejemplo n.º 2
0
    def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None):
        """
        We suppose that there are max 2 ranges and from each operation results exactly one dataType.

        :param datatype_group_gid: the group id for the `DataType` to be visualised
        :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid`
        :param size_metric:  a list of `DataTypeMeasure` which has been executed on `datatype_group_gid`

        :returns: `ContextDiscretePSE`

        :raises Exception: when `datatype_group_id` is invalid (not in database)
        """
        datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid)
        if datatype_group is None:
            raise Exception("Selected DataTypeGroup is no longer present in the database. "
                            "It might have been remove or the specified id is not the correct one.")

        operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group)

        range1_name, range1_values, range1_labels = DiscretePSEAdapter.prepare_range_labels(operation_group,
                                                                                            operation_group.range1)
        range2_name, range2_values, range2_labels = DiscretePSEAdapter.prepare_range_labels(operation_group,
                                                                                            operation_group.range2)

        pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page)
        pse_context.setRanges(range1_name, range1_values, range1_labels, range2_name, range2_values, range2_labels)
        final_dict = {}
        operations = dao.get_operations_in_group(operation_group.id)

        for operation_ in operations:
            if not operation_.has_finished:
                pse_context.has_started_ops = True
            range_values = eval(operation_.range_values)
            key_1 = range_values[range1_name]
            key_2 = model.RANGE_MISSING_STRING
            if range2_name is not None:
                key_2 = range_values[range2_name]

            datatype = None
            if operation_.status == model.STATUS_FINISHED:
                datatypes = dao.get_results_for_operation(operation_.id)
                if len(datatypes) > 0:
                    datatype = datatypes[0]
                    if datatype.type == "DatatypeMeasure":
                        ## Load proper entity class from DB.
                        measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
                    else:
                        measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
                    pse_context.prepare_metrics_datatype(measures, datatype)

            if key_1 not in final_dict:
                final_dict[key_1] = {}

            final_dict[key_1][key_2] = pse_context.build_node_info(operation_, datatype)

        pse_context.fill_object(final_dict)
        ## datatypes_dict is not actually used in the drawing of the PSE and actually
        ## causes problems in case of NaN values, so just remove it before creating the json
        pse_context.datatypes_dict = {}
        return pse_context
Ejemplo n.º 3
0
 def launch_visualization(visualization, frame_width=None, frame_height=None, 
                          method_name=ABCAdapter.LAUNCH_METHOD, is_preview=True):
     """
     :param visualization: a visualization workflow step
     """
     dynamic_params = visualization.dynamic_param
     static_params = visualization.static_param
     parameters_dict = static_params
     current_project_id = 0
     ## Current operation id needed for export mechanism. So far just use ##
     ## the operation of the workflow_step from which the inputs are taken    ####
     for param in dynamic_params:
         step_index = dynamic_params[param][WorkflowStepConfiguration.STEP_INDEX_KEY]
         datatype_index = dynamic_params[param][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]
         referred_workflow_step = dao.get_workflow_step_by_step_index(visualization.fk_workflow, step_index)
         referred_operation_id = referred_workflow_step.fk_operation
         referred_operation = dao.get_operation_by_id(referred_operation_id)
         current_project_id = referred_operation.fk_launched_in
         if type(datatype_index) is IntType:
             ## Entry is the output of a previous step ##
             datatypes = dao.get_results_for_operation(referred_operation_id)
             parameters_dict[param] = datatypes[datatype_index].gid
         else:
             ## Entry is the input of a previous step ###
             parameters_dict[param] = json.loads(referred_operation.parameters)[datatype_index]
     algorithm = dao.get_algorithm_by_id(visualization.fk_algorithm)
     adapter_instance = ABCAdapter.build_adapter(algorithm.algo_group)
     adapter_instance.current_project_id = current_project_id
     prepared_inputs = adapter_instance.prepare_ui_inputs(parameters_dict)
     if frame_width is not None:
         prepared_inputs[ABCDisplayer.PARAM_FIGURE_SIZE] = (frame_width, frame_height)
     if isinstance(adapter_instance, ABCMPLH5Displayer) and is_preview is True:
         prepared_inputs[ABCMPLH5Displayer.SHOW_FULL_TOOLBAR] = False
     result = eval("adapter_instance." + method_name + "(**prepared_inputs)")
     return result, parameters_dict
 def prepare_next_step(self, last_executed_op_id):
     """
     If the operation with id 'last_executed_op_id' resulted after
     the execution of a workflow step then this method will launch
     the operation corresponding to the next step from the workflow.
     """
     try:
         current_step, next_workflow_step = self._get_data(last_executed_op_id)
         if next_workflow_step is not None:
             operation = dao.get_operation_by_id(next_workflow_step.fk_operation)
             dynamic_param_names = next_workflow_step.dynamic_workflow_param_names
             if len(dynamic_param_names) > 0:
                 op_params = json.loads(operation.parameters)
                 for param_name in dynamic_param_names:
                     dynamic_param = op_params[param_name]
                     former_step = dao.get_workflow_step_by_step_index(next_workflow_step.fk_workflow,
                                                                       dynamic_param[wf_cfg.STEP_INDEX_KEY])
                     if type(dynamic_param[wf_cfg.DATATYPE_INDEX_KEY]) is IntType: 
                         datatypes = dao.get_results_for_operation(former_step.fk_operation)
                         op_params[param_name] = datatypes[dynamic_param[wf_cfg.DATATYPE_INDEX_KEY]].gid
                     else:
                         previous_operation = dao.get_operation_by_id(former_step.fk_operation)
                         op_params[param_name] = json.loads(previous_operation.parameters)[
                             dynamic_param[wf_cfg.DATATYPE_INDEX_KEY]]
                 operation.parameters = json.dumps(op_params)
                 operation = dao.store_entity(operation)
             return operation.id
         else:
             if current_step is not None:
                 current_workflow = dao.get_workflow_by_id(current_step.fk_workflow)
                 current_workflow.status = current_workflow.STATUS_FINISHED
                 dao.store_entity(current_workflow)
                 burst_entity = dao.get_burst_by_id(current_workflow.fk_burst)
                 parallel_workflows = dao.get_workflows_for_burst(burst_entity.id)
                 all_finished = True
                 for workflow in parallel_workflows:
                     if workflow.status == workflow.STATUS_STARTED:
                         all_finished = False
                 if all_finished:
                     self.mark_burst_finished(burst_entity, success=True)
                     disk_size = dao.get_burst_disk_size(burst_entity.id)  # Transform from kB to MB
                     if disk_size > 0:
                         user = dao.get_project_by_id(burst_entity.fk_project).administrator
                         user.used_disk_space = user.used_disk_space + disk_size
                         dao.store_entity(user)
             else:
                 operation = dao.get_operation_by_id(last_executed_op_id)
                 disk_size = dao.get_disk_size_for_operation(operation.id)  # Transform from kB to MB
                 if disk_size > 0:
                     user = dao.get_user_by_id(operation.fk_launched_by)
                     user.used_disk_space = user.used_disk_space + disk_size
                     dao.store_entity(user)
         return None
     except Exception, excep:
         self.logger.error(excep)
         self.logger.exception(excep)
         raise WorkflowInterStepsException(excep)
 def remove_operation(self, operation_id):
     """
     Remove a given operation 
     """
     operation = dao.get_operation_by_id(operation_id)
     if operation is not None:
         datatypes_for_op = dao.get_results_for_operation(operation_id)
         for dt in datatypes_for_op:
             self.remove_datatype(operation.project.id, dt.gid, True)
         dao.remove_entity(model.Operation, operation.id)
     else:
         self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
Ejemplo n.º 6
0
 def remove_operation(self, operation_id):
     """
     Remove a given operation
     """
     operation = dao.try_get_operation_by_id(operation_id)
     if operation is not None:
         self.logger.debug("Deleting operation %s " % operation)
         datatypes_for_op = dao.get_results_for_operation(operation_id)
         for dt in reversed(datatypes_for_op):
             self.remove_datatype(operation.project.id, dt.gid, False)
         dao.remove_entity(model.Operation, operation.id)
         self.logger.debug("Finished deleting operation %s " % operation)
     else:
         self.logger.warning("Attempt to delete operation with id=%s which no longer exists." % operation_id)
    def launch(self, datatype_group, **kwargs):
        """
        Also overwrite launch from ABCDisplayer, since we want to handle a list of figures,
        instead of only one Matplotlib figure.

        :raises LaunchException: when called before all operations in the group are finished
        """
        if self.PARAM_FIGURE_SIZE in kwargs:
            figsize = kwargs[self.PARAM_FIGURE_SIZE]
            figsize = ((figsize[0]) / 80, (figsize[1]) / 80)
            del kwargs[self.PARAM_FIGURE_SIZE]
        else:
            figsize = (15, 7)

        operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group)
        _, range1_name, self.range1 = operation_group.load_range_numbers(operation_group.range1)
        _, range2_name, self.range2 = operation_group.load_range_numbers(operation_group.range2)

        for operation in dao.get_operations_in_group(operation_group.id):
            if operation.status == model.STATUS_STARTED:
                raise LaunchException("Can not display until all operations from this range are finished!")

            op_results = dao.get_results_for_operation(operation.id)
            if len(op_results):
                datatype = op_results[0]
                if datatype.type == "DatatypeMeasure":
                    ## Load proper entity class from DB.
                    dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.id)[0]
                else:
                    dt_measure = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
                    if dt_measure:
                        dt_measure = dt_measure[0]
                break
            else:
                dt_measure = None

        figure_nrs = {}
        metrics = dt_measure.metrics if dt_measure else {}
        if metrics:
            for metric in metrics:
                # Separate plot for each metric.
                self._create_plot(metric, figsize, operation_group, range1_name, range2_name, figure_nrs)
        else:
            raise LaunchException("No datatypes were generated due to simulation errors. Nothing to display.")

        parameters = dict(title=self._ui_name, showFullToolbar=True,
                          serverIp=config.SERVER_IP, serverPort=config.MPLH5_SERVER_PORT,
                          figureNumbers=figure_nrs, metrics=metrics, figuresJSON=json.dumps(figure_nrs))

        return self.build_display_result("pse_isocline/view", parameters)
Ejemplo n.º 8
0
    def prepare_node_data(datatype_group):
        if datatype_group is None:
            raise Exception("Selected DataTypeGroup is no longer present in the database. "
                            "It might have been remove or the specified id is not the correct one.")

        operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group)
        operations = dao.get_operations_in_group(operation_group.id)
        node_info_dict = dict()
        for operation_ in operations:
            datatypes = dao.get_results_for_operation(operation_.id)
            if len(datatypes) > 0:
                datatype = datatypes[0]
                node_info_dict[datatype.gid] = dict(operation_id=operation_.id,
                                                    datatype_gid=datatype.gid,
                                                    datatype_type=datatype.type,
                                                    datatype_subject=datatype.subject,
                                                    datatype_invalid=datatype.invalid)
        return node_info_dict
Ejemplo n.º 9
0
 def launch_visualization(visualization,
                          frame_width=None,
                          frame_height=None,
                          method_name=ABCAdapter.LAUNCH_METHOD,
                          is_preview=True):
     """
     :param visualization: a visualization workflow step
     """
     dynamic_params = visualization.dynamic_param
     static_params = visualization.static_param
     parameters_dict = static_params
     operation_id = 0
     ## Current operation id needed for export mechanism. So far just use ##
     ## the operation of the workflow_step from which the inputs are taken    ####
     for param in dynamic_params:
         step_index = dynamic_params[param][
             WorkflowStepConfiguration.STEP_INDEX_KEY]
         datatype_index = dynamic_params[param][
             WorkflowStepConfiguration.DATATYPE_INDEX_KEY]
         workflow_step = dao.get_workflow_step_by_step_index(
             visualization.fk_workflow, step_index)
         if type(datatype_index) is IntType:
             ## Entry is the output of a previous step ##
             operation_id = workflow_step.fk_operation
             datatypes = dao.get_results_for_operation(operation_id)
             parameters_dict[param] = datatypes[datatype_index].gid
         else:
             ## Entry is the input of a previous step ###
             operation = dao.get_operation_by_id(workflow_step.fk_operation)
             parameters_dict[param] = json.loads(
                 operation.parameters)[datatype_index]
     algorithm = dao.get_algorithm_by_id(visualization.fk_algorithm)
     adapter_instance = ABCAdapter.build_adapter(algorithm.algo_group)
     prepared_inputs = adapter_instance.prepare_ui_inputs(parameters_dict)
     if frame_width is not None:
         prepared_inputs[ABCDisplayer.PARAM_FIGURE_SIZE] = (frame_width,
                                                            frame_height)
     if isinstance(adapter_instance,
                   ABCMPLH5Displayer) and is_preview is True:
         prepared_inputs[ABCMPLH5Displayer.SHOW_FULL_TOOLBAR] = False
     result = eval("adapter_instance." + method_name +
                   "(**prepared_inputs)")
     return result, parameters_dict, operation_id
Ejemplo n.º 10
0
 def remove_operation(self, operation_id):
     """
     Remove a given operation
     """
     operation = dao.try_get_operation_by_id(operation_id)
     if operation is not None:
         self.logger.debug("Deleting operation %s " % operation)
         datatypes_for_op = dao.get_results_for_operation(operation_id)
         for dt in reversed(datatypes_for_op):
             self.remove_datatype(operation.project.id, dt.gid, False)
         # Here the Operation is mot probably already removed - in case DTs were found inside
         # but we still remove it for the case when no DTs exist
         dao.remove_entity(Operation, operation.id)
         self.storage_interface.remove_operation_data(
             operation.project.name, operation_id)
         self.storage_interface.push_folder_to_sync(operation.project.name)
         self.logger.debug("Finished deleting operation %s " % operation)
     else:
         self.logger.warning(
             "Attempt to delete operation with id=%s which no longer exists."
             % operation_id)
Ejemplo n.º 11
0
    def prepare_node_data(datatype_group):
        if datatype_group is None:
            raise Exception(
                "Selected DataTypeGroup is no longer present in the database. "
                "It might have been remove or the specified id is not the correct one."
            )

        operation_group = dao.get_operationgroup_by_id(
            datatype_group.fk_operation_group)
        operations = dao.get_operations_in_group(operation_group.id)
        node_info_dict = dict()
        for operation_ in operations:
            datatypes = dao.get_results_for_operation(operation_.id)
            if len(datatypes) > 0:
                datatype = datatypes[0]
                node_info_dict[datatype.gid] = dict(
                    operation_id=operation_.id,
                    datatype_gid=datatype.gid,
                    datatype_type=datatype.type,
                    datatype_subject=datatype.subject,
                    datatype_invalid=datatype.invalid)
        return node_info_dict
Ejemplo n.º 12
0
    def launch_visualization(visualization, frame_width=None, frame_height=None, is_preview=True):
        """
        :param visualization: a visualization workflow step
        """
        dynamic_params = visualization.dynamic_param
        static_params = visualization.static_param
        parameters_dict = static_params
        current_project_id = 0
        # Current operation id needed for export mechanism. So far just use ##
        # the operation of the workflow_step from which the inputs are taken    ####
        for param in dynamic_params:
            step_index = dynamic_params[param][WorkflowStepConfiguration.STEP_INDEX_KEY]
            datatype_index = dynamic_params[param][WorkflowStepConfiguration.DATATYPE_INDEX_KEY]
            referred_workflow_step = dao.get_workflow_step_by_step_index(visualization.fk_workflow, step_index)
            referred_operation_id = referred_workflow_step.fk_operation
            referred_operation = dao.get_operation_by_id(referred_operation_id)
            current_project_id = referred_operation.fk_launched_in
            if type(datatype_index) is IntType:
                # Entry is the output of a previous step ##
                datatypes = dao.get_results_for_operation(referred_operation_id)
                parameters_dict[param] = datatypes[datatype_index].gid
            else:
                # Entry is the input of a previous step ###
                parameters_dict[param] = json.loads(referred_operation.parameters)[datatype_index]
        algorithm = dao.get_algorithm_by_id(visualization.fk_algorithm)
        adapter_instance = ABCAdapter.build_adapter(algorithm)
        adapter_instance.current_project_id = current_project_id
        prepared_inputs = adapter_instance.prepare_ui_inputs(parameters_dict)
        if frame_width is not None:
            prepared_inputs[ABCDisplayer.PARAM_FIGURE_SIZE] = (frame_width, frame_height)

        if is_preview:
            result = adapter_instance.generate_preview(**prepared_inputs)
        else:
            result = adapter_instance.launch(**prepared_inputs)
        return result, parameters_dict
Ejemplo n.º 13
0
    def _fill_apriori_data(self, operations):
        """ Gather apriori data from the operations. Also gather the datatype gid's"""
        for metric in self.metrics:
            self.apriori_data[metric] = numpy.zeros((self.apriori_x.size, self.apriori_y.size))

        # An 2D array of GIDs which is used later to launch overlay for a DataType
        self.datatypes_gids = [[None for _ in self.range2] for _ in self.range1]

        for operation in operations:
            self.log.debug("Gathering data from operation : %s" % operation.id)
            range_values = eval(operation.range_values)
            key_1 = range_values[self.range1_name]
            index_x = self.range1.index(key_1)
            key_2 = range_values[self.range2_name]
            index_y = self.range2.index(key_2)
            if operation.status == STATUS_STARTED:
                raise LaunchException("Not all operations from this range are complete. Cannot view until then.")

            operation_results = dao.get_results_for_operation(operation.id)
            if operation_results:
                datatype = operation_results[0]
                self.datatypes_gids[index_x][index_y] = str(datatype.gid)

                if datatype.type == "DatatypeMeasure":
                    measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
                else:
                    measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
            else:
                self.datatypes_gids[index_x][index_y] = None
                measures = None

            for metric in self.metrics:
                if measures:
                    self.apriori_data[metric][index_x][index_y] = measures[0].metrics[metric]
                else:
                    self.apriori_data[metric][index_x][index_y] = numpy.NaN
Ejemplo n.º 14
0
    def _fill_apriori_data(self, operations):
        """ Gather apriori data from the operations. Also gather the datatype gid's"""
        for metric in self.metrics:
            self.apriori_data[metric] = numpy.zeros((self.apriori_x.size, self.apriori_y.size))

        # An 2D array of GIDs which is used later to launch overlay for a DataType
        self.datatypes_gids = [[None for _ in self.range2] for _ in self.range1]

        for operation in operations:
            self.log.debug("Gathering data from operation : %s" % operation.id)
            range_values = eval(operation.range_values)
            key_1 = range_values[self.range1_name]
            index_x = self.range1.index(key_1)
            key_2 = range_values[self.range2_name]
            index_y = self.range2.index(key_2)
            if operation.status == STATUS_STARTED:
                raise LaunchException("Not all operations from this range are complete. Cannot view until then.")

            operation_results = dao.get_results_for_operation(operation.id)
            if operation_results:
                datatype = operation_results[0]
                self.datatypes_gids[index_x][index_y] = datatype.gid

                if datatype.type == "DatatypeMeasure":
                    measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
                else:
                    measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
            else:
                self.datatypes_gids[index_x][index_y] = None
                measures = None

            for metric in self.metrics:
                if measures:
                    self.apriori_data[metric][index_x][index_y] = measures[0].metrics[metric]
                else:
                    self.apriori_data[metric][index_x][index_y] = numpy.NaN
Ejemplo n.º 15
0
    def prepare_parameters(datatype_group_gid,
                           back_page,
                           color_metric=None,
                           size_metric=None):
        """
        We suppose that there are max 2 ranges and from each operation results exactly one dataType.

        :param datatype_group_gid: the group id for the `DataType` to be visualised
        :param back_page: Page where back button will direct
        :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid`
        :param size_metric:  a list of `DataTypeMeasure` which has been executed on `datatype_group_gid`

        :returns: `ContextDiscretePSE`
        :raises Exception: when `datatype_group_id` is invalid (not in database)
        """
        datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid)
        if datatype_group is None:
            raise Exception(
                "Selected DataTypeGroup is no longer present in the database. "
                "It might have been remove or the specified id is not the correct one."
            )

        operation_group = dao.get_operationgroup_by_id(
            datatype_group.fk_operation_group)

        name1, values1, labels1, only_numbers1 = DiscretePSEAdapter.prepare_range_labels(
            operation_group, operation_group.range1)
        name2, values2, labels2, only_numbers2 = DiscretePSEAdapter.prepare_range_labels(
            operation_group, operation_group.range2)

        pse_context = ContextDiscretePSE(datatype_group_gid, color_metric,
                                         size_metric, back_page)
        pse_context.setRanges(name1, values1, labels1, name2, values2, labels2,
                              only_numbers1 and only_numbers2)
        final_dict = {}
        operations = dao.get_operations_in_group(operation_group.id)

        fake_numbers1 = dict(zip(values1, range(len(list(values1)))))
        fake_numbers2 = dict(zip(values2, range(len(list(values2)))))

        for operation_ in operations:
            if not operation_.has_finished:
                pse_context.has_started_ops = True
            range_values = eval(operation_.range_values)
            key_1 = DiscretePSEAdapter.get_value_on_axe(
                range_values, only_numbers1, name1, fake_numbers1)
            key_2 = DiscretePSEAdapter.get_value_on_axe(
                range_values, only_numbers2, name2, fake_numbers2)

            datatype = None
            if operation_.status == model.STATUS_FINISHED:
                datatypes = dao.get_results_for_operation(operation_.id)
                if len(datatypes) > 0:
                    datatype = datatypes[0]
                    if datatype.type == "DatatypeMeasure":
                        ## Load proper entity class from DB.
                        measures = dao.get_generic_entity(
                            DatatypeMeasure, datatype.id)
                    else:
                        measures = dao.get_generic_entity(
                            DatatypeMeasure, datatype.gid,
                            '_analyzed_datatype')
                    pse_context.prepare_metrics_datatype(measures, datatype)

            if key_1 not in final_dict:
                final_dict[key_1] = {}

            final_dict[key_1][key_2] = pse_context.build_node_info(
                operation_, datatype)

        pse_context.fill_object(final_dict)
        ## datatypes_dict is not actually used in the drawing of the PSE and actually
        ## causes problems in case of NaN values, so just remove it before creating the json
        pse_context.datatypes_dict = {}
        if not only_numbers1:
            pse_context.values_x = range(len(list(values1)))
        if not only_numbers2:
            pse_context.values_y = range(len(list(values2)))
        return pse_context
Ejemplo n.º 16
0
    def plot(self, figure, operation_group, metric, range1_name, range2_name):
        """
        Do the plot for the given figure. Also need operation group, metric and ranges
        in order to compute the data to be plotted.
        """
        operations = dao.get_operations_in_group(operation_group.id)
        # Data from which to interpolate larger 2-D space
        apriori_x = numpy.array(self.range1)
        apriori_y = numpy.array(self.range2)
        apriori_data = numpy.zeros((apriori_x.size, apriori_y.size))

        # An 2D array of GIDs which is used later to launch overlay for a DataType
        datatypes_gids = [[None for _ in self.range2] for _ in self.range1]
        for operation_ in operations:
            range_values = eval(operation_.range_values)
            key_1 = range_values[range1_name]
            index_x = self.range1.index(key_1)
            key_2 = range_values[range2_name]
            index_y = self.range2.index(key_2)
            if operation_.status != model.STATUS_FINISHED:
                raise LaunchException("Not all operations from this range are complete. Cannot view until then.")

            datatype = dao.get_results_for_operation(operation_.id)[0]
            datatypes_gids[index_x][index_y] = datatype.gid

            if datatype.type == "DatatypeMeasure":
                measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
            else:
                measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')

            if measures:
                apriori_data[index_x][index_y] = measures[0].metrics[metric]
            else:
                apriori_data[index_x][index_y] = 0

        # Attempt order-3 interpolation.
        kx = ky = 3
        if len(self.range1) <= 3 or len(self.range2) <= 3:
            # Number of points is too small, just do linear interpolation
            kx = ky = 1
        s = interpolate.RectBivariateSpline(apriori_x, apriori_y, apriori_data, kx=kx, ky=ky)
        # Get data of higher resolution that we'll plot later on
        posteriori_x = numpy.arange(self.range1[0], self.range1[-1],
                                    (self.range1[-1] - self.range1[0]) / RESOLUTION[0])
        posteriori_y = numpy.arange(self.range2[0], self.range2[-1],
                                    (self.range2[-1] - self.range2[0]) / RESOLUTION[1])
        posteriori_data = numpy.rot90(s(posteriori_x, posteriori_y))

        self.interp_models[figure.number] = s
        # Do actual plot.        
        axes = figure.gca()
        img = axes.imshow(posteriori_data, extent=(min(self.range1), max(self.range1),
                                                   min(self.range2), max(self.range2)),
                          aspect='auto', interpolation='nearest')
        axes.set_title("Interpolated values for metric %s" % (metric,))
        figure.colorbar(img)
        axes.set_xlabel(range1_name)
        axes.set_ylabel(range2_name)


        def format_coord(x, y):
            return 'x=%1.4f, y=%1.4f' % (x, y)


        axes.format_coord = format_coord
        return datatypes_gids
Ejemplo n.º 17
0
    def prepare_parameters(datatype_group_gid,
                           back_page,
                           color_metric=None,
                           size_metric=None):
        """
        We suppose that there are max 2 ranges and from each operation results exactly one dataType.

        :param datatype_group_gid: the group id for the `DataType` to be visualised
        :param color_metric: a list of `DataTypeMeasure` which has been executed on `datatype_group_gid`
        :param size_metric:  a list of `DataTypeMeasure` which has been executed on `datatype_group_gid`

        :returns: `ContextDiscretePSE`

        :raises Exception: when `datatype_group_id` is invalid (not in database)
        """
        datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid)
        if datatype_group is None:
            raise Exception(
                "Selected DataTypeGroup is no longer present in the database. "
                "It might have been remove or the specified id is not the correct one."
            )

        operation_group = dao.get_operationgroup_by_id(
            datatype_group.fk_operation_group)
        _, range1_name, range1_labels = operation_group.load_range_numbers(
            operation_group.range1)
        has_range2, range2_name, range2_labels = operation_group.load_range_numbers(
            operation_group.range2)

        pse_context = ContextDiscretePSE(datatype_group_gid, range1_labels,
                                         range2_labels, color_metric,
                                         size_metric, back_page)
        final_dict = dict()
        operations = dao.get_operations_in_group(operation_group.id)
        for operation_ in operations:
            if operation_.status == model.STATUS_STARTED:
                pse_context.has_started_ops = True
            range_values = eval(operation_.range_values)
            key_1 = range_values[range1_name]
            key_2 = model.RANGE_MISSING_STRING
            if has_range2 is not None:
                key_2 = range_values[range2_name]

            datatype = None
            if operation_.status == model.STATUS_FINISHED:
                datatype = dao.get_results_for_operation(operation_.id)[0]
                measures = dao.get_generic_entity(DatatypeMeasure,
                                                  datatype.gid,
                                                  '_analyzed_datatype')
                pse_context.prepare_metrics_datatype(measures, datatype)

            if key_1 not in final_dict:
                final_dict[key_1] = {
                    key_2: pse_context.build_node_info(operation_, datatype)
                }
            else:
                final_dict[key_1][key_2] = pse_context.build_node_info(
                    operation_, datatype)

        pse_context.fill_object(final_dict)
        ## datatypes_dict is not actually used in the drawing of the PSE and actually
        ## causes problems in case of NaN values, so just remove it before creating the json
        pse_context.datatypes_dict = {}
        return pse_context
Ejemplo n.º 18
0
 def prepare_next_step(self, last_executed_op_id):
     """
     If the operation with id 'last_executed_op_id' resulted after
     the execution of a workflow step then this method will launch
     the operation corresponding to the next step from the workflow.
     """
     try:
         current_step, next_workflow_step = self._get_data(
             last_executed_op_id)
         if next_workflow_step is not None:
             operation = dao.get_operation_by_id(
                 next_workflow_step.fk_operation)
             dynamic_param_names = next_workflow_step.dynamic_workflow_param_names
             if len(dynamic_param_names) > 0:
                 op_params = json.loads(operation.parameters)
                 for param_name in dynamic_param_names:
                     dynamic_param = op_params[param_name]
                     former_step = dao.get_workflow_step_by_step_index(
                         next_workflow_step.fk_workflow,
                         dynamic_param[wf_cfg.STEP_INDEX_KEY])
                     if type(dynamic_param[
                             wf_cfg.DATATYPE_INDEX_KEY]) is IntType:
                         datatypes = dao.get_results_for_operation(
                             former_step.fk_operation)
                         op_params[param_name] = datatypes[dynamic_param[
                             wf_cfg.DATATYPE_INDEX_KEY]].gid
                     else:
                         previous_operation = dao.get_operation_by_id(
                             former_step.fk_operation)
                         op_params[param_name] = json.loads(
                             previous_operation.parameters)[dynamic_param[
                                 wf_cfg.DATATYPE_INDEX_KEY]]
                 operation.parameters = json.dumps(op_params)
                 operation = dao.store_entity(operation)
             return operation.id
         else:
             if current_step is not None:
                 current_workflow = dao.get_workflow_by_id(
                     current_step.fk_workflow)
                 current_workflow.status = current_workflow.STATUS_FINISHED
                 dao.store_entity(current_workflow)
                 burst_entity = dao.get_burst_by_id(
                     current_workflow.fk_burst)
                 parallel_workflows = dao.get_workflows_for_burst(
                     burst_entity.id)
                 all_finished = True
                 for workflow in parallel_workflows:
                     if workflow.status == workflow.STATUS_STARTED:
                         all_finished = False
                 if all_finished:
                     self.mark_burst_finished(burst_entity, success=True)
                     disk_size = dao.get_burst_disk_size(
                         burst_entity.id)  #Transform from kB to MB
                     if disk_size > 0:
                         user = dao.get_project_by_id(
                             burst_entity.fk_project).administrator
                         user.used_disk_space = user.used_disk_space + disk_size
                         dao.store_entity(user)
             else:
                 operation = dao.get_operation_by_id(last_executed_op_id)
                 disk_size = dao.get_disk_size_for_operation(
                     operation.id)  #Transform from kB to MB
                 if disk_size > 0:
                     user = dao.get_user_by_id(operation.fk_launched_by)
                     user.used_disk_space = user.used_disk_space + disk_size
                     dao.store_entity(user)
         return None
     except Exception, excep:
         self.logger.error(excep)
         self.logger.exception(excep)
         raise WorkflowInterStepsException(excep)
Ejemplo n.º 19
0
 def get_results_for_operation(operation_id, selected_filter=None):
     """
     Retrieve the DataTypes entities resulted after the execution of the given operation.
     """
     return dao.get_results_for_operation(operation_id, selected_filter)
Ejemplo n.º 20
0
    def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
        """
        Return a Tuple with Project entity and Operations for current Project.
        :param project_id: Current Project Identifier
        :param applied_filters: Filters to apply on Operations
        :param current_page: Number for current page in operations
        """
        selected_project = self.find_project(project_id)
        total_filtered = self.count_filtered_operations(project_id, applied_filters)
        pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
        total_ops_nr = self.count_filtered_operations(project_id)

        start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
        current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
        if current_ops is None:
            return selected_project, 0, [], 0

        operations = []
        view_categ_id = dao.get_visualisers_categories()[0].id
        for one_op in current_ops:
            try:
                result = {}
                if one_op[0] != one_op[1]:
                    result["id"] = str(one_op[0]) + "-" + str(one_op[1])
                else:
                    result["id"] = str(one_op[0])
                burst = dao.get_burst_for_operation_id(one_op[0])
                result["burst_name"] = burst.name if burst else '-'
                result["count"] = one_op[2]
                result["gid"] = one_op[14]
                if one_op[3] is not None and one_op[3]:
                    try:
                        operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
                        result["group"] = operation_group.name
                        result["group"] = result["group"].replace("_", " ")
                        result["operation_group_id"] = operation_group.id
                        datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
                        result["datatype_group_gid"] = datatype_group.gid
                        result["gid"] = operation_group.gid

                        ## Filter only viewers for current DataTypeGroup entity:
                        launcher = self.retrieve_launchers(datatype_group.gid,
                                                           include_categories=[view_categ_id]).values()[0]
                        view_groups = []
                        for launcher in launcher.values():
                            url = '/flow/' + str(launcher['category']) + '/' + str(launcher['id'])
                            if launcher['part_of_group']:
                                url = '/flow/prepare_group_launch/' + datatype_group.gid + '/' + \
                                      str(launcher['category']) + '/' + str(launcher['id'])
                            view_groups.append(dict(name=launcher["displayName"],
                                                    url=url,
                                                    param_name=launcher['children'][0]['param_name'],
                                                    part_of_group=launcher['part_of_group']))
                        result["view_groups"] = view_groups

                    except Exception:
                        self.logger.exception("We will ignore group on entity:" + str(one_op))
                        result["datatype_group_gid"] = None
                else:
                    result['group'] = None
                    result['datatype_group_gid'] = None
                result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
                result["method"] = one_op[5]
                result["user"] = dao.get_user_by_id(one_op[6])
                if type(one_op[7]) in (str, unicode):
                    result["create"] = string2date(str(one_op[7]))
                else:
                    result["create"] = one_op[7]
                if type(one_op[8]) in (str, unicode):
                    result["start"] = string2date(str(one_op[8]))
                else:
                    result["start"] = one_op[8]
                if type(one_op[9]) in (str, unicode):
                    result["complete"] = string2date(str(one_op[9]))
                else:
                    result["complete"] = one_op[9]

                if result["complete"] is not None and result["start"] is not None:
                    result["duration"] = format_timedelta(result["complete"] - result["start"])
                result["status"] = one_op[10]
                result["additional"] = one_op[11]
                result["visible"] = True if one_op[12] > 0 else False
                result['operation_tag'] = one_op[13]
                result['figures'] = None
                if not result['group']:
                    datatype_results = dao.get_results_for_operation(result['id'])
                    result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type,
                                                                dt.gid, 'gid')[0] for dt in datatype_results]
                    operation_figures = dao.get_figures_for_operation(result['id'])

                    # Compute the full path to the figure / image on disk
                    for figure in operation_figures:
                        figures_folder = self.structure_helper.get_images_folder(figure.project.name)
                        figure_full_path = os.path.join(figures_folder, figure.file_path)
                        # Compute the path available from browser
                        figure.figure_path = utils.path2url_part(figure_full_path)

                    result['figures'] = operation_figures
                else:
                    result['results'] = None
                operations.append(result)
            except Exception:
                ## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
                self.logger.exception("Could not prepare operation for display:" + str(one_op))
        return selected_project, total_ops_nr, operations, pages_no
Ejemplo n.º 21
0
def list_operation_results(operation_id):
    fmt = "%16s %24s %32s %12s"
    print(fmt % ('id', 'type', 'gid', 'date'))
    for dt in dao.get_results_for_operation(operation_id):
        print(fmt % (dt.id, dt.type, dt.gid, dt.create_date))
    def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
        """
        Return a Tuple with Project entity and Operations for current Project.
        :param project_id: Current Project Identifier
        :param applied_filters: Filters to apply on Operations
        :param current_page: Number for current page in operations
        """
        selected_project = self.find_project(project_id)
        total_filtered = self.count_filtered_operations(project_id, applied_filters)
        start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)

        if total_filtered >= start_idx + OPERATIONS_PAGE_SIZE:
            end_idx = OPERATIONS_PAGE_SIZE
        else:
            end_idx = total_filtered - start_idx

        pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
        total_ops_nr = self.count_filtered_operations(project_id)
        current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, end_idx)
        started_ops = 0
        if current_ops is None:
            return selected_project, [], 0
        operations = []
        for one_op in current_ops:
            try:
                result = {}
                if one_op[0] != one_op[1]:
                    result["id"] = str(one_op[0]) + "-" + str(one_op[1])
                else:
                    result["id"] = str(one_op[0])
                burst = dao.get_burst_for_operation_id(one_op[0])
                result["burst_name"] = burst.name if burst else '-'
                result["count"] = one_op[2]
                result["gid"] = one_op[14]
                if one_op[3] is not None and one_op[3]:
                    try:
                        operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0]
                        result["group"] = operation_group.name
                        result["group"] = result["group"].replace("_", " ")
                        result["operation_group_id"] = operation_group.id
                        datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
                        datatype = dao.get_datatype_by_id(datatype_group.id)
                        result["datatype_group_gid"] = datatype.gid
                        result["gid"] = operation_group.gid
                        
                        all_categs = dao.get_algorithm_categories()
                        view_categ = dao.get_visualisers_categories()[0]
                        excludes = [categ.id for categ in all_categs if categ.id != view_categ.id]
                        algo = self.retrieve_launchers("DataTypeGroup", datatype.gid,
                                                       exclude_categories=excludes).values()[0]

                        view_groups = []
                        for algo in algo.values():
                            url = '/flow/' + str(algo['category']) + '/' + str(algo['id'])
                            if algo['part_of_group']:
                                url = '/flow/prepare_group_launch/' + datatype.gid + '/' + \
                                      str(algo['category']) + '/' + str(algo['id'])
                            view_groups.append(dict(name=algo["displayName"],
                                                    url=url,
                                                    param_name=algo['children'][0]['param_name'],
                                                    part_of_group=algo['part_of_group']))
                        result["view_groups"] = view_groups

                    except Exception, excep:
                        self.logger.error(excep)
                        self.logger.warning("Will ignore group on entity:" + str(one_op))
                        result["datatype_group_gid"] = None
                else:
                    result['group'] = None
                    result['datatype_group_gid'] = None
                result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
                result["method"] = one_op[5]
                result["user"] = dao.get_user_by_id(one_op[6])
                if type(one_op[7]) in (str, unicode):
                    result["create"] = string2date(str(one_op[7]))
                else:
                    result["create"] = one_op[7]
                if type(one_op[8]) in (str, unicode):
                    result["start"] = string2date(str(one_op[8]))
                else:
                    result["start"] = one_op[8]
                if type(one_op[9]) in (str, unicode):
                    result["complete"] = string2date(str(one_op[9]))
                else:
                    result["complete"] = one_op[9]

                if result["complete"] is not None and result["start"] is not None:
                    result["duration"] = timedelta2string(result["complete"] - result["start"])
                result["status"] = one_op[10]
                if result["status"] == model.STATUS_STARTED:
                    started_ops += 1
                result["additional"] = one_op[11]
                result["visible"] = True if one_op[12] > 0 else False
                result['operation_tag'] = one_op[13]
                result['figures'] = None
                if not result['group']:
                    datatype_results = dao.get_results_for_operation(result['id'])
                    result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type,
                                                                dt.gid, 'gid')[0] for dt in datatype_results]
                    operation_figures = dao.get_figures_for_operation(result['id'])

                    # Compute the full path to the figure / image on disk
                    for figure in operation_figures:
                        figures_folder = self.structure_helper.get_images_folder(figure.project.name,
                                                                                 figure.operation.id)
                        figure_full_path = os.path.join(figures_folder, figure.file_path)
                        # Compute the path available from browser 
                        figure.figure_path = utils.path2url_part(figure_full_path)

                    result['figures'] = operation_figures
                else:
                    result['results'] = None
                operations.append(result)
Ejemplo n.º 23
0
    def prepare_parameters(datatype_group_gid, back_page, color_metric=None, size_metric=None):
        """
        We suppose that there are max 2 ranges and from each operation results exactly one dataType.

        :param datatype_group_gid: the group id for the `DataType` to be visualised
        :param back_page: Page where back button will direct
        :param color_metric: String referring to metric to apply on colors
        :param size_metric:  String referring to metric to apply on sizes

        :returns: `ContextDiscretePSE`
        :raises Exception: when `datatype_group_id` is invalid (not in database)
        """
        datatype_group = dao.get_datatype_group_by_gid(datatype_group_gid)
        if datatype_group is None:
            raise Exception("Selected DataTypeGroup is no longer present in the database. "
                            "It might have been remove or the specified id is not the correct one.")

        operation_group = dao.get_operationgroup_by_id(datatype_group.fk_operation_group)

        name1, values1, labels1, only_numbers1 = DiscretePSEAdapter.prepare_range_labels(operation_group,
                                                                                         operation_group.range1)
        name2, values2, labels2, only_numbers2 = DiscretePSEAdapter.prepare_range_labels(operation_group,
                                                                                         operation_group.range2)

        pse_context = ContextDiscretePSE(datatype_group_gid, color_metric, size_metric, back_page)
        pse_context.setRanges(name1, values1, labels1, name2, values2, labels2,
                              only_numbers1 and only_numbers2)
        final_dict = {}
        operations = dao.get_operations_in_group(operation_group.id)

        fake_numbers1 = dict(zip(values1, range(len(list(values1)))))
        fake_numbers2 = dict(zip(values2, range(len(list(values2)))))

        for operation_ in operations:
            if not operation_.has_finished:
                pse_context.has_started_ops = True
            range_values = eval(operation_.range_values)
            key_1 = DiscretePSEAdapter.get_value_on_axe(range_values, only_numbers1, name1, fake_numbers1)
            key_2 = DiscretePSEAdapter.get_value_on_axe(range_values, only_numbers2, name2, fake_numbers2)

            datatype = None
            if operation_.status == model.STATUS_FINISHED:
                datatypes = dao.get_results_for_operation(operation_.id)
                if len(datatypes) > 0:
                    datatype = datatypes[0]
                    if datatype.type == "DatatypeMeasure":
                        ## Load proper entity class from DB.
                        measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
                    else:
                        measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
                    pse_context.prepare_metrics_datatype(measures, datatype)

            if key_1 not in final_dict:
                final_dict[key_1] = {}

            final_dict[key_1][key_2] = pse_context.build_node_info(operation_, datatype)

        pse_context.fill_object(final_dict)
        ## datatypes_dict is not actually used in the drawing of the PSE and actually
        ## causes problems in case of NaN values, so just remove it before creating the json
        pse_context.datatypes_dict = {}
        if not only_numbers1:
            pse_context.values_x = range(len(list(values1)))
        if not only_numbers2:
            pse_context.values_y = range(len(list(values2)))
        return pse_context
Ejemplo n.º 24
0
    def retrieve_project_full(self, project_id, applied_filters=None, current_page=1):
        """
        Return a Tuple with Project entity and Operations for current Project.
        :param project_id: Current Project Identifier
        :param applied_filters: Filters to apply on Operations
        :param current_page: Number for current page in operations
        """
        selected_project = self.find_project(project_id)
        total_filtered = self.count_filtered_operations(project_id, applied_filters)
        pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
        total_ops_nr = self.count_filtered_operations(project_id)

        start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
        current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE)
        if current_ops is None:
            return selected_project, 0, [], 0

        operations = []
        view_categ_id = dao.get_visualisers_categories()[0].id
        for one_op in current_ops:
            try:
                result = {}
                if one_op[0] != one_op[1]:
                    result["id"] = str(one_op[0]) + "-" + str(one_op[1])
                else:
                    result["id"] = str(one_op[0])
                burst = dao.get_burst_for_operation_id(one_op[0])
                result["burst_name"] = burst.name if burst else '-'
                result["count"] = one_op[2]
                result["gid"] = one_op[13]
                if one_op[3] is not None and one_op[3]:
                    try:
                        operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0]
                        result["group"] = operation_group.name
                        result["group"] = result["group"].replace("_", " ")
                        result["operation_group_id"] = operation_group.id
                        datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3])
                        result["datatype_group_gid"] = datatype_group.gid
                        result["gid"] = operation_group.gid
                        ## Filter only viewers for current DataTypeGroup entity:
                        result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid)
                    except Exception:
                        self.logger.exception("We will ignore group on entity:" + str(one_op))
                        result["datatype_group_gid"] = None
                else:
                    result['group'] = None
                    result['datatype_group_gid'] = None
                result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
                result["user"] = dao.get_user_by_id(one_op[5])
                if type(one_op[6]) is str:
                    result["create"] = string2date(str(one_op[6]))
                else:
                    result["create"] = one_op[6]
                if type(one_op[7]) is str:
                    result["start"] = string2date(str(one_op[7]))
                else:
                    result["start"] = one_op[7]
                if type(one_op[8]) is str:
                    result["complete"] = string2date(str(one_op[8]))
                else:
                    result["complete"] = one_op[8]

                if result["complete"] is not None and result["start"] is not None:
                    result["duration"] = format_timedelta(result["complete"] - result["start"])
                result["status"] = one_op[9]
                result["additional"] = one_op[10]
                result["visible"] = True if one_op[11] > 0 else False
                result['operation_tag'] = one_op[12]
                result['figures'] = None
                if not result['group']:
                    datatype_results = dao.get_results_for_operation(result['id'])
                    result['results'] = []
                    for dt in datatype_results:
                        dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid)
                        if dt_loaded:
                            result['results'].append(dt_loaded)
                        else:
                            self.logger.warning("Could not retrieve datatype %s" % str(dt))

                    operation_figures = dao.get_figures_for_operation(result['id'])

                    # Compute the full path to the figure / image on disk
                    for figure in operation_figures:
                        figures_folder = self.structure_helper.get_images_folder(figure.project.name)
                        figure_full_path = os.path.join(figures_folder, figure.file_path)
                        # Compute the path available from browser
                        figure.figure_path = utils.path2url_part(figure_full_path)

                    result['figures'] = operation_figures
                else:
                    result['results'] = None
                operations.append(result)
            except Exception:
                ## We got an exception when processing one Operation Row. We will continue with the rest of the rows.
                self.logger.exception("Could not prepare operation for display:" + str(one_op))
        return selected_project, total_ops_nr, operations, pages_no
    def plot(self, figure, operation_group, metric, range1_name, range2_name):
        """
        Do the plot for the given figure. Also need operation group, metric and ranges
        in order to compute the data to be plotted.
        """
        operations = dao.get_operations_in_group(operation_group.id)
        # Data from which to interpolate larger 2-D space
        apriori_x = numpy.array(self.range1)
        apriori_y = numpy.array(self.range2)
        apriori_data = numpy.zeros((apriori_x.size, apriori_y.size))

        # An 2D array of GIDs which is used later to launch overlay for a DataType
        datatypes_gids = [[None for _ in self.range2] for _ in self.range1]
        for operation_ in operations:
            range_values = eval(operation_.range_values)
            key_1 = range_values[range1_name]
            index_x = self.range1.index(key_1)
            key_2 = range_values[range2_name]
            index_y = self.range2.index(key_2)
            if operation_.status == model.STATUS_STARTED:
                raise LaunchException("Not all operations from this range are complete. Cannot view until then.")

            operation_results = dao.get_results_for_operation(operation_.id)
            if operation_results:
                datatype = operation_results[0]
                datatypes_gids[index_x][index_y] = datatype.gid
    
                if datatype.type == "DatatypeMeasure":
                    measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
                else:
                    measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
            else:
                datatypes_gids[index_x][index_y] = None
                measures = None

            if measures:
                apriori_data[index_x][index_y] = measures[0].metrics[metric]
            else:
                apriori_data[index_x][index_y] = numpy.NaN
            
        # Convert array to 0 but keep track of nan values so we can replace after interpolation
        # since interpolating with nan values will just break the whole process
        nan_indices = numpy.isnan(apriori_data)
        self.nan_indices[figure.number] = nan_indices
        apriori_data = numpy.nan_to_num(apriori_data)
        # Attempt order-3 interpolation.
        kx = ky = 3
        if len(self.range1) <= 3 or len(self.range2) <= 3:
            # Number of points is too small, just do linear interpolation
            kx = ky = 1
        s = interpolate.RectBivariateSpline(apriori_x, apriori_y, apriori_data, kx=kx, ky=ky)
        # Get data of higher resolution that we'll plot later on
        posteriori_x = numpy.arange(self.range1[0], self.range1[-1],
                                    (self.range1[-1] - self.range1[0]) / RESOLUTION[0])
        posteriori_y = numpy.arange(self.range2[0], self.range2[-1],
                                    (self.range2[-1] - self.range2[0]) / RESOLUTION[1])
        posteriori_data = s(posteriori_x, posteriori_y)
        x_granularity = RESOLUTION[0] / len(self.range1)
        y_granularity = RESOLUTION[1] / len(self.range2)
        for idx, row in enumerate(nan_indices):
            for idy, was_nan in enumerate(row):
                if was_nan:
                    # Now we want to set back all the values that were NaN before interpolation
                    # and keep track of the change in granularity. For this reason for each nan
                    # value we had before, we will now have a matrix of the shape [x_granularity x y_granularity]
                    # full of NaN values
                    start_x = idx * x_granularity
                    end_x = (idx + 1) * x_granularity
                    start_y = idy * y_granularity
                    end_y = (idy + 1) * y_granularity
                    for x_scaled in xrange(start_x, end_x, 1):
                        for y_scaled in xrange(start_y, end_y, 1):
                            posteriori_data[x_scaled, y_scaled] = numpy.NaN
        # Rotate to get good plot
        posteriori_data = numpy.rot90(posteriori_data)
        
        self.interp_models[figure.number] = s
        # Do actual plot.        
        axes = figure.gca()
        img = axes.imshow(posteriori_data, extent=(min(self.range1), max(self.range1),
                                                   min(self.range2), max(self.range2)),
                          aspect='auto', interpolation='nearest')
        axes.set_title("Interpolated values for metric %s" % (metric,))
        figure.colorbar(img)
        axes.set_xlabel(range1_name)
        axes.set_ylabel(range2_name)


        def format_coord(x, y):
            return 'x=%1.4f, y=%1.4f' % (x, y)


        axes.format_coord = format_coord
        return datatypes_gids
Ejemplo n.º 26
0
 def get_results_for_operation(operation_id, selected_filter=None):
     """
     Retrieve the DataTypes entities resulted after the execution of the given operation.
     """
     return dao.get_results_for_operation(operation_id, selected_filter)
Ejemplo n.º 27
0
    def retrieve_project_full(self,
                              project_id,
                              applied_filters=None,
                              current_page=1):
        """
        Return a Tuple with Project entity and Operations for current Project.
        :param project_id: Current Project Identifier
        :param applied_filters: Filters to apply on Operations
        :param current_page: Number for current page in operations
        """
        selected_project = self.find_project(project_id)
        total_filtered = self.count_filtered_operations(
            project_id, applied_filters)
        pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (
            1 if total_filtered % OPERATIONS_PAGE_SIZE else 0)
        total_ops_nr = self.count_filtered_operations(project_id)

        start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1)
        current_ops = dao.get_filtered_operations(project_id, applied_filters,
                                                  start_idx,
                                                  OPERATIONS_PAGE_SIZE)
        if current_ops is None:
            return selected_project, 0, [], 0

        operations = []
        for one_op in current_ops:
            try:
                result = {}
                if one_op[0] != one_op[1]:
                    result["id"] = str(one_op[0]) + "-" + str(one_op[1])
                else:
                    result["id"] = str(one_op[0])
                burst = dao.get_burst_for_operation_id(one_op[0])
                result["burst_name"] = burst.name if burst else '-'
                result["count"] = one_op[2]
                result["gid"] = one_op[13]
                operation_group_id = one_op[3]
                if operation_group_id is not None and operation_group_id:
                    try:
                        operation_group = dao.get_generic_entity(
                            OperationGroup, operation_group_id)[0]
                        result["group"] = operation_group.name
                        result["group"] = result["group"].replace("_", " ")
                        result["operation_group_id"] = operation_group.id
                        datatype_group = dao.get_datatypegroup_by_op_group_id(
                            operation_group_id)
                        result[
                            "datatype_group_gid"] = datatype_group.gid if datatype_group is not None else None
                        result["gid"] = operation_group.gid
                        # Filter only viewers for current DataTypeGroup entity:

                        if datatype_group is None:
                            view_groups = None
                        else:
                            view_groups = AlgorithmService(
                            ).get_visualizers_for_group(datatype_group.gid)
                        result["view_groups"] = view_groups
                    except Exception:
                        self.logger.exception(
                            "We will ignore group on entity:" + str(one_op))
                        result["datatype_group_gid"] = None
                else:
                    result['group'] = None
                    result['datatype_group_gid'] = None
                result["algorithm"] = dao.get_algorithm_by_id(one_op[4])
                result["user"] = dao.get_user_by_id(one_op[5])
                if type(one_op[6]) is str:
                    result["create"] = string2date(str(one_op[6]))
                else:
                    result["create"] = one_op[6]
                if type(one_op[7]) is str:
                    result["start"] = string2date(str(one_op[7]))
                else:
                    result["start"] = one_op[7]
                if type(one_op[8]) is str:
                    result["complete"] = string2date(str(one_op[8]))
                else:
                    result["complete"] = one_op[8]

                if result["complete"] is not None and result[
                        "start"] is not None:
                    result["duration"] = format_timedelta(result["complete"] -
                                                          result["start"])
                result["status"] = one_op[9]
                result["additional"] = one_op[10]
                result["visible"] = True if one_op[11] > 0 else False
                result['operation_tag'] = one_op[12]
                if not result['group']:
                    result['results'] = dao.get_results_for_operation(
                        result['id'])
                else:
                    result['results'] = None
                operations.append(result)
            except Exception:
                # We got an exception when processing one Operation Row. We will continue with the rest of the rows.
                self.logger.exception(
                    "Could not prepare operation for display:" + str(one_op))
        return selected_project, total_ops_nr, operations, pages_no