def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations(project_id, applied_filters) pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE) if current_ops is None: return selected_project, 0, [], 0 operations = [] view_categ_id = dao.get_visualisers_categories()[0].id for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[13] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid ## Filter only viewers for current DataTypeGroup entity: result["view_groups"] = FlowService().get_visualizers_for_group(datatype_group.gid) except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["user"] = dao.get_user_by_id(one_op[5]) if type(one_op[6]) is str: result["create"] = string2date(str(one_op[6])) else: result["create"] = one_op[6] if type(one_op[7]) is str: result["start"] = string2date(str(one_op[7])) else: result["start"] = one_op[7] if type(one_op[8]) is str: result["complete"] = string2date(str(one_op[8])) else: result["complete"] = one_op[8] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[9] result["additional"] = one_op[10] result["visible"] = True if one_op[11] > 0 else False result['operation_tag'] = one_op[12] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [] for dt in datatype_results: dt_loaded = ABCAdapter.load_entity_by_gid(dt.gid) if dt_loaded: result['results'].append(dt_loaded) else: self.logger.warning("Could not retrieve datatype %s" % str(dt)) operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: ## We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no
def retrieve_launchers(self, datatype_gid, inspect_group=False, include_categories=None): """ Returns all the available launch-able algorithms from the database. Filter the ones accepting as required input a specific DataType. :param datatype_gid: GID, to filter algorithms for this particular entity. :param inspect_group: TRUE if we are now in the inspection of sub-entities in a DataTypeGroup :param include_categories: List of categories to be included in the result. When None, all lanchable categories are included """ try: all_launch_categ = dao.get_launchable_categories() launch_categ = dict((categ.id, categ.displayname) for categ in all_launch_categ if include_categories is None or categ.id in include_categories) datatype_instance = dao.get_datatype_by_gid(datatype_gid) data_class = datatype_instance.__class__ all_compatible_classes = [data_class.__name__] for one_class in getmro(data_class): if issubclass(one_class, MappedType) and one_class.__name__ not in all_compatible_classes: all_compatible_classes.append(one_class.__name__) self.logger.debug("Searching in categories: " + str(len(launch_categ)) + " - " + str(launch_categ.keys()) + "-" + str(include_categories)) launchable_groups = dao.get_apliable_algo_groups(all_compatible_classes, launch_categ.keys()) to_remove = [] for one_group in launchable_groups: compatible_algorithms = [] for one_algo in one_group.children: filter_chain = FilterChain.from_json(one_algo.datatype_filter) if not filter_chain or filter_chain.get_python_filter_equivalent(datatype_instance): compatible_algorithms.append(one_algo) if len(compatible_algorithms) > 0: one_group.children = copy.deepcopy(compatible_algorithms) else: to_remove.append(one_group) for one_group in to_remove: launchable_groups.remove(one_group) del one_group launchers = ProjectService.__prepare_group_result(launchable_groups, launch_categ, inspect_group) if data_class.__name__ == model.DataTypeGroup.__name__: # If part of a group, update also with specific launchers of the child datatype dt_group = dao.get_datatype_group_by_gid(datatype_gid) datatypes = dao.get_datatypes_from_datatype_group(dt_group.id) if len(datatypes): datatype = datatypes[-1] datatype = dao.get_datatype_by_gid(datatype.gid) views_categ_id = dao.get_visualisers_categories()[0].id categories_for_small_type = [categ.id for categ in all_launch_categ if categ.id != views_categ_id and (include_categories is None or categ.id in include_categories)] if categories_for_small_type: specific_launchers = self.retrieve_launchers(datatype.gid, True, categories_for_small_type) for key in specific_launchers: if key in launchers: launchers[key].update(specific_launchers[key]) else: launchers[key] = specific_launchers[key] return launchers except Exception, excep: ProjectService().logger.exception(excep) ProjectService().logger.warning("Attempting to filter launcher for group despite exception!") return ProjectService.__prepare_group_result([], [], inspect_group)
def get_visualisers_category(): """Retrieve all Algorithm categories, with display capability""" result = dao.get_visualisers_categories() if result is None or len(result) < 1: raise Exception("View Category not found!!!") return result[0]
def get_visualisers_category(): """Retrieve all Algorithm categories, with display capability""" result = dao.get_visualisers_categories() if not result: raise ValueError("View Category not found!!!") return result[0]
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations(project_id, applied_filters) pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, OPERATIONS_PAGE_SIZE) if current_ops is None: return selected_project, 0, [], 0 operations = [] view_categ_id = dao.get_visualisers_categories()[0].id for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[14] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) result["datatype_group_gid"] = datatype_group.gid result["gid"] = operation_group.gid ## Filter only viewers for current DataTypeGroup entity: launcher = self.retrieve_launchers(datatype_group.gid, include_categories=[view_categ_id]).values()[0] view_groups = [] for launcher in launcher.values(): url = '/flow/' + str(launcher['category']) + '/' + str(launcher['id']) if launcher['part_of_group']: url = '/flow/prepare_group_launch/' + datatype_group.gid + '/' + \ str(launcher['category']) + '/' + str(launcher['id']) view_groups.append(dict(name=launcher["displayName"], url=url, param_name=launcher['children'][0]['param_name'], part_of_group=launcher['part_of_group'])) result["view_groups"] = view_groups except Exception: self.logger.exception("We will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["method"] = one_op[5] result["user"] = dao.get_user_by_id(one_op[6]) if type(one_op[7]) in (str, unicode): result["create"] = string2date(str(one_op[7])) else: result["create"] = one_op[7] if type(one_op[8]) in (str, unicode): result["start"] = string2date(str(one_op[8])) else: result["start"] = one_op[8] if type(one_op[9]) in (str, unicode): result["complete"] = string2date(str(one_op[9])) else: result["complete"] = one_op[9] if result["complete"] is not None and result["start"] is not None: result["duration"] = format_timedelta(result["complete"] - result["start"]) result["status"] = one_op[10] result["additional"] = one_op[11] result["visible"] = True if one_op[12] > 0 else False result['operation_tag'] = one_op[13] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type, dt.gid, 'gid')[0] for dt in datatype_results] operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result) except Exception: ## We got an exception when processing one Operation Row. We will continue with the rest of the rows. self.logger.exception("Could not prepare operation for display:" + str(one_op)) return selected_project, total_ops_nr, operations, pages_no
def get_visualisers_category(): """Retrieve all Algorithm categories, with display capability""" result = dao.get_visualisers_categories() if result is None or len(result) < 1: raise Exception("View Category not found!!!") return result[0]
def get_visualizers_for_group(self, dt_group_gid): categories = dao.get_visualisers_categories() return self._get_launchable_algorithms(dt_group_gid, categories)[1]
def get_visualisers_category(): """Retrieve all Algorithm categories, with display capability""" result = dao.get_visualisers_categories() if not result: raise ValueError("View Category not found!!!") return result[0]
def get_visualizers_for_group(self, dt_group_gid): categories = dao.get_visualisers_categories() return self._get_launchable_algorithms(dt_group_gid, categories)[1]
def retrieve_launchers(self, dataname, datatype_gid=None, inspect_group=False, exclude_categories=None): """ Returns all the available launch-able algorithms from the database. Filter the ones accepting as required input a specific DataType. :param dataname: String or class representing DataType to retrieve filters for it. :param datatype_gid: Optional GID, to filter algorithms for this particular entity. :param inspect_group: TRUE if we are now in the inspection of sub-entities in a DataTypeGroup :param exclude_categories: List of categories to be excluded from the result. """ if exclude_categories is None: exclude_categories = [] launch_categ = dao.get_launchable_categories() launch_categ = dict((categ.id, categ.displayname) for categ in launch_categ if categ.id not in exclude_categories) launch_groups = dao.get_apliable_algo_groups(dataname, launch_categ.keys()) if datatype_gid is None: return ProjectService.__prepare_group_result(launch_groups, launch_categ, inspect_group) try: datatype_instance = dao.get_datatype_by_gid(datatype_gid) data_class = datatype_instance.__class__ for one_class in data_class.__bases__: launch_groups.extend(dao.get_apliable_algo_groups(one_class.__name__, launch_categ.keys())) specific_datatype = dao.get_generic_entity(data_class, datatype_gid, "gid") to_remove = [] for one_group in launch_groups: valid_algorithms = [] for one_algo in one_group.children: filter_chain = FilterChain.from_json(one_algo.datatype_filter) if not filter_chain or filter_chain.get_python_filter_equivalent(specific_datatype[0]): valid_algorithms.append(one_algo) if len(valid_algorithms) > 0: one_group.children = copy.deepcopy(valid_algorithms) else: to_remove.append(one_group) for one_group in to_remove: launch_groups.remove(one_group) del one_group launchers = ProjectService.__prepare_group_result(launch_groups, launch_categ, inspect_group) if dataname == model.DataTypeGroup.__name__: # If part of a group, update also with specific launchers of that datatype dt_group = dao.get_datatype_group_by_gid(datatype_gid) datatypes = dao.get_datatypes_from_datatype_group(dt_group.id) if len(datatypes): datatype = datatypes[-1] datatype = dao.get_datatype_by_gid(datatype.gid) views_categ_id = dao.get_visualisers_categories()[0].id specific_launchers = self.retrieve_launchers(datatype.__class__.__name__, datatype.gid, True, [views_categ_id] + exclude_categories) for key in specific_launchers: if key in launchers: launchers[key].update(specific_launchers[key]) else: launchers[key] = specific_launchers[key] return launchers except Exception, excep: ProjectService().logger.exception(excep) ProjectService().logger.warning("Attempting to filter launcher for group despite exception!") return ProjectService.__prepare_group_result(launch_groups, launch_categ, inspect_group)
def retrieve_project_full(self, project_id, applied_filters=None, current_page=1): """ Return a Tuple with Project entity and Operations for current Project. :param project_id: Current Project Identifier :param applied_filters: Filters to apply on Operations :param current_page: Number for current page in operations """ selected_project = self.find_project(project_id) total_filtered = self.count_filtered_operations(project_id, applied_filters) start_idx = OPERATIONS_PAGE_SIZE * (current_page - 1) if total_filtered >= start_idx + OPERATIONS_PAGE_SIZE: end_idx = OPERATIONS_PAGE_SIZE else: end_idx = total_filtered - start_idx pages_no = total_filtered // OPERATIONS_PAGE_SIZE + (1 if total_filtered % OPERATIONS_PAGE_SIZE else 0) total_ops_nr = self.count_filtered_operations(project_id) current_ops = dao.get_filtered_operations(project_id, applied_filters, start_idx, end_idx) started_ops = 0 if current_ops is None: return selected_project, [], 0 operations = [] for one_op in current_ops: try: result = {} if one_op[0] != one_op[1]: result["id"] = str(one_op[0]) + "-" + str(one_op[1]) else: result["id"] = str(one_op[0]) burst = dao.get_burst_for_operation_id(one_op[0]) result["burst_name"] = burst.name if burst else '-' result["count"] = one_op[2] result["gid"] = one_op[14] if one_op[3] is not None and one_op[3]: try: operation_group = dao.get_generic_entity(model.OperationGroup, one_op[3])[0] result["group"] = operation_group.name result["group"] = result["group"].replace("_", " ") result["operation_group_id"] = operation_group.id datatype_group = dao.get_datatypegroup_by_op_group_id(one_op[3]) datatype = dao.get_datatype_by_id(datatype_group.id) result["datatype_group_gid"] = datatype.gid result["gid"] = operation_group.gid all_categs = dao.get_algorithm_categories() view_categ = dao.get_visualisers_categories()[0] excludes = [categ.id for categ in all_categs if categ.id != view_categ.id] algo = self.retrieve_launchers("DataTypeGroup", datatype.gid, exclude_categories=excludes).values()[0] view_groups = [] for algo in algo.values(): url = '/flow/' + str(algo['category']) + '/' + str(algo['id']) if algo['part_of_group']: url = '/flow/prepare_group_launch/' + datatype.gid + '/' + \ str(algo['category']) + '/' + str(algo['id']) view_groups.append(dict(name=algo["displayName"], url=url, param_name=algo['children'][0]['param_name'], part_of_group=algo['part_of_group'])) result["view_groups"] = view_groups except Exception, excep: self.logger.error(excep) self.logger.warning("Will ignore group on entity:" + str(one_op)) result["datatype_group_gid"] = None else: result['group'] = None result['datatype_group_gid'] = None result["algorithm"] = dao.get_algorithm_by_id(one_op[4]) result["method"] = one_op[5] result["user"] = dao.get_user_by_id(one_op[6]) if type(one_op[7]) in (str, unicode): result["create"] = string2date(str(one_op[7])) else: result["create"] = one_op[7] if type(one_op[8]) in (str, unicode): result["start"] = string2date(str(one_op[8])) else: result["start"] = one_op[8] if type(one_op[9]) in (str, unicode): result["complete"] = string2date(str(one_op[9])) else: result["complete"] = one_op[9] if result["complete"] is not None and result["start"] is not None: result["duration"] = timedelta2string(result["complete"] - result["start"]) result["status"] = one_op[10] if result["status"] == model.STATUS_STARTED: started_ops += 1 result["additional"] = one_op[11] result["visible"] = True if one_op[12] > 0 else False result['operation_tag'] = one_op[13] result['figures'] = None if not result['group']: datatype_results = dao.get_results_for_operation(result['id']) result['results'] = [dao.get_generic_entity(dt.module + '.' + dt.type, dt.gid, 'gid')[0] for dt in datatype_results] operation_figures = dao.get_figures_for_operation(result['id']) # Compute the full path to the figure / image on disk for figure in operation_figures: figures_folder = self.structure_helper.get_images_folder(figure.project.name, figure.operation.id) figure_full_path = os.path.join(figures_folder, figure.file_path) # Compute the path available from browser figure.figure_path = utils.path2url_part(figure_full_path) result['figures'] = operation_figures else: result['results'] = None operations.append(result)
def retrieve_launchers(self, datatype_gid, inspect_group=False, include_categories=None): """ Returns all the available launch-able algorithms from the database. Filter the ones accepting as required input a specific DataType. :param datatype_gid: GID, to filter algorithms for this particular entity. :param inspect_group: TRUE if we are now in the inspection of sub-entities in a DataTypeGroup :param include_categories: List of categories to be included in the result. When None, all lanchable categories are included """ try: all_launch_categ = dao.get_launchable_categories() launch_categ = dict((categ.id, categ.displayname) for categ in all_launch_categ if include_categories is None or categ.id in include_categories) datatype_instance = dao.get_datatype_by_gid(datatype_gid) data_class = datatype_instance.__class__ all_compatible_classes = [data_class.__name__] for one_class in getmro(data_class): if issubclass(one_class, MappedType) and one_class.__name__ not in all_compatible_classes: all_compatible_classes.append(one_class.__name__) self.logger.debug("Searching in categories: " + str(len(launch_categ)) + " - " + str(launch_categ.keys()) + "-" + str(include_categories)) launchable_groups = dao.get_apliable_algo_groups(all_compatible_classes, launch_categ.keys()) to_remove = [] for one_group in launchable_groups: compatible_algorithms = [] for one_algo in one_group.children: filter_chain = FilterChain.from_json(one_algo.datatype_filter) if not filter_chain or filter_chain.get_python_filter_equivalent(datatype_instance): compatible_algorithms.append(one_algo) if len(compatible_algorithms) > 0: one_group.children = copy.deepcopy(compatible_algorithms) else: to_remove.append(one_group) for one_group in to_remove: launchable_groups.remove(one_group) del one_group launchers = ProjectService.__prepare_group_result(launchable_groups, launch_categ, inspect_group) if data_class.__name__ == model.DataTypeGroup.__name__: # If part of a group, update also with specific launchers of the child datatype dt_group = dao.get_datatype_group_by_gid(datatype_gid) datatypes = dao.get_datatypes_from_datatype_group(dt_group.id) if len(datatypes): datatype = datatypes[-1] datatype = dao.get_datatype_by_gid(datatype.gid) views_categ_id = dao.get_visualisers_categories()[0].id categories_for_small_type = [categ.id for categ in all_launch_categ if categ.id != views_categ_id and (include_categories is None or categ.id in include_categories)] if categories_for_small_type: specific_launchers = self.retrieve_launchers(datatype.gid, True, categories_for_small_type) for key in specific_launchers: if key in launchers: launchers[key].update(specific_launchers[key]) else: launchers[key] = specific_launchers[key] return launchers except Exception, excep: ProjectService().logger.exception(excep) ProjectService().logger.warning("Attempting to filter launcher for group despite exception!") return ProjectService.__prepare_group_result([], [], inspect_group)