def _init_refinements(): """Init the refinements.""" try: # Get global template. template = template_registry_api.get_current_registry_template() # Init. refinement.init_refinements(template) except Exception as e: logger.error("Impossible to init the refinements: {0}".format(str(e)))
def __init__(self, *args, **kwargs): super(RefinementForm, self).__init__(*args, **kwargs) # Get global template. template = template_registry_api.get_current_registry_template() # Get refinements. refinements = refinement_api.get_all_filtered_by_template_hash( template.hash) for refinement in refinements: categories = category_api.get_all_filtered_by_refinement_id( refinement.id) self.fields[refinement.slug] = forms.ModelMultipleChoiceField( queryset=categories, required=False, label=refinement.name, widget=FancyTreeWidget(queryset=categories, count_mode=True)) self.fields[refinement.slug].has_selected_values = \ kwargs.get('data').get("{0}-{1}".format(self.prefix, refinement.slug)) is not None
def build_count(self): """ Count the number of records corresponding to each refinement. Returns: """ # Get global template. template = template_registry_api.get_current_registry_template() # Get refinements. refinements = refinement_api.get_all_filtered_by_template_hash(template.hash) # For each refinement for refinement in refinements: data_sources_res = [] # Get categories categories = category_api.get_all().filter(refinement=refinement) # Prepare pipeline self._prepare_pipeline_categories(categories) # Only selected data provider # FIXME: Decouple data source. for data_source in self.query.data_sources: # find local data source if data_source.name == LOCAL_QUERY_NAME: self._get_local_data(data_source, data_sources_res) # OAI-PMH elif data_source.url_query.endswith(reverse( "core_explore_oaipmh_rest_execute_query")): self._get_oai_data(data_source, data_sources_res) # Not supported else: logger.info("No treatment available for the data source {0}, " "{1). Counters will not take into account this data " "source.".format(data_source.name, data_source.url_query)) # Create a map to group the results from the data sources by category id res_map = defaultdict(list) # Formatting results for elt in data_sources_res: res_map[elt.get(self.id_key)].extend(elt.get(self.ids_key)) self._build_results(categories, res_map)
def _render_module(self, request): # get the xml path of the element on which the module is placed xml_xpath = request.GET.get('xml_xpath', None) # xml path is provided if xml_xpath is not None: try: # create unique field id from xpath field_id = re.sub('[/.:\[\]]', '', xml_xpath) # split the xpath split_xml_xpath = xml_xpath.split('/') # get the last element of the xpath xml_element = split_xml_xpath[-1] # check if namespace is present if ":" in xml_element: # split element name split_xml_element = xml_element.split(":") # only keep element name if namespace is present xml_element = split_xml_element[-1] # get registry template template = template_registry_api.get_current_registry_template( ) # get all refinements for this template refinements = refinement_api.get_all_filtered_by_template_hash( template.hash) # get the refinement for the xml element refinement = refinements.get(xsd_name=xml_element) # initialize reload data for the form reload_form_data = {} # data to reload were provided if self.data != '': # build filed for the refinement form for the current xml element refinement_form_field = "{0}-{1}".format( RefinementForm.prefix, field_id) # get the categories for the current refinement categories = category_api.get_all_filtered_by_refinement_id( refinement.id) # Initialize list of categories id reload_categories_id_list = [] # load list of data to reload from XML reload_data = XSDTree.fromstring("<root>" + self.data + "</root>") # Iterate xml elements for reload_data_element in list(reload_data): try: if len(reload_data_element) > 0: # The xml element to be reloaded is the child element child = reload_data_element[0] # get its value selected_value = child.text # find the corresponding category and add its id to the list reload_categories_id_list.append( categories.get(value=selected_value).id) except Exception, e: raise ModuleError( "Something went wrong when reloading data from XML." + e.message) # set data to reload in the form reload_form_data[ refinement_form_field] = reload_categories_id_list return AbstractModule.render_template( 'core_module_fancy_tree_registry_app/fancy_tree.html', { 'form': RefinementForm(refinement=refinement, field_id=field_id, data=reload_form_data) }) except Exception, e: raise ModuleError( "Something went wrong when rendering the module: " + e.message)
def tiles(request): """ :param request: :return: """ from django.conf import settings installed_apps = settings.INSTALLED_APPS context = {"tiles": []} if "core_explore_keyword_registry_app" in installed_apps: from core_explore_keyword_registry_app.views.user.forms import RefinementForm from core_explore_common_app.components.query import api as query_api from core_explore_common_app.components.query.models import Query from core_explore_common_app.views.user.ajax import add_local_data_source from core_main_registry_app.components.refinement import api as refinement_api from core_main_registry_app.components.category import api as category_api from core_main_registry_app.components.template import ( api as template_registry_api, ) from core_main_registry_app.components.custom_resource import ( api as custom_resource_api, ) from core_main_app.commons import exceptions as exceptions # create Query query = Query(user_id=str(request.user.id), templates=[]) # add local data source to the query add_local_data_source(request, query) # set visibility query_api.set_visibility_to_query(query) # upsert the query query_api.upsert(query) # add information in context to populate keyword form context.update( { "query_id": str(query.id), "user_id": query.user_id, "order_by_field": ",".join(DATA_SORTING_FIELDS), } ) try: # Get current template template = template_registry_api.get_current_registry_template() # Get type refinement refinement = refinement_api.get_by_template_hash_and_by_slug( template_hash=template.hash, slug="type" ) # Refinement form ID refinement_form_id = "{0}-{1}".format( RefinementForm.prefix, refinement.slug ) context["refinement_form_id"] = refinement_form_id # Shorter api name get_categories = ( category_api.get_all_categories_ids_from_name_and_refinement_id ) custom_resources = custom_resource_api.get_all_of_current_template().order_by( "sort" ) for custom_resource in custom_resources: if ( custom_resource.display_icon and custom_resource.role_type is not None ): tile = { "logo": custom_resource.icon, "color": custom_resource.icon_color, "categories": get_categories( custom_resource.role_type.split(":")[0], refinement.id ), "title": custom_resource.title, "text": "Click here to explore the {0}.".format( custom_resource.title ), } context["tiles"].append(tile) except (exceptions.DoesNotExist, exceptions.ModelError) as e: logger.error( "Error while getting information from the database: {0}".format(str(e)) ) except Exception as ex: logger.error( "Something wrong occurred during the tiles " "generation: {0}".format(str(ex)) ) return render(request, "nmrr_home/tiles.html", context)
def get_refinement_selected_values_from_query(query): """get the refinement selected values from a json query Args: query: Returns: { refinement_name: [cat_id, cat_id ,cat_id], refinement_name: [cat_id, cat_id ,cat_id], refinement_name: [cat_id, cat_id ,cat_id], } """ # create a list of key (category), value_list (selected values) category_values_list = {} return_value = {} # No "$and" means no refinement selected if "$and" not in query: return {} for element_or in query["$and"]: if "$or" not in element_or.keys(): continue # Go through all '$or' => where refinement are for element in element_or["$or"]: for key, value in list(element.items()): # Do not parse path ending with "#text" and parse only "$in" if key.endswith("#text") or "$in" not in value: continue for selected_value in value["$in"]: # Do not parse categories ending with '__category' if selected_value.endswith("__category"): continue # Build category value list if key in category_values_list: category_values_list[key].append(selected_value) else: category_values_list.update({key: [selected_value]}) # get global template. template = template_registry_api.get_current_registry_template() # get refinements. refinements = refinement_api.get_all_filtered_by_template_hash(template.hash) refinements_ids = [x.id for x in refinements] # get all category. q_list = [] for key, values in list(category_values_list.items()): # prepare the query q_list.append( Q(path=key) & Q(refinement_id__in=refinements_ids) & Q(value__in=values) ) if len(q_list) == 0: # No refinement found return return_value # if refinement are found, we build the structure categories = category_api.get_all().filter((reduce(operator.or_, q_list))) # now we have to build a list of {refinement name: category ids, } for category in categories: key = category.refinement.slug display_name = category.refinement.name if key in return_value: return_value[key][display_name].append( {"id": category.id, "value": category.value.split(":")[0]} ) else: return_value.update( {key: {display_name: [{"id": category.id, "value": category.value}]}} ) # return the structure return return_value
def _post(self, request): """Update the POST context Args: request: Returns: """ context = super(KeywordSearchRegistryView, self)._post(request) # get refinement form refinement_form = RefinementForm(data=request.POST, request=request) # get query_id and error from the context error = context.get("error", None) query_id = context.get("query_id", None) refinement_selected_types = [] # validate form, test if no errors occurred in the parent treatment and query_id exists if refinement_form.is_valid( ) and error is None and query_id is not None: try: query = query_api.get_by_id(query_id, request.user) content = json.loads(query.content) refinements = [] # Update content with status update_content_not_deleted_status_criteria(content) # get selected refinements (categories) for refinement_name, selected_categories in list( refinement_form.cleaned_data.items()): if len(selected_categories) > 0: # Add categories ids refinements.append([x.id for x in selected_categories]) # generate query if len(refinements) > 0: # get refinement query refinement_query = mongo_query_api.build_refinements_query( refinements) # if we have a refinement query if len(list(refinement_query.keys())) > 0: if ("$and" in content.keys() and "$and" in refinement_query.keys()): content["$and"] += refinement_query["$and"] elif ("$or" in content.keys() or "$text" in content.keys() ) and "$and" in refinement_query.keys(): new_content = refinement_query # copy the text or the search operators in the main $and for key, value in content.items(): if key == "$or" or key == "$text": key_pair = {} key_pair[key] = value new_content["$and"].append(key_pair) else: new_content[key] = value content = new_content else: content.update(refinement_query) # Update content query.content = json.dumps(content) # save query query_api.upsert(query, request.user) except DoesNotExist: error = "An unexpected error occurred while retrieving the query." context.update({"error": error}) except Exception as e: error = "An unexpected error occurred: {}.".format(str(e)) context.update({"error": error}) context.update({"refinement_form": refinement_form}) # get all categories which must be selected in the table if refinement_form.cleaned_data: # get the current template template = template_registry_api.get_current_registry_template( request=request) # get the refinement 'Type' refinement = refinement_api.get_by_template_hash_and_by_slug( template.hash, "type") # get the selected_types selected_types = refinement_form.cleaned_data.get( refinement.slug, None) # create the list of type if selected_types: refinement_selected_types = get_all_parent_name_from_category_list( selected_types) # create the list of category category_list = "" for key in refinement_form.cleaned_data: if len(refinement_form.cleaned_data[key]) > 0: category_list = "%s,%s|%s" % ( category_list, refinement_form.cleaned_data[key][0].refinement.name, key, ) context.update({"category_list": category_list}) # get all categories which must be selected in the table context.update( {"refinement_selected_types": refinement_selected_types}) # Custom registry self._update_context_with_custom_resources(context) return context
def _render_module(self, request): # get the xml path of the element on which the module is placed xml_xpath = request.GET.get("xml_xpath", None) # xml path is provided if xml_xpath is not None: try: # create unique field id from xpath field_id = re.sub("[/.:\[\]]", "", xml_xpath) # split the xpath split_xml_xpath = xml_xpath.split("/") # get the last element of the xpath xml_element = split_xml_xpath[-1] # check if namespace is present if ":" in xml_element: # split element name split_xml_element = xml_element.split(":") # only keep element name if namespace is present xml_element = split_xml_element[-1] # get registry template template = template_registry_api.get_current_registry_template( request=request) # get all refinements for this template refinements = refinement_api.get_all_filtered_by_template_hash( template.hash) # get the refinement for the xml element refinement = refinements.get(xsd_name=xml_element) # initialize reload data for the form reload_form_data = {} # data to reload were provided if self.data != "": # build filed for the refinement form for the current xml element refinement_form_field = "{0}-{1}".format( RefinementForm.prefix, field_id) # get the categories for the current refinement categories = category_api.get_all_filtered_by_refinement_id( refinement.id) # Initialize list of categories id reload_categories_id_list = [] # load list of data to reload from XML reload_data = XSDTree.fromstring("<root>" + self.data + "</root>") # Iterate xml elements for reload_data_element in list(reload_data): try: if len(reload_data_element) > 0: # The xml element to be reloaded is the child element child = reload_data_element[0] # get its value selected_value = child.text # find the corresponding category and add its id to the list category = categories.get(value=selected_value) # if the element is an unspecified element if category.slug.startswith(UNSPECIFIED_LABEL): # get the parent category selected_value += CATEGORY_SUFFIX category = categories.get( value=selected_value) reload_categories_id_list.append(category.id) except Exception as e: raise ModuleError( "Something went wrong when reloading data from XML." + str(e)) # set data to reload in the form reload_form_data[ refinement_form_field] = reload_categories_id_list return AbstractModule.render_template( "core_module_fancy_tree_registry_app/fancy_tree.html", { "form": RefinementForm( refinement=refinement, field_id=field_id, data=reload_form_data, ) }, ) except Exception as e: raise ModuleError( "Something went wrong when rendering the module: " + str(e)) else: raise ModuleError( "xml_xpath was not found in request GET parameters.")
def tiles(request): """ :param request: :return: """ from django.conf import settings installed_apps = settings.INSTALLED_APPS context = {"tiles": []} if "core_explore_keyword_registry_app" in installed_apps: from core_explore_keyword_registry_app.views.user.forms import RefinementForm from core_explore_common_app.components.query import api as query_api from core_explore_common_app.components.query.models import Query from core_explore_common_app.views.user.ajax import add_local_data_source from core_main_registry_app.components.refinement import api as refinement_api from core_main_registry_app.components.category import api as category_api from core_main_registry_app.components.template import api as template_registry_api from core_main_app.commons import exceptions as exceptions # create Query query = Query(user_id=str(request.user.id), templates=[]) query_api.upsert(query) # add local data source to the query add_local_data_source(request, query) # add information in context to populate keyword form context.update({"query_id": str(query.id), "user_id": query.user_id}) try: # Get current template template = template_registry_api.get_current_registry_template() # Get type refinement refinement = refinement_api.get_by_template_hash_and_by_slug( template_hash=template.hash, slug='type') # Refinement form ID refinement_form_id = "{0}-{1}".format(RefinementForm.prefix, refinement.slug) context["refinement_form_id"] = refinement_form_id # Shorter api name get_categories = category_api.get_all_categories_ids_by_parent_slug_and_refinement_id organizations_tile = { "logo": "fa-university", "color": "#2CAAE2", "categories": get_categories('organization', refinement.id), "title": "Organizations", "text": "Click here to explore the Organizations." } context["tiles"].append(organizations_tile) data_collections_tile = { "logo": "fa-table", "color": "#A1C057", "categories": get_categories('collection', refinement.id), "title": "Data Collections", "text": "Click here to explore the Data Collections." } context["tiles"].append(data_collections_tile) datasets_tile = { "logo": "fa-database", "color": "grey", "categories": get_categories('dataset', refinement.id), "title": "Datasets", "text": "Click here to explore the Datasets." } context["tiles"].append(datasets_tile) services_tile = { "logo": "fa-cogs", "color": "#EBB057;", "categories": get_categories('service', refinement.id), "title": "Services", "text": "Click here to explore the Services." } context["tiles"].append(services_tile) informational_tile = { "logo": "fa-laptop", "color": "#257ABC;", "categories": get_categories('web-site', refinement.id), "title": "Informational Sites", "text": "Click here to explore the Informational Sites." } context["tiles"].append(informational_tile) software_tile = { "logo": "fa-tachometer", "color": "#79B320;", "categories": get_categories('software', refinement.id), "title": "Software", "text": "Click here to explore the Software." } context["tiles"].append(software_tile) except (exceptions.DoesNotExist, exceptions.ModelError), e: logger.error( "Error while getting information from the database: {0}". format(e.message)) except Exception, ex: logger.error("Something wrong occurred during the tiles " "generation: {0}".format(ex.message))
def get_refinement_selected_values_from_query(query): """ get the refinement selected values from a json query Args: query: Returns: { refinement_name: [cat_id, cat_id ,cat_id], refinement_name: [cat_id, cat_id ,cat_id], refinement_name: [cat_id, cat_id ,cat_id], } """ # create a list of key (category), value_list (selected values) category_values_list = {} # if the query have the key '$and', there is refinement selected if '$and' in query: for element_or in query['$and']: # go through all '$or' => where refinement are for element in element_or['$or']: for key, value in element.iteritems(): # we only want path which does not contain '#text' at the end of it if not key.endswith('#text'): for selected_value in value['$in']: # then we build our list and we don't want categories ending with '__category' if not selected_value.endswith('__category'): if key in category_values_list: category_values_list[key].append( selected_value) else: category_values_list.update( {key: [selected_value]}) # get global template. template = template_registry_api.get_current_registry_template() # get refinements. refinements = refinement_api.get_all_filtered_by_template_hash( template.hash) refinements_ids = [x.id for x in refinements] # get all category. q_list = [] for key, values in category_values_list.iteritems(): # prepare the query q_list.append( Q(path=key) & Q(refinement_id__in=refinements_ids) & Q(value__in=values)) return_value = {} # if refinement are found, we build the structure if len(q_list) > 0: categories = category_api.get_all().filter( (reduce(operator.or_, q_list))) # now we have to build a list of {refinement name: category ids, } for category in categories: key = category.refinement.slug display_name = category.refinement.name if key in return_value: return_value[key][display_name].append({ "id": category.id, "value": category.value }) else: return_value.update({ key: { display_name: [{ "id": category.id, "value": category.value }] } }) # return the structure return return_value