def _init_refinements():
"""Init the refinements."""
try:
# Get global template.
template = template_registry_api.get_current_registry_template()
# Init.
refinement.init_refinements(template)
except Exception as e:
logger.error("Impossible to init the refinements: {0}".format(str(e)))
def __init__(self, *args, **kwargs):
super(RefinementForm, self).__init__(*args, **kwargs)
# Get global template.
template = template_registry_api.get_current_registry_template()
# Get refinements.
refinements = refinement_api.get_all_filtered_by_template_hash(
template.hash)
for refinement in refinements:
categories = category_api.get_all_filtered_by_refinement_id(
refinement.id)
self.fields[refinement.slug] = forms.ModelMultipleChoiceField(
queryset=categories,
required=False,
label=refinement.name,
widget=FancyTreeWidget(queryset=categories, count_mode=True))
self.fields[refinement.slug].has_selected_values = \
kwargs.get('data').get("{0}-{1}".format(self.prefix, refinement.slug)) is not None
def build_count(self):
""" Count the number of records corresponding to each refinement.
Returns:
"""
# Get global template.
template = template_registry_api.get_current_registry_template()
# Get refinements.
refinements = refinement_api.get_all_filtered_by_template_hash(template.hash)
# For each refinement
for refinement in refinements:
data_sources_res = []
# Get categories
categories = category_api.get_all().filter(refinement=refinement)
# Prepare pipeline
self._prepare_pipeline_categories(categories)
# Only selected data provider
# FIXME: Decouple data source.
for data_source in self.query.data_sources:
# find local data source
if data_source.name == LOCAL_QUERY_NAME:
self._get_local_data(data_source, data_sources_res)
# OAI-PMH
elif data_source.url_query.endswith(reverse(
"core_explore_oaipmh_rest_execute_query")):
self._get_oai_data(data_source, data_sources_res)
# Not supported
else:
logger.info("No treatment available for the data source {0}, "
"{1). Counters will not take into account this data "
"source.".format(data_source.name, data_source.url_query))
# Create a map to group the results from the data sources by category id
res_map = defaultdict(list)
# Formatting results
for elt in data_sources_res:
res_map[elt.get(self.id_key)].extend(elt.get(self.ids_key))
self._build_results(categories, res_map)
def _render_module(self, request):
# get the xml path of the element on which the module is placed
xml_xpath = request.GET.get('xml_xpath', None)
# xml path is provided
if xml_xpath is not None:
try:
# create unique field id from xpath
field_id = re.sub('[/.:\[\]]', '', xml_xpath)
# split the xpath
split_xml_xpath = xml_xpath.split('/')
# get the last element of the xpath
xml_element = split_xml_xpath[-1]
# check if namespace is present
if ":" in xml_element:
# split element name
split_xml_element = xml_element.split(":")
# only keep element name if namespace is present
xml_element = split_xml_element[-1]
# get registry template
template = template_registry_api.get_current_registry_template(
)
# get all refinements for this template
refinements = refinement_api.get_all_filtered_by_template_hash(
template.hash)
# get the refinement for the xml element
refinement = refinements.get(xsd_name=xml_element)
# initialize reload data for the form
reload_form_data = {}
# data to reload were provided
if self.data != '':
# build filed for the refinement form for the current xml element
refinement_form_field = "{0}-{1}".format(
RefinementForm.prefix, field_id)
# get the categories for the current refinement
categories = category_api.get_all_filtered_by_refinement_id(
refinement.id)
# Initialize list of categories id
reload_categories_id_list = []
# load list of data to reload from XML
reload_data = XSDTree.fromstring("<root>" + self.data +
"</root>")
# Iterate xml elements
for reload_data_element in list(reload_data):
try:
if len(reload_data_element) > 0:
# The xml element to be reloaded is the child element
child = reload_data_element[0]
# get its value
selected_value = child.text
# find the corresponding category and add its id to the list
reload_categories_id_list.append(
categories.get(value=selected_value).id)
except Exception, e:
raise ModuleError(
"Something went wrong when reloading data from XML."
+ e.message)
# set data to reload in the form
reload_form_data[
refinement_form_field] = reload_categories_id_list
return AbstractModule.render_template(
'core_module_fancy_tree_registry_app/fancy_tree.html', {
'form':
RefinementForm(refinement=refinement,
field_id=field_id,
data=reload_form_data)
})
except Exception, e:
raise ModuleError(
"Something went wrong when rendering the module: " +
e.message)
def tiles(request):
"""
:param request:
:return:
"""
from django.conf import settings
installed_apps = settings.INSTALLED_APPS
context = {"tiles": []}
if "core_explore_keyword_registry_app" in installed_apps:
from core_explore_keyword_registry_app.views.user.forms import RefinementForm
from core_explore_common_app.components.query import api as query_api
from core_explore_common_app.components.query.models import Query
from core_explore_common_app.views.user.ajax import add_local_data_source
from core_main_registry_app.components.refinement import api as refinement_api
from core_main_registry_app.components.category import api as category_api
from core_main_registry_app.components.template import (
api as template_registry_api,
)
from core_main_registry_app.components.custom_resource import (
api as custom_resource_api,
)
from core_main_app.commons import exceptions as exceptions
# create Query
query = Query(user_id=str(request.user.id), templates=[])
# add local data source to the query
add_local_data_source(request, query)
# set visibility
query_api.set_visibility_to_query(query)
# upsert the query
query_api.upsert(query)
# add information in context to populate keyword form
context.update(
{
"query_id": str(query.id),
"user_id": query.user_id,
"order_by_field": ",".join(DATA_SORTING_FIELDS),
}
)
try:
# Get current template
template = template_registry_api.get_current_registry_template()
# Get type refinement
refinement = refinement_api.get_by_template_hash_and_by_slug(
template_hash=template.hash, slug="type"
)
# Refinement form ID
refinement_form_id = "{0}-{1}".format(
RefinementForm.prefix, refinement.slug
)
context["refinement_form_id"] = refinement_form_id
# Shorter api name
get_categories = (
category_api.get_all_categories_ids_from_name_and_refinement_id
)
custom_resources = custom_resource_api.get_all_of_current_template().order_by(
"sort"
)
for custom_resource in custom_resources:
if (
custom_resource.display_icon
and custom_resource.role_type is not None
):
tile = {
"logo": custom_resource.icon,
"color": custom_resource.icon_color,
"categories": get_categories(
custom_resource.role_type.split(":")[0], refinement.id
),
"title": custom_resource.title,
"text": "Click here to explore the {0}.".format(
custom_resource.title
),
}
context["tiles"].append(tile)
except (exceptions.DoesNotExist, exceptions.ModelError) as e:
logger.error(
"Error while getting information from the database: {0}".format(str(e))
)
except Exception as ex:
logger.error(
"Something wrong occurred during the tiles "
"generation: {0}".format(str(ex))
)
return render(request, "nmrr_home/tiles.html", context)
def get_refinement_selected_values_from_query(query):
"""get the refinement selected values from a json query
Args:
query:
Returns:
{
refinement_name: [cat_id, cat_id ,cat_id],
refinement_name: [cat_id, cat_id ,cat_id],
refinement_name: [cat_id, cat_id ,cat_id],
}
"""
# create a list of key (category), value_list (selected values)
category_values_list = {}
return_value = {}
# No "$and" means no refinement selected
if "$and" not in query:
return {}
for element_or in query["$and"]:
if "$or" not in element_or.keys():
continue
# Go through all '$or' => where refinement are
for element in element_or["$or"]:
for key, value in list(element.items()):
# Do not parse path ending with "#text" and parse only "$in"
if key.endswith("#text") or "$in" not in value:
continue
for selected_value in value["$in"]:
# Do not parse categories ending with '__category'
if selected_value.endswith("__category"):
continue
# Build category value list
if key in category_values_list:
category_values_list[key].append(selected_value)
else:
category_values_list.update({key: [selected_value]})
# get global template.
template = template_registry_api.get_current_registry_template()
# get refinements.
refinements = refinement_api.get_all_filtered_by_template_hash(template.hash)
refinements_ids = [x.id for x in refinements]
# get all category.
q_list = []
for key, values in list(category_values_list.items()):
# prepare the query
q_list.append(
Q(path=key) & Q(refinement_id__in=refinements_ids) & Q(value__in=values)
)
if len(q_list) == 0: # No refinement found
return return_value
# if refinement are found, we build the structure
categories = category_api.get_all().filter((reduce(operator.or_, q_list)))
# now we have to build a list of {refinement name: category ids, }
for category in categories:
key = category.refinement.slug
display_name = category.refinement.name
if key in return_value:
return_value[key][display_name].append(
{"id": category.id, "value": category.value.split(":")[0]}
)
else:
return_value.update(
{key: {display_name: [{"id": category.id, "value": category.value}]}}
)
# return the structure
return return_value
def _post(self, request):
"""Update the POST context
Args:
request:
Returns:
"""
context = super(KeywordSearchRegistryView, self)._post(request)
# get refinement form
refinement_form = RefinementForm(data=request.POST, request=request)
# get query_id and error from the context
error = context.get("error", None)
query_id = context.get("query_id", None)
refinement_selected_types = []
# validate form, test if no errors occurred in the parent treatment and query_id exists
if refinement_form.is_valid(
) and error is None and query_id is not None:
try:
query = query_api.get_by_id(query_id, request.user)
content = json.loads(query.content)
refinements = []
# Update content with status
update_content_not_deleted_status_criteria(content)
# get selected refinements (categories)
for refinement_name, selected_categories in list(
refinement_form.cleaned_data.items()):
if len(selected_categories) > 0:
# Add categories ids
refinements.append([x.id for x in selected_categories])
# generate query
if len(refinements) > 0:
# get refinement query
refinement_query = mongo_query_api.build_refinements_query(
refinements)
# if we have a refinement query
if len(list(refinement_query.keys())) > 0:
if ("$and" in content.keys()
and "$and" in refinement_query.keys()):
content["$and"] += refinement_query["$and"]
elif ("$or" in content.keys()
or "$text" in content.keys()
) and "$and" in refinement_query.keys():
new_content = refinement_query
# copy the text or the search operators in the main $and
for key, value in content.items():
if key == "$or" or key == "$text":
key_pair = {}
key_pair[key] = value
new_content["$and"].append(key_pair)
else:
new_content[key] = value
content = new_content
else:
content.update(refinement_query)
# Update content
query.content = json.dumps(content)
# save query
query_api.upsert(query, request.user)
except DoesNotExist:
error = "An unexpected error occurred while retrieving the query."
context.update({"error": error})
except Exception as e:
error = "An unexpected error occurred: {}.".format(str(e))
context.update({"error": error})
context.update({"refinement_form": refinement_form})
# get all categories which must be selected in the table
if refinement_form.cleaned_data:
# get the current template
template = template_registry_api.get_current_registry_template(
request=request)
# get the refinement 'Type'
refinement = refinement_api.get_by_template_hash_and_by_slug(
template.hash, "type")
# get the selected_types
selected_types = refinement_form.cleaned_data.get(
refinement.slug, None)
# create the list of type
if selected_types:
refinement_selected_types = get_all_parent_name_from_category_list(
selected_types)
# create the list of category
category_list = ""
for key in refinement_form.cleaned_data:
if len(refinement_form.cleaned_data[key]) > 0:
category_list = "%s,%s|%s" % (
category_list,
refinement_form.cleaned_data[key][0].refinement.name,
key,
)
context.update({"category_list": category_list})
# get all categories which must be selected in the table
context.update(
{"refinement_selected_types": refinement_selected_types})
# Custom registry
self._update_context_with_custom_resources(context)
return context
def _render_module(self, request):
# get the xml path of the element on which the module is placed
xml_xpath = request.GET.get("xml_xpath", None)
# xml path is provided
if xml_xpath is not None:
try:
# create unique field id from xpath
field_id = re.sub("[/.:\[\]]", "", xml_xpath)
# split the xpath
split_xml_xpath = xml_xpath.split("/")
# get the last element of the xpath
xml_element = split_xml_xpath[-1]
# check if namespace is present
if ":" in xml_element:
# split element name
split_xml_element = xml_element.split(":")
# only keep element name if namespace is present
xml_element = split_xml_element[-1]
# get registry template
template = template_registry_api.get_current_registry_template(
request=request)
# get all refinements for this template
refinements = refinement_api.get_all_filtered_by_template_hash(
template.hash)
# get the refinement for the xml element
refinement = refinements.get(xsd_name=xml_element)
# initialize reload data for the form
reload_form_data = {}
# data to reload were provided
if self.data != "":
# build filed for the refinement form for the current xml element
refinement_form_field = "{0}-{1}".format(
RefinementForm.prefix, field_id)
# get the categories for the current refinement
categories = category_api.get_all_filtered_by_refinement_id(
refinement.id)
# Initialize list of categories id
reload_categories_id_list = []
# load list of data to reload from XML
reload_data = XSDTree.fromstring("<root>" + self.data +
"</root>")
# Iterate xml elements
for reload_data_element in list(reload_data):
try:
if len(reload_data_element) > 0:
# The xml element to be reloaded is the child element
child = reload_data_element[0]
# get its value
selected_value = child.text
# find the corresponding category and add its id to the list
category = categories.get(value=selected_value)
# if the element is an unspecified element
if category.slug.startswith(UNSPECIFIED_LABEL):
# get the parent category
selected_value += CATEGORY_SUFFIX
category = categories.get(
value=selected_value)
reload_categories_id_list.append(category.id)
except Exception as e:
raise ModuleError(
"Something went wrong when reloading data from XML."
+ str(e))
# set data to reload in the form
reload_form_data[
refinement_form_field] = reload_categories_id_list
return AbstractModule.render_template(
"core_module_fancy_tree_registry_app/fancy_tree.html",
{
"form":
RefinementForm(
refinement=refinement,
field_id=field_id,
data=reload_form_data,
)
},
)
except Exception as e:
raise ModuleError(
"Something went wrong when rendering the module: " +
str(e))
else:
raise ModuleError(
"xml_xpath was not found in request GET parameters.")
def tiles(request):
"""
:param request:
:return:
"""
from django.conf import settings
installed_apps = settings.INSTALLED_APPS
context = {"tiles": []}
if "core_explore_keyword_registry_app" in installed_apps:
from core_explore_keyword_registry_app.views.user.forms import RefinementForm
from core_explore_common_app.components.query import api as query_api
from core_explore_common_app.components.query.models import Query
from core_explore_common_app.views.user.ajax import add_local_data_source
from core_main_registry_app.components.refinement import api as refinement_api
from core_main_registry_app.components.category import api as category_api
from core_main_registry_app.components.template import api as template_registry_api
from core_main_app.commons import exceptions as exceptions
# create Query
query = Query(user_id=str(request.user.id), templates=[])
query_api.upsert(query)
# add local data source to the query
add_local_data_source(request, query)
# add information in context to populate keyword form
context.update({"query_id": str(query.id), "user_id": query.user_id})
try:
# Get current template
template = template_registry_api.get_current_registry_template()
# Get type refinement
refinement = refinement_api.get_by_template_hash_and_by_slug(
template_hash=template.hash, slug='type')
# Refinement form ID
refinement_form_id = "{0}-{1}".format(RefinementForm.prefix,
refinement.slug)
context["refinement_form_id"] = refinement_form_id
# Shorter api name
get_categories = category_api.get_all_categories_ids_by_parent_slug_and_refinement_id
organizations_tile = {
"logo": "fa-university",
"color": "#2CAAE2",
"categories": get_categories('organization', refinement.id),
"title": "Organizations",
"text": "Click here to explore the Organizations."
}
context["tiles"].append(organizations_tile)
data_collections_tile = {
"logo": "fa-table",
"color": "#A1C057",
"categories": get_categories('collection', refinement.id),
"title": "Data Collections",
"text": "Click here to explore the Data Collections."
}
context["tiles"].append(data_collections_tile)
datasets_tile = {
"logo": "fa-database",
"color": "grey",
"categories": get_categories('dataset', refinement.id),
"title": "Datasets",
"text": "Click here to explore the Datasets."
}
context["tiles"].append(datasets_tile)
services_tile = {
"logo": "fa-cogs",
"color": "#EBB057;",
"categories": get_categories('service', refinement.id),
"title": "Services",
"text": "Click here to explore the Services."
}
context["tiles"].append(services_tile)
informational_tile = {
"logo": "fa-laptop",
"color": "#257ABC;",
"categories": get_categories('web-site', refinement.id),
"title": "Informational Sites",
"text": "Click here to explore the Informational Sites."
}
context["tiles"].append(informational_tile)
software_tile = {
"logo": "fa-tachometer",
"color": "#79B320;",
"categories": get_categories('software', refinement.id),
"title": "Software",
"text": "Click here to explore the Software."
}
context["tiles"].append(software_tile)
except (exceptions.DoesNotExist, exceptions.ModelError), e:
logger.error(
"Error while getting information from the database: {0}".
format(e.message))
except Exception, ex:
logger.error("Something wrong occurred during the tiles "
"generation: {0}".format(ex.message))
def get_refinement_selected_values_from_query(query):
""" get the refinement selected values from a json query
Args:
query:
Returns:
{
refinement_name: [cat_id, cat_id ,cat_id],
refinement_name: [cat_id, cat_id ,cat_id],
refinement_name: [cat_id, cat_id ,cat_id],
}
"""
# create a list of key (category), value_list (selected values)
category_values_list = {}
# if the query have the key '$and', there is refinement selected
if '$and' in query:
for element_or in query['$and']:
# go through all '$or' => where refinement are
for element in element_or['$or']:
for key, value in element.iteritems():
# we only want path which does not contain '#text' at the end of it
if not key.endswith('#text'):
for selected_value in value['$in']:
# then we build our list and we don't want categories ending with '__category'
if not selected_value.endswith('__category'):
if key in category_values_list:
category_values_list[key].append(
selected_value)
else:
category_values_list.update(
{key: [selected_value]})
# get global template.
template = template_registry_api.get_current_registry_template()
# get refinements.
refinements = refinement_api.get_all_filtered_by_template_hash(
template.hash)
refinements_ids = [x.id for x in refinements]
# get all category.
q_list = []
for key, values in category_values_list.iteritems():
# prepare the query
q_list.append(
Q(path=key) & Q(refinement_id__in=refinements_ids)
& Q(value__in=values))
return_value = {}
# if refinement are found, we build the structure
if len(q_list) > 0:
categories = category_api.get_all().filter(
(reduce(operator.or_, q_list)))
# now we have to build a list of {refinement name: category ids, }
for category in categories:
key = category.refinement.slug
display_name = category.refinement.name
if key in return_value:
return_value[key][display_name].append({
"id": category.id,
"value": category.value
})
else:
return_value.update({
key: {
display_name: [{
"id": category.id,
"value": category.value
}]
}
})
# return the structure
return return_value
