def _get(self, user, query_id):
""" Update the GET context
Args:
user:
query_id:
Returns:
"""
context = super(KeywordSearchRegistryView, self)._get(user, query_id)
data_form = {}
refinement_selected_types = []
category_list = ""
query_id = context.get("query_id", None)
# in case we don't find the query id in the context
# a default query is created by inheritance
if query_id is None:
raise ExploreRequestError("query id is missing")
try:
# get the query id
query = query_api.get_by_id(query_id)
# here we have to make sure to set the visibility and status criteria
# set visibility
set_visibility_to_query(query)
# load content
content = json.loads(query.content)
# update content with status
update_content_not_deleted_status_criteria(content)
query.content = json.dumps(content)
# save query
query_api.upsert(query)
# get all keywords back
refinement_selected_values = mongo_query_api.get_refinement_selected_values_from_query(
content)
# build the data_form structure
for key in refinement_selected_values:
for display_name in refinement_selected_values[key]:
list_element = refinement_selected_values[key][
display_name]
data_form.update({
RefinementForm.prefix + '-' + key:
[element["id"] for element in list_element]
})
refinement_selected_types = [
element["value"] for element in list_element
]
# create the list of category
if len(refinement_selected_values[key]) > 0:
category_list = "%s,%s|%s" % (category_list,
display_name, key)
context.update({'category_list': category_list})
except Exception, e:
context.update({
'error':
"An unexpected error occurred while loading the query: {}.".
format(e.message)
})
def _get(self, request, query_id):
""" Prepare the GET context
Args:
user:
query_id:
Returns:
"""
error = None
display_persistent_query_button = True
if query_id is None:
# create query
query = create_default_query(request, [])
# upsert the query
query_api.upsert(query)
# create keyword form
# create all data for select values in forms
keywords_data_form = {
'query_id': str(query.id),
'user_id': query.user_id,
}
else:
try:
# get the query id
query = query_api.get_by_id(query_id)
user_id = query.user_id
# get all keywords back
query_json = json.loads(query.content)
keywords = None
if '$text' in query_json:
keywords = query_json['$text']['$search'].replace(
" ", ",").replace('"', '')
# get all version managers
version_managers = []
for template in query.templates:
version_managers.append(
str(version_manager_api.get_from_version(template).id))
# create all data for select values in forms
keywords_data_form = {
'query_id': str(query.id),
'user_id': user_id,
'keywords': keywords,
'global_templates': version_managers,
'user_templates': version_managers
}
except Exception, e:
error = "An unexpected error occurred while loading the query: {}.".format(
e.message)
return {'error': error}
def _post(self, request):
""" Prepare the POST context
Args:
request:
Returns:
"""
error = None
warning = None
search_form = KeywordForm(data=request.POST)
display_persistent_query_button = False
# validate form
if search_form.is_valid():
try:
# get form values
query_id = search_form.cleaned_data.get('query_id', None)
keywords = search_form.cleaned_data.get('keywords', None)
global_templates = search_form.cleaned_data.get(
'global_templates', [])
user_templates = search_form.cleaned_data.get(
'user_templates', [])
# get all template version manager ids
template_version_manager_ids = global_templates + user_templates
# from ids, get all version manager
version_manager_list = version_manager_api.get_by_id_list(
template_version_manager_ids)
# from all version manager, build a list of all version (template)
template_ids = []
map(lambda x: template_ids.extend(x.versions),
version_manager_list)
if query_id is None or keywords is None:
error = "Expected parameters are not provided"
else:
# get query
query = query_api.get_by_id(query_id)
if len(query.data_sources) == 0:
warning = "Please select at least 1 data source."
else:
# update query
query.templates = template_api.get_all_by_id_list(
template_ids)
query.content = json.dumps(
get_full_text_query(keywords))
query_api.upsert(query)
display_persistent_query_button = True
except DoesNotExist:
error = "An unexpected error occurred while retrieving the query."
except Exception, e:
error = "An unexpected error occurred: {}.".format(e.message)
def _post(self, request):
""" Update the POST context
Args:
request:
Returns:
"""
context = super(KeywordSearchRegistryView, self)._post(request)
# get refinement form
refinement_form = RefinementForm(data=request.POST)
# get query_id and error from the context
error = context.get("error", None)
query_id = context.get("query_id", None)
# validate form, test if no errors occurred in the parent treatment and query_id exists
if refinement_form.is_valid(
) and error is None and query_id is not None:
try:
query = query_api.get_by_id(query_id)
content = json.loads(query.content)
refinements = []
# Update content with status
update_content_not_deleted_status_criteria(content)
# get selected refinements (categories)
for refinement_name, selected_categories in refinement_form.cleaned_data.iteritems(
):
if len(selected_categories) > 0:
# Add categories ids
refinements.append([x.id for x in selected_categories])
# generate query
if len(refinements) > 0:
# get refinement query
refinement_query = mongo_query_api.build_refinements_query(
refinements)
# if we have a refinement query
if len(refinement_query.keys()) > 0:
content.update(refinement_query)
# Update content
query.content = json.dumps(content)
# save query
query_api.upsert(query)
except DoesNotExist:
error = "An unexpected error occurred while retrieving the query."
context.update({'error': error})
except Exception, e:
error = "An unexpected error occurred: {}.".format(e.message)
context.update({'error': error})
def add_oaipmh_data_source(query, data_source, user):
"""Add an oaipmh data source to the query
Args:
query:
data_source:
user:
Returns:
"""
# the unique identifier for an oaipmh data source is its instance_id
# because there is no constraint on the name
data_source_found = False
for data_source_item in query.data_sources:
if ("instance_id" in data_source_item.query_options
and data_source_item.query_options["instance_id"]
== data_source.query_options["instance_id"]):
data_source_found = True
if not data_source_found:
# add data source to query if not present
query.data_sources.append(data_source)
# update query
return upsert(query, user)
def post(self, request):
"""Get a query
Args:
request:
Returns:
"""
try:
template_id = request.POST['templateID']
query_id = request.POST['queryID']
form_values = json.loads(request.POST['formValues'])
# save current query builder in session to restore it when coming back to the page
query_form = request.POST['queryForm']
request.session['savedQueryFormExplore'] = query_form
errors = check_query_form(form_values, template_id)
query_object = query_api.get_by_id(query_id)
if len(query_object.data_sources) == 0:
errors.append("Please select at least 1 data source.")
if len(errors) == 0:
query_content = self.fields_to_query_func(
form_values, template_id)
query_object.content = json.dumps(query_content)
query_api.upsert(query_object)
else:
return HttpResponseBadRequest(
_render_errors(errors),
content_type='application/javascript')
return HttpResponse(json.dumps({}),
content_type='application/javascript')
except exceptions.ModelError:
return HttpResponseBadRequest(
'Invalid input.', content_type='application/javascript')
except Exception:
return HttpResponseBadRequest(
'An unexpected error occurred.',
content_type='application/javascript')
def _create_new_query(user_id, type_):
""" Create a new query
Args:
user_id:
type_:
"""
# Get templates using the given Type
# Here we should get all dependencies recurcively and remove all template.type from the list
# Passing an empty list is a quick fix
# templates = template_api.get_all_templates_by_dependencies([type_])
# Create query
query = Query(user_id=str(user_id), templates=[])
return query_api.upsert(query)
def _create_new_query(request, template):
""" Create a new query
Args:
request:
template:
"""
# from the template, we get the version manager
template_version_manager = template_version_manager_api.get_by_version_id(
str(template.id))
# from the version manager, we get all the version
template_ids = template_api.get_all_by_id_list(
template_version_manager.versions)
# create query
query = create_default_query(request, template_ids)
# then upsert
return query_api.upsert(query)
def get_redirect_url(self, *args, **kwargs):
try:
# here we receive a PersistentQuery id
persistent_query_example = self._get_persistent_query(kwargs['persistent_query_id'])
# from it we have to duplicate it to a Query with the new user_id
# we should probably add the query_id into the persistent query?
# to avoid to recreate this query each time we visit the persistent URL
query = Query(user_id=str(self.request.user.id),
content=persistent_query_example.content,
templates=persistent_query_example.templates,
data_sources=persistent_query_example.data_sources)
query = query_api.upsert(query)
# then redirect to the result page core_explore_example_results with /<template_id>/<query_id>
return self._get_reversed_url(query)
except Exception, e:
# add success message
messages.add_message(self.request, messages.ERROR, 'The given URL is not valid.')
return self._get_reversed_url_if_failed()
def get_redirect_url(self, *args, **kwargs):
try:
# here we receive a PersistentQuery name or id
persistent_query_id = self.request.GET.get("id", None)
persistent_query_name = self.request.GET.get("name", None)
if persistent_query_id is not None:
persistent_query = self._get_persistent_query_by_id(
persistent_query_id, self.request.user)
elif persistent_query_name is not None:
persistent_query = self._get_persistent_query_by_name(
persistent_query_name, self.request.user)
else:
messages.add_message(self.request, messages.ERROR,
"Expecting id or name.")
return self._get_reversed_url_if_failed()
# from it we have to duplicate it to a Query with the new user_id
# we should probably add the query_id into the persistent query?
# to avoid to recreate this query each time we visit the persistent URL
query = Query(
user_id=str(self.request.user.id),
content=persistent_query.content,
templates=persistent_query.templates,
data_sources=persistent_query.data_sources,
)
query = query_api.upsert(query, self.request.user)
# then redirect to the result page core_explore_example_results with /<template_id>/<query_id>
return self._get_reversed_url(query)
except AccessControlError:
# add error message
messages.add_message(self.request, messages.ERROR,
"Access Forbidden.")
return self._get_reversed_url_if_failed()
except Exception as e:
# add error message
messages.add_message(self.request, messages.ERROR,
"The given URL is not valid.")
return self._get_reversed_url_if_failed()
def tiles(request):
"""
:param request:
:return:
"""
from django.conf import settings
installed_apps = settings.INSTALLED_APPS
context = {"tiles": []}
if "core_explore_keyword_registry_app" in installed_apps:
from core_explore_keyword_registry_app.views.user.forms import RefinementForm
from core_explore_common_app.components.query import api as query_api
from core_explore_common_app.components.query.models import Query
from core_explore_common_app.views.user.ajax import add_local_data_source
from core_main_registry_app.components.refinement import api as refinement_api
from core_main_registry_app.components.category import api as category_api
from core_main_registry_app.components.template import (
api as template_registry_api,
)
from core_main_registry_app.components.custom_resource import (
api as custom_resource_api,
)
from core_main_app.commons import exceptions as exceptions
# create Query
query = Query(user_id=str(request.user.id), templates=[])
# add local data source to the query
add_local_data_source(request, query)
# set visibility
query_api.set_visibility_to_query(query)
# upsert the query
query_api.upsert(query)
# add information in context to populate keyword form
context.update(
{
"query_id": str(query.id),
"user_id": query.user_id,
"order_by_field": ",".join(DATA_SORTING_FIELDS),
}
)
try:
# Get current template
template = template_registry_api.get_current_registry_template()
# Get type refinement
refinement = refinement_api.get_by_template_hash_and_by_slug(
template_hash=template.hash, slug="type"
)
# Refinement form ID
refinement_form_id = "{0}-{1}".format(
RefinementForm.prefix, refinement.slug
)
context["refinement_form_id"] = refinement_form_id
# Shorter api name
get_categories = (
category_api.get_all_categories_ids_from_name_and_refinement_id
)
custom_resources = custom_resource_api.get_all_of_current_template().order_by(
"sort"
)
for custom_resource in custom_resources:
if (
custom_resource.display_icon
and custom_resource.role_type is not None
):
tile = {
"logo": custom_resource.icon,
"color": custom_resource.icon_color,
"categories": get_categories(
custom_resource.role_type.split(":")[0], refinement.id
),
"title": custom_resource.title,
"text": "Click here to explore the {0}.".format(
custom_resource.title
),
}
context["tiles"].append(tile)
except (exceptions.DoesNotExist, exceptions.ModelError) as e:
logger.error(
"Error while getting information from the database: {0}".format(str(e))
)
except Exception as ex:
logger.error(
"Something wrong occurred during the tiles "
"generation: {0}".format(str(ex))
)
return render(request, "nmrr_home/tiles.html", context)
def test_upsert_query_returns_query(self, mock_save):
query = _create_query()
mock_save.return_value = query
mock_user = create_mock_user("1")
self.assertTrue(isinstance(query_api.upsert(query, mock_user), Query))
def post(self, request):
"""Get a query
Args:
request:
Returns:
"""
try:
template_id = request.POST["templateID"]
query_id = request.POST["queryID"]
form_values = (json.loads(request.POST["formValues"])
if "formValues" in request.POST else None)
order_by_field = request.POST["orderByField"].strip()
order_by_field_array = order_by_field.split(";")
# save current query builder in session to restore it when coming
# back to the page
if "queryForm" in request.POST:
query_form = request.POST["queryForm"]
request.session["savedQueryFormExplore"] = query_form
errors = []
query_object = query_api.get_by_id(query_id, request.user)
# set the data-sources sorting value according to the POST request field
for data_sources_index in range(len(query_object.data_sources)):
# updating only the existing data-sources (the new data-source already got
# the default filter value)
if data_sources_index in range(0, len(order_by_field_array)):
query_object.data_sources[
data_sources_index].order_by_field = order_by_field_array[
data_sources_index]
if len(query_object.data_sources) == 0:
errors.append("Please select at least 1 data source.")
if len(errors) == 0 and form_values:
errors.append(
check_query_form(form_values, template_id,
request=request))
query_content = self.fields_to_query_func(form_values,
template_id,
request=request)
query_object.content = json.dumps(query_content)
elif len(errors) > 0:
return HttpResponseBadRequest(
_render_errors(errors),
content_type="application/javascript")
query_api.upsert(query_object, request.user)
return HttpResponse(json.dumps({}),
content_type="application/javascript")
except exceptions.ModelError:
return HttpResponseBadRequest(
"Invalid input.", content_type="application/javascript")
except Exception as e:
return HttpResponseBadRequest(
"An unexpected error occurred: %s" % escape(str(e)),
content_type="application/javascript",
)
def _get(self, user, query_id):
"""Update the GET context
Args:
user:
query_id:
Returns:
"""
context = super(KeywordSearchRegistryView, self)._get(user, query_id)
data_form = {}
refinement_selected_types = []
category_list = ""
query_id = context.get("query_id", None)
# in case we don't find the query id in the context
# a default query is created by inheritance
if query_id is None:
return {"error": "query id is missing"}
try:
# get the query id
query = query_api.get_by_id(query_id, self.request.user)
# here we have to make sure to set the visibility and status criteria
# set visibility
query_api.set_visibility_to_query(query, self.request.user)
# load content
content = json.loads(query.content)
# update content with status
update_content_not_deleted_status_criteria(content)
query.content = json.dumps(content)
# save query
query_api.upsert(query, self.request.user)
# get all keywords back
refinement_selected_values = (
mongo_query_api.get_refinement_selected_values_from_query(
content, self.request))
# build the data_form structure
for key in refinement_selected_values:
for display_name in refinement_selected_values[key]:
list_element = refinement_selected_values[key][
display_name]
data_form.update({
RefinementForm.prefix + "-" + key:
[element["id"] for element in list_element]
})
# add the selected type only for the display_name = Type
if display_name == "Type":
refinement_selected_types = [
element["value"] for element in list_element
]
# create the list of category
if len(refinement_selected_values[key]) > 0:
category_list = "%s,%s|%s" % (category_list,
display_name, key)
context.update({"category_list": category_list})
except Exception as e:
context.update({
"error":
"An unexpected error occurred while loading the query: %s." %
str(e)
})
context.update({
"refinement_form":
RefinementForm(data=data_form, request=self.request)
})
# get all categories which must be selected in the table
context.update(
{"refinement_selected_types": refinement_selected_types})
# Custom registry
self._update_context_with_custom_resources(context)
return context
def _post(self, request):
"""Update the POST context
Args:
request:
Returns:
"""
context = super(KeywordSearchRegistryView, self)._post(request)
# get refinement form
refinement_form = RefinementForm(data=request.POST, request=request)
# get query_id and error from the context
error = context.get("error", None)
query_id = context.get("query_id", None)
refinement_selected_types = []
# validate form, test if no errors occurred in the parent treatment and query_id exists
if refinement_form.is_valid(
) and error is None and query_id is not None:
try:
query = query_api.get_by_id(query_id, request.user)
content = json.loads(query.content)
refinements = []
# Update content with status
update_content_not_deleted_status_criteria(content)
# get selected refinements (categories)
for refinement_name, selected_categories in list(
refinement_form.cleaned_data.items()):
if len(selected_categories) > 0:
# Add categories ids
refinements.append([x.id for x in selected_categories])
# generate query
if len(refinements) > 0:
# get refinement query
refinement_query = mongo_query_api.build_refinements_query(
refinements)
# if we have a refinement query
if len(list(refinement_query.keys())) > 0:
if ("$and" in content.keys()
and "$and" in refinement_query.keys()):
content["$and"] += refinement_query["$and"]
elif ("$or" in content.keys()
or "$text" in content.keys()
) and "$and" in refinement_query.keys():
new_content = refinement_query
# copy the text or the search operators in the main $and
for key, value in content.items():
if key == "$or" or key == "$text":
key_pair = {}
key_pair[key] = value
new_content["$and"].append(key_pair)
else:
new_content[key] = value
content = new_content
else:
content.update(refinement_query)
# Update content
query.content = json.dumps(content)
# save query
query_api.upsert(query, request.user)
except DoesNotExist:
error = "An unexpected error occurred while retrieving the query."
context.update({"error": error})
except Exception as e:
error = "An unexpected error occurred: {}.".format(str(e))
context.update({"error": error})
context.update({"refinement_form": refinement_form})
# get all categories which must be selected in the table
if refinement_form.cleaned_data:
# get the current template
template = template_registry_api.get_current_registry_template(
request=request)
# get the refinement 'Type'
refinement = refinement_api.get_by_template_hash_and_by_slug(
template.hash, "type")
# get the selected_types
selected_types = refinement_form.cleaned_data.get(
refinement.slug, None)
# create the list of type
if selected_types:
refinement_selected_types = get_all_parent_name_from_category_list(
selected_types)
# create the list of category
category_list = ""
for key in refinement_form.cleaned_data:
if len(refinement_form.cleaned_data[key]) > 0:
category_list = "%s,%s|%s" % (
category_list,
refinement_form.cleaned_data[key][0].refinement.name,
key,
)
context.update({"category_list": category_list})
# get all categories which must be selected in the table
context.update(
{"refinement_selected_types": refinement_selected_types})
# Custom registry
self._update_context_with_custom_resources(context)
return context
def tiles(request):
"""
:param request:
:return:
"""
from django.conf import settings
installed_apps = settings.INSTALLED_APPS
context = {"tiles": []}
if "core_explore_keyword_registry_app" in installed_apps:
from core_explore_keyword_registry_app.views.user.forms import RefinementForm
from core_explore_common_app.components.query import api as query_api
from core_explore_common_app.components.query.models import Query
from core_explore_common_app.views.user.ajax import add_local_data_source
from core_main_registry_app.components.refinement import api as refinement_api
from core_main_registry_app.components.category import api as category_api
from core_main_registry_app.components.template import api as template_registry_api
from core_main_app.commons import exceptions as exceptions
# create Query
query = Query(user_id=str(request.user.id), templates=[])
query_api.upsert(query)
# add local data source to the query
add_local_data_source(request, query)
# add information in context to populate keyword form
context.update({"query_id": str(query.id), "user_id": query.user_id})
try:
# Get current template
template = template_registry_api.get_current_registry_template()
# Get type refinement
refinement = refinement_api.get_by_template_hash_and_by_slug(
template_hash=template.hash, slug='type')
# Refinement form ID
refinement_form_id = "{0}-{1}".format(RefinementForm.prefix,
refinement.slug)
context["refinement_form_id"] = refinement_form_id
# Shorter api name
get_categories = category_api.get_all_categories_ids_by_parent_slug_and_refinement_id
organizations_tile = {
"logo": "fa-university",
"color": "#2CAAE2",
"categories": get_categories('organization', refinement.id),
"title": "Organizations",
"text": "Click here to explore the Organizations."
}
context["tiles"].append(organizations_tile)
data_collections_tile = {
"logo": "fa-table",
"color": "#A1C057",
"categories": get_categories('collection', refinement.id),
"title": "Data Collections",
"text": "Click here to explore the Data Collections."
}
context["tiles"].append(data_collections_tile)
datasets_tile = {
"logo": "fa-database",
"color": "grey",
"categories": get_categories('dataset', refinement.id),
"title": "Datasets",
"text": "Click here to explore the Datasets."
}
context["tiles"].append(datasets_tile)
services_tile = {
"logo": "fa-cogs",
"color": "#EBB057;",
"categories": get_categories('service', refinement.id),
"title": "Services",
"text": "Click here to explore the Services."
}
context["tiles"].append(services_tile)
informational_tile = {
"logo": "fa-laptop",
"color": "#257ABC;",
"categories": get_categories('web-site', refinement.id),
"title": "Informational Sites",
"text": "Click here to explore the Informational Sites."
}
context["tiles"].append(informational_tile)
software_tile = {
"logo": "fa-tachometer",
"color": "#79B320;",
"categories": get_categories('software', refinement.id),
"title": "Software",
"text": "Click here to explore the Software."
}
context["tiles"].append(software_tile)
except (exceptions.DoesNotExist, exceptions.ModelError), e:
logger.error(
"Error while getting information from the database: {0}".
format(e.message))
except Exception, ex:
logger.error("Something wrong occurred during the tiles "
"generation: {0}".format(ex.message))