def check_data_from_template(request, pk):
"""Check if data found for provided xpaths
Args:
request:
pk:
Returns:
"""
try:
# Get elasticsearch configuration for the given template
es_template = elasticsearch_template_api.get_by_id(pk)
# Check if titles found with given path
title_results = data_api.execute_query(
query=get_exists_query_from_path(es_template.title_path),
user=request.user)
# Check if descriptions found with given path
desc_count = 0
for path in es_template.description_paths:
description_results = data_api.execute_query(
query=get_exists_query_from_path(path),
user=request.user,
)
desc_count += description_results.count()
message = (f"Title: {title_results.count()} data found. "
f"Description: {desc_count} data fields found.")
except Exception as e:
message = str(e)
return HttpResponse(json.dumps(message),
content_type="application/javascript")
def test_execute_query_force_workspace_1_does_not_return_data_if_no_access(
self, get_all_workspaces_with_read_access_by_user):
mock_user = _create_user('3')
get_all_workspaces_with_read_access_by_user.return_value = []
data_list = data_api.execute_query(
{'workspace': fixture_data.workspace_1.id}, mock_user)
self.assertTrue(len(data_list) == 0)
def get(self, request, *args, **kwargs):
# Get arguments
is_published = request.GET.get('ispublished', None)
is_published = None if is_published not in ['true', 'false'
] else is_published
role_name_list = request.GET.getlist('role', ['all'])
page = request.GET.get('page', 1)
role_name = ','.join(role_name_list)
context = {
'page': page,
'roles': role_name,
'ispublished': is_published
}
# Get resources
try:
loaded_data = data_api.execute_query(
create_query_dashboard_resources(request, role_name_list,
self.administration),
request.user, '-last_modification_date')
except AccessControlError, ace:
loaded_data = []
def test_data_execute_query_api_without_sorting_param_use_default_data_sorting_setting(self):
# Arrange
mock_user = _create_user('1', is_superuser=True)
# Act
data = data_api.execute_query({}, mock_user)
#Assert
self.assertListEqual(list(data), self.fixture.data_collection)
def test_execute_query_returns_data_in_workspace_1_and_2(self, get_all_workspaces_with_read_access_by_user):
mock_user = _create_user('3')
get_all_workspaces_with_read_access_by_user.return_value = [fixture_data.workspace_1,
fixture_data.workspace_2]
data_list = data_api.execute_query({}, mock_user)
self.assertTrue(len(data_list) == 2)
self.assertTrue(data.workspace == '1' or data.workspace == '2' for data in data_list)
def test_execute_query_returns_data(
self, get_all_workspaces_with_read_access_by_user):
mock_user = _create_user('3')
get_all_workspaces_with_read_access_by_user.return_value = [
fixture_data.workspace_1
]
data_list = data_api.execute_query({}, mock_user)
self.assertTrue(len(data_list) > 0)
self.assertTrue(all(isinstance(data, Data) for data in data_list))
def execute_raw_query(self, raw_query):
""" Execute the raw query in database.
Args:
raw_query: Query to execute.
Returns:
Results of the query.
"""
return data_api.execute_query(raw_query, self.request.user)
def test_execute_query_force_workspace_1_returns_data_from_workspace_1(
self, get_all_workspaces_with_read_access_by_user):
mock_user = _create_user('3')
get_all_workspaces_with_read_access_by_user.return_value = [
fixture_data.workspace_1
]
data_list = data_api.execute_query(
{'workspace': fixture_data.workspace_1.id}, mock_user)
self.assertTrue(len(data_list) == 2)
self.assertTrue(data.workspace == '1' for data in data_list)
def execute_raw_query(self, raw_query, order_by_field):
"""Execute the raw query in database
Args:
raw_query: Query to execute
order_by_field:
Returns:
Results of the query
"""
return data_api.execute_query(raw_query, self.request.user, order_by_field)
def get_last(user):
""" Return last T-SNE map.
Returns:
"""
# Get latest T-SNE map
tsne_map = TSNEMap.get_last()
# Get ids of accessible data
accessible_data = map(str,
data_api.execute_query({}, user).values_list('id'))
# read csv content
csv_content = tsne_map.file.read()
# filter csv content, keeping only accessible data
filtered_csv_content = filter_rows(csv_content,
column_index=-1,
allowed_values=accessible_data)
return filtered_csv_content
def get_data_by_pid(pid, user):
"""Return data object with the given pid.
Parameters:
pid:
user:
Returns: data object
"""
json_pid_path = "dict_content.%s" % PID_XPATH
query_result = data_api.execute_query({json_pid_path: pid}, user)
query_result_length = len(query_result)
if query_result_length == 0:
raise DoesNotExist("PID is not attached to any data.")
elif query_result_length != 1:
raise ApiError("PID must be unique.")
else:
return query_result[0]
def post(self, request):
""" Execute query
Parameters:
{
"query" : "value",
"options" : "number",
"templates" : "[]"
}
Args:
request: HTTP request
Returns:
- code: 200
content: Paginated list of Data
- code: 500
content: Internal server error
"""
try:
# serialization
serializer = QueryExecuteSerializer(data=request.data)
# validation
serializer.is_valid(True)
# get the options
options = None
if 'options' in serializer.validated_data:
options = serializer.validated_data['options']
# init a QueryBuilder with the query
query_builder = QueryBuilder(serializer.validated_data['query'],
'dict_content')
# update the content query with given templates
if 'templates' in serializer.validated_data:
_update_query_builder(query_builder,
serializer.validated_data['templates'])
# create a raw query
raw_query = query_builder.get_raw_query()
# execute query
data_list = data_api.execute_query(raw_query, request.user)
# get paginator
paginator = StandardResultsSetPagination()
# get request page from list of results
page = paginator.paginate_queryset(data_list, request)
# Serialize object
results = []
url = reverse("core_explore_federated_search_app_data_detail")
url_access_data = reverse(
"core_explore_federated_search_app_rest_get_result_from_data_id"
)
instance_name = ''
if options is not None:
if 'instance_name' in options:
instance_name = options['instance_name']
# Template info
template_info = dict()
for data in page:
# get data's template
template = data.template
# get and store data's template information
if template not in template_info:
template_info[template] = result_utils.get_template_info(
template, include_template_id=False)
results.append(
Result(
title=data.title,
xml_content=data.xml_content,
template_info=template_info[template],
detail_url="{0}?id={1}&instance_name={2}".format(
url, data.id, instance_name),
access_data_url="{0}?id={1}&instance_name={2}".format(
url_access_data, data.id, instance_name)))
return_value = ResultSerializer(results, many=True)
return paginator.get_paginated_response(return_value.data)
except Exception as api_exception:
content = {'message': api_exception.message}
return Response(content,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def test_execute_query_as_superuser_returns_all_data(self):
mock_user = _create_user('1', is_superuser=True)
data_list = data_api.execute_query({}, mock_user)
self.assertTrue(len(data_list) == 5)
def test_execute_query_force_workspace_none_does_not_return_data_if_no_access(
self, get_all_workspaces_with_read_access_by_user):
mock_user = _create_user("3")
get_all_workspaces_with_read_access_by_user.return_value = []
data_list = data_api.execute_query({"workspace": None}, mock_user)
self.assertTrue(len(data_list) == 0)
def async_template_migration_task(templates, xslt_id, target_template_id,
user_id, migrate):
"""Async task which perform a migration / validation of all the data which belong to the given template id list
Args:
templates:
xslt_id:
target_template_id:
user_id
migrate: (boolean) Perform the migration
Return:
{"valid": <number>, "wrong": <number>}
"""
# get the data list to check
current_data_progress = 0
current_template_progress = -1
total_data = 0
total_template = len(templates)
success = []
error = []
try:
if target_template_id and total_template > 0:
# get the user
user = user_api.get_user_by_id(user_id)
# get the target template
target_template = system_api.get_template_by_id(target_template_id)
# get xsl transformation if selected
if xslt_id is not None:
xslt = xsl_transformation_api.get_by_id(str(xslt_id))
for template_id in templates:
# increase the number of processed template
current_template_progress += 1
# rest de number of data
current_data_progress = 0
# get a QuerySet of all the data with the given template
data_list = data_api.execute_query(
{"template": ObjectId(template_id)}, user=user)
total_data = data_list.count()
# extract the data id from the list
data_list_id = data_list.values_list("id")
for data_id in data_list_id:
# get the data
data = data_api.get_by_id(data_id, user)
# modify the data temporarily with the new targeted template
data.template = target_template
if xslt_id is not None:
# modify the xml content temporarily with the transformed data content
data.xml_content = xsl_transformation_api.xsl_transform(
data.xml_content, xslt.name)
# check if the data is valid
try:
# save the new template for the data if the migration is True
if migrate:
system_api.upsert_data(data)
else:
data_api.check_xml_file_is_valid(data)
success.append(str(data.id))
except Exception as e:
error.append(str(data.id))
finally:
# increase the current progress and update the task state
current_data_progress += 1
async_template_migration_task.update_state(
state="PROGRESS",
meta={
"template_current": current_template_progress,
"template_total": total_template,
"data_current": current_data_progress,
"data_total": total_data,
},
)
return {"valid": success, "wrong": error}
else:
async_template_migration_task.update_state(
state="ABORT",
meta={
"template_current": current_template_progress,
"template_total": total_template,
"data_current": current_data_progress,
"data_total": total_data,
},
)
raise Exception("Wrong template id." if not target_template_id else
"Please provide template id.")
except Exception as e:
async_template_migration_task.update_state(
state="ABORT",
meta={
"template_current": current_template_progress,
"template_total": total_data,
"data_current": current_data_progress,
"data_total": total_data,
},
)
raise Exception(f"Something went wrong: {str(e)}")
def post(self, request):
"""Execute query
Parameters:
{
"query" : "value",
"options" : "number",
"templates" : "[]"
}
Args:
request: HTTP request
Returns:
- code: 200
content: Paginated list of Data
- code: 500
content: Internal server error
"""
try:
# serialization
serializer = QueryExecuteSerializer(data=request.data)
# validation
serializer.is_valid(True)
# get the options
options = None
if "options" in serializer.validated_data:
options = serializer.validated_data["options"]
# get the query order by field
order_by_field = (request.data["order_by_field"].split(",")
if "order_by_field" in request.data else "")
# init a QueryBuilder with the query
query_builder = QueryBuilder(serializer.validated_data["query"],
"dict_content")
# update the content query with given templates
if "templates" in serializer.validated_data:
_update_query_builder(
query_builder,
serializer.validated_data["templates"],
request=request,
)
# create a raw query
raw_query = query_builder.get_raw_query()
# execute query
data_list = data_api.execute_query(raw_query, request.user,
order_by_field)
# get paginator
paginator = StandardResultsSetPagination()
# get request page from list of results
page = paginator.paginate_queryset(data_list, request)
# Serialize object
results = []
url = reverse("core_explore_federated_search_app_data_detail")
url_access_data = reverse(
"core_explore_federated_search_app_rest_get_result_from_data_id"
)
instance_name = ""
if options is not None:
if "instance_name" in options:
instance_name = options["instance_name"]
# Template info
template_info = dict()
# get the request session time zone
request_time_zone = (request.META.get("HTTP_TZ")
if request.META.get("HTTP_TZ") else "UTC")
# activate the timezone according to the request
activate(pytz.timezone(request_time_zone))
for data in page:
# get data's template
template = data.template
# get and store data's template information
if template not in template_info:
template_info[template] = result_utils.get_template_info(
template, include_template_id=False)
results.append(
Result(
title=data.title,
xml_content=data.xml_content,
template_info=template_info[template],
permission_url=None,
detail_url="{0}?id={1}&instance_name={2}".format(
url, data.id, instance_name),
last_modification_date=pytz.utc.localize(
data.last_modification_date),
access_data_url="{0}?id={1}&instance_name={2}".format(
url_access_data, data.id, instance_name),
))
return_value = ResultSerializer(results, many=True)
return paginator.get_paginated_response(return_value.data)
except Exception as api_exception:
content = {"message": str(api_exception)}
return Response(content,
status=status.HTTP_500_INTERNAL_SERVER_ERROR)
