def check_data_from_template(request, pk): """Check if data found for provided xpaths Args: request: pk: Returns: """ try: # Get elasticsearch configuration for the given template es_template = elasticsearch_template_api.get_by_id(pk) # Check if titles found with given path title_results = data_api.execute_query( query=get_exists_query_from_path(es_template.title_path), user=request.user) # Check if descriptions found with given path desc_count = 0 for path in es_template.description_paths: description_results = data_api.execute_query( query=get_exists_query_from_path(path), user=request.user, ) desc_count += description_results.count() message = (f"Title: {title_results.count()} data found. " f"Description: {desc_count} data fields found.") except Exception as e: message = str(e) return HttpResponse(json.dumps(message), content_type="application/javascript")
def test_execute_query_force_workspace_1_does_not_return_data_if_no_access( self, get_all_workspaces_with_read_access_by_user): mock_user = _create_user('3') get_all_workspaces_with_read_access_by_user.return_value = [] data_list = data_api.execute_query( {'workspace': fixture_data.workspace_1.id}, mock_user) self.assertTrue(len(data_list) == 0)
def get(self, request, *args, **kwargs): # Get arguments is_published = request.GET.get('ispublished', None) is_published = None if is_published not in ['true', 'false' ] else is_published role_name_list = request.GET.getlist('role', ['all']) page = request.GET.get('page', 1) role_name = ','.join(role_name_list) context = { 'page': page, 'roles': role_name, 'ispublished': is_published } # Get resources try: loaded_data = data_api.execute_query( create_query_dashboard_resources(request, role_name_list, self.administration), request.user, '-last_modification_date') except AccessControlError, ace: loaded_data = []
def test_data_execute_query_api_without_sorting_param_use_default_data_sorting_setting(self): # Arrange mock_user = _create_user('1', is_superuser=True) # Act data = data_api.execute_query({}, mock_user) #Assert self.assertListEqual(list(data), self.fixture.data_collection)
def test_execute_query_returns_data_in_workspace_1_and_2(self, get_all_workspaces_with_read_access_by_user): mock_user = _create_user('3') get_all_workspaces_with_read_access_by_user.return_value = [fixture_data.workspace_1, fixture_data.workspace_2] data_list = data_api.execute_query({}, mock_user) self.assertTrue(len(data_list) == 2) self.assertTrue(data.workspace == '1' or data.workspace == '2' for data in data_list)
def test_execute_query_returns_data( self, get_all_workspaces_with_read_access_by_user): mock_user = _create_user('3') get_all_workspaces_with_read_access_by_user.return_value = [ fixture_data.workspace_1 ] data_list = data_api.execute_query({}, mock_user) self.assertTrue(len(data_list) > 0) self.assertTrue(all(isinstance(data, Data) for data in data_list))
def execute_raw_query(self, raw_query): """ Execute the raw query in database. Args: raw_query: Query to execute. Returns: Results of the query. """ return data_api.execute_query(raw_query, self.request.user)
def test_execute_query_force_workspace_1_returns_data_from_workspace_1( self, get_all_workspaces_with_read_access_by_user): mock_user = _create_user('3') get_all_workspaces_with_read_access_by_user.return_value = [ fixture_data.workspace_1 ] data_list = data_api.execute_query( {'workspace': fixture_data.workspace_1.id}, mock_user) self.assertTrue(len(data_list) == 2) self.assertTrue(data.workspace == '1' for data in data_list)
def execute_raw_query(self, raw_query, order_by_field): """Execute the raw query in database Args: raw_query: Query to execute order_by_field: Returns: Results of the query """ return data_api.execute_query(raw_query, self.request.user, order_by_field)
def get_last(user): """ Return last T-SNE map. Returns: """ # Get latest T-SNE map tsne_map = TSNEMap.get_last() # Get ids of accessible data accessible_data = map(str, data_api.execute_query({}, user).values_list('id')) # read csv content csv_content = tsne_map.file.read() # filter csv content, keeping only accessible data filtered_csv_content = filter_rows(csv_content, column_index=-1, allowed_values=accessible_data) return filtered_csv_content
def get_data_by_pid(pid, user): """Return data object with the given pid. Parameters: pid: user: Returns: data object """ json_pid_path = "dict_content.%s" % PID_XPATH query_result = data_api.execute_query({json_pid_path: pid}, user) query_result_length = len(query_result) if query_result_length == 0: raise DoesNotExist("PID is not attached to any data.") elif query_result_length != 1: raise ApiError("PID must be unique.") else: return query_result[0]
def post(self, request): """ Execute query Parameters: { "query" : "value", "options" : "number", "templates" : "[]" } Args: request: HTTP request Returns: - code: 200 content: Paginated list of Data - code: 500 content: Internal server error """ try: # serialization serializer = QueryExecuteSerializer(data=request.data) # validation serializer.is_valid(True) # get the options options = None if 'options' in serializer.validated_data: options = serializer.validated_data['options'] # init a QueryBuilder with the query query_builder = QueryBuilder(serializer.validated_data['query'], 'dict_content') # update the content query with given templates if 'templates' in serializer.validated_data: _update_query_builder(query_builder, serializer.validated_data['templates']) # create a raw query raw_query = query_builder.get_raw_query() # execute query data_list = data_api.execute_query(raw_query, request.user) # get paginator paginator = StandardResultsSetPagination() # get request page from list of results page = paginator.paginate_queryset(data_list, request) # Serialize object results = [] url = reverse("core_explore_federated_search_app_data_detail") url_access_data = reverse( "core_explore_federated_search_app_rest_get_result_from_data_id" ) instance_name = '' if options is not None: if 'instance_name' in options: instance_name = options['instance_name'] # Template info template_info = dict() for data in page: # get data's template template = data.template # get and store data's template information if template not in template_info: template_info[template] = result_utils.get_template_info( template, include_template_id=False) results.append( Result( title=data.title, xml_content=data.xml_content, template_info=template_info[template], detail_url="{0}?id={1}&instance_name={2}".format( url, data.id, instance_name), access_data_url="{0}?id={1}&instance_name={2}".format( url_access_data, data.id, instance_name))) return_value = ResultSerializer(results, many=True) return paginator.get_paginated_response(return_value.data) except Exception as api_exception: content = {'message': api_exception.message} return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def test_execute_query_as_superuser_returns_all_data(self): mock_user = _create_user('1', is_superuser=True) data_list = data_api.execute_query({}, mock_user) self.assertTrue(len(data_list) == 5)
def test_execute_query_force_workspace_none_does_not_return_data_if_no_access( self, get_all_workspaces_with_read_access_by_user): mock_user = _create_user("3") get_all_workspaces_with_read_access_by_user.return_value = [] data_list = data_api.execute_query({"workspace": None}, mock_user) self.assertTrue(len(data_list) == 0)
def async_template_migration_task(templates, xslt_id, target_template_id, user_id, migrate): """Async task which perform a migration / validation of all the data which belong to the given template id list Args: templates: xslt_id: target_template_id: user_id migrate: (boolean) Perform the migration Return: {"valid": <number>, "wrong": <number>} """ # get the data list to check current_data_progress = 0 current_template_progress = -1 total_data = 0 total_template = len(templates) success = [] error = [] try: if target_template_id and total_template > 0: # get the user user = user_api.get_user_by_id(user_id) # get the target template target_template = system_api.get_template_by_id(target_template_id) # get xsl transformation if selected if xslt_id is not None: xslt = xsl_transformation_api.get_by_id(str(xslt_id)) for template_id in templates: # increase the number of processed template current_template_progress += 1 # rest de number of data current_data_progress = 0 # get a QuerySet of all the data with the given template data_list = data_api.execute_query( {"template": ObjectId(template_id)}, user=user) total_data = data_list.count() # extract the data id from the list data_list_id = data_list.values_list("id") for data_id in data_list_id: # get the data data = data_api.get_by_id(data_id, user) # modify the data temporarily with the new targeted template data.template = target_template if xslt_id is not None: # modify the xml content temporarily with the transformed data content data.xml_content = xsl_transformation_api.xsl_transform( data.xml_content, xslt.name) # check if the data is valid try: # save the new template for the data if the migration is True if migrate: system_api.upsert_data(data) else: data_api.check_xml_file_is_valid(data) success.append(str(data.id)) except Exception as e: error.append(str(data.id)) finally: # increase the current progress and update the task state current_data_progress += 1 async_template_migration_task.update_state( state="PROGRESS", meta={ "template_current": current_template_progress, "template_total": total_template, "data_current": current_data_progress, "data_total": total_data, }, ) return {"valid": success, "wrong": error} else: async_template_migration_task.update_state( state="ABORT", meta={ "template_current": current_template_progress, "template_total": total_template, "data_current": current_data_progress, "data_total": total_data, }, ) raise Exception("Wrong template id." if not target_template_id else "Please provide template id.") except Exception as e: async_template_migration_task.update_state( state="ABORT", meta={ "template_current": current_template_progress, "template_total": total_data, "data_current": current_data_progress, "data_total": total_data, }, ) raise Exception(f"Something went wrong: {str(e)}")
def post(self, request): """Execute query Parameters: { "query" : "value", "options" : "number", "templates" : "[]" } Args: request: HTTP request Returns: - code: 200 content: Paginated list of Data - code: 500 content: Internal server error """ try: # serialization serializer = QueryExecuteSerializer(data=request.data) # validation serializer.is_valid(True) # get the options options = None if "options" in serializer.validated_data: options = serializer.validated_data["options"] # get the query order by field order_by_field = (request.data["order_by_field"].split(",") if "order_by_field" in request.data else "") # init a QueryBuilder with the query query_builder = QueryBuilder(serializer.validated_data["query"], "dict_content") # update the content query with given templates if "templates" in serializer.validated_data: _update_query_builder( query_builder, serializer.validated_data["templates"], request=request, ) # create a raw query raw_query = query_builder.get_raw_query() # execute query data_list = data_api.execute_query(raw_query, request.user, order_by_field) # get paginator paginator = StandardResultsSetPagination() # get request page from list of results page = paginator.paginate_queryset(data_list, request) # Serialize object results = [] url = reverse("core_explore_federated_search_app_data_detail") url_access_data = reverse( "core_explore_federated_search_app_rest_get_result_from_data_id" ) instance_name = "" if options is not None: if "instance_name" in options: instance_name = options["instance_name"] # Template info template_info = dict() # get the request session time zone request_time_zone = (request.META.get("HTTP_TZ") if request.META.get("HTTP_TZ") else "UTC") # activate the timezone according to the request activate(pytz.timezone(request_time_zone)) for data in page: # get data's template template = data.template # get and store data's template information if template not in template_info: template_info[template] = result_utils.get_template_info( template, include_template_id=False) results.append( Result( title=data.title, xml_content=data.xml_content, template_info=template_info[template], permission_url=None, detail_url="{0}?id={1}&instance_name={2}".format( url, data.id, instance_name), last_modification_date=pytz.utc.localize( data.last_modification_date), access_data_url="{0}?id={1}&instance_name={2}".format( url_access_data, data.id, instance_name), )) return_value = ResultSerializer(results, many=True) return paginator.get_paginated_response(return_value.data) except Exception as api_exception: content = {"message": str(api_exception)} return Response(content, status=status.HTTP_500_INTERNAL_SERVER_ERROR)