def bulk_delete(self, request, *args, **kwargs): """ Bulk delete instances """ xform = self.__validate_permission_on_bulk_action( request, 'change_xform') payload = self.__get_payload(request) postgres_query, mongo_query = self.__build_db_queries(xform, payload) # Delete Postgres & Mongo updated_records_count = Instance.objects.filter( **postgres_query).count() # Since Django 1.9, `.delete()` returns an dict with number of rows # deleted per object. # FixMe remove `.count()` query and use that dict instance Instance.objects.filter(**postgres_query).delete() ParsedInstance.bulk_delete(mongo_query) return Response( { 'detail': _('{} submissions have been deleted').format( updated_records_count) }, status.HTTP_200_OK)
def test_edit_updated_geopoint_cache(self): query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args['count'] = False records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]['gps'].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def bulk_validation_status(self, request, *args, **kwargs): xform = self.__validate_permission_on_bulk_action( request, 'validate_xform') payload = self.__get_payload(request) try: new_validation_status_uid = payload['validation_status.uid'] except KeyError: raise ValidationError( {'payload': _('No `validation_status.uid` provided')}) # Create new validation_status object new_validation_status = get_validation_status( new_validation_status_uid, xform, request.user.username) postgres_query, mongo_query = self.__build_db_queries(xform, payload) # Update Postgres & Mongo updated_records_count = Instance.objects.\ filter(**postgres_query).update(validation_status=new_validation_status) ParsedInstance.bulk_update_validation_statuses(mongo_query, new_validation_status) return Response( { 'detail': _('{} submissions have been updated').format( updated_records_count) }, status.HTTP_200_OK)
def test_xform_delete_cascades_mongo_instances(self): initial_mongo_count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]["count"] # submit instance for i in range(len(self.surveys)): self._submit_transport_instance(i) # check mongo record exists mongo_count = ParsedInstance.query_mongo(self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]["count"] self.assertEqual(mongo_count, initial_mongo_count + len(self.surveys)) # delete form xform_delete_url = reverse(delete_xform, kwargs={ 'username': self.user.username, 'id_string': self.xform.id_string }) self.client.post(xform_delete_url) mongo_count = ParsedInstance.query_mongo(self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]["count"] self.assertEqual(mongo_count, initial_mongo_count)
def test_edit_updated_geopoint_cache(self): query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args['count'] = False records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]['gps'].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_deleted_submission_not_in_export(self): self._publish_transportation_form() initial_count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]['count'] self._submit_transport_instance(0) self._submit_transport_instance(1) count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]['count'] self.assertEqual(count, initial_count + 2) # get id of second submission instance_id = Instance.objects.filter( xform=self.xform).order_by('id').reverse()[0].id delete_url = reverse( delete_data, kwargs={"username": self.user.username, "id_string": self.xform.id_string}) params = {'id': instance_id} self.client.post(delete_url, params) count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]['count'] self.assertEqual(count, initial_count + 1) # create the export csv_export_url = reverse( 'csv_export', kwargs={"username": self.user.username, "id_string": self.xform.id_string}) response = self.client.get(csv_export_url) self.assertEqual(response.status_code, 200) f = StringIO.StringIO(self._get_response_content(response)) csv_reader = csv.reader(f) num_rows = len([row for row in csv_reader]) f.close() # number of rows == 2 i.e. initial_count + header plus one row self.assertEqual(num_rows, initial_count + 2)
def _get_data(self, query, fields, sort, start, limit, is_public_request): try: where, where_params = ParsedInstance._get_where_clause(query) except ValueError as e: raise ParseError(unicode(e)) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (sort or limit or start or fields) and not is_public_request: if self.object_list.count(): xform = self.object_list[0].xform self.object_list = \ ParsedInstance.query_data( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) if not isinstance(self.object_list, types.GeneratorType): page = self.paginate_queryset(self.object_list) serializer = self.get_pagination_serializer(page) else: serializer = self.get_serializer(self.object_list, many=True) page = None return Response(serializer.data)
def test_edited_submission(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { "username": self.user.username, "id_string": self.xform.id_string, "query": "{}", "fields": "[]", "sort": "[]", "count": True, } cursor = ParsedInstance.query_mongo(**query_args) num_mongo_instances = cursor[0]["count"] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_mongo(**query_args) self.assertEqual(cursor[0]["count"], num_mongo_instances + 1) # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_mongo(**query_args) self.assertEqual(cursor[0]["count"], num_mongo_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args["count"] = False cursor = ParsedInstance.query_mongo(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record["name"], edited_name)
def test_edited_submission(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'username': self.user.username, 'id_string': self.xform.id_string, 'query': '{}', 'fields': '[]', 'sort': '[]', 'count': True } cursor = ParsedInstance.query_mongo(**query_args) num_mongo_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_mongo(**query_args) self.assertEqual(cursor[0]['count'], num_mongo_instances + 1) # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) client = DigestClient() client.set_authorization('bob', 'bob', 'Digest') self._make_submission(xml_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_mongo(**query_args) self.assertEqual(cursor[0]['count'], num_mongo_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_mongo(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def test_edited_submission(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'username': self.user.username, 'id_string': self.xform.id_string, 'query': '{}', 'fields': '[]', 'sort': '[]', 'count': True } cursor = ParsedInstance.query_mongo(**query_args) num_mongo_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_mongo(**query_args) self.assertEqual(cursor[0]['count'], num_mongo_instances + 1) # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") client = DigestClient() client.set_authorization('bob', 'bob', 'Digest') self._make_submission(xml_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_mongo(**query_args) self.assertEqual(cursor[0]['count'], num_mongo_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_mongo(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def to_representation(self, obj): request = self.context.get('request') if not isinstance(obj, XForm): return super(DataListSerializer, self).to_representation(obj) query_params = (request and request.query_params) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.user.username, obj.id_string) } try: query.update(json.loads(query_params.get('query', '{}'))) except ValueError: raise ParseError(_("Invalid query: %(query)s" % {'query': query_params.get('query')})) query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) return list(cursor)
def to_representation(self, obj): if not hasattr(obj, 'xform'): return super(DataInstanceSerializer, self).to_representation(obj) request = self.context.get('request') query_params = (request and request.query_params) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.xform.user.username, obj.xform.id_string), u'_id': obj.pk } query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) records = list(record for record in cursor) returned_dict = {} if (len(records)): returned_dict = records[0] #break code fix # returned_dict = (len(records) and records[0]) or records return MongoHelper.to_readable_dict(returned_dict)
def to_native(self, obj): request = self.context.get('request') if obj is None: return super(DataListSerializer, self).to_native(obj) query_params = (request and request.QUERY_PARAMS) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.user.username, obj.id_string) } try: query.update(json.loads(query_params.get('query', '{}'))) except ValueError: raise ParseError( _("Invalid query: %(query)s" % {'query': query_params.get('query')})) query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) records = list(record for record in cursor) return records
def to_native(self, obj): request = self.context.get('request') if obj is None: return super(DataListSerializer, self).to_native(obj) query_params = (request and request.QUERY_PARAMS) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.user.username, obj.id_string) } try: query.update(json.loads(query_params.get('query', '{}'))) except ValueError: raise ParseError(_("Invalid query: %(query)s" % {'query': query_params.get('query')})) query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) records = list(record for record in cursor) return records
def api(request, username=None, id_string=None): """ Returns all results as JSON. If a parameter string is passed, it takes the 'query' parameter, converts this string to a dictionary, an that is then used as a MongoDB query string. NOTE: only a specific set of operators are allow, currently $or and $and. Please send a request if you'd like another operator to be enabled. NOTE: Your query must be valid JSON, double check it here, http://json.parser.online.fr/ E.g. api?query='{"last_name": "Smith"}' """ if request.method == "OPTIONS": response = HttpResponse() add_cors_headers(response) return response helper_auth_helper(request) helper_auth_helper(request) xform, owner = check_and_set_user_and_form(username, id_string, request) if not xform: return HttpResponseForbidden(_(u'Not shared.')) try: args = { 'username': username, 'id_string': id_string, 'query': request.GET.get('query'), 'fields': request.GET.get('fields'), 'sort': request.GET.get('sort') } if 'start' in request.GET: args["start"] = int(request.GET.get('start')) if 'limit' in request.GET: args["limit"] = int(request.GET.get('limit')) if 'count' in request.GET: args["count"] = True if int(request.GET.get('count')) > 0\ else False cursor = ParsedInstance.query_mongo(**args) except ValueError as e: return HttpResponseBadRequest(e.__str__()) records = list(record for record in cursor) response_text = json_util.dumps(records) if 'callback' in request.GET and request.GET.get('callback') != '': callback = request.GET.get('callback') response_text = ("%s(%s)" % (callback, response_text)) response = HttpResponse(response_text, mimetype='application/json') add_cors_headers(response) return response
def test_edit_updated_geopoint_cache(self): query_args = { "username": self.user.username, "id_string": self.xform.id_string, "query": "{}", "fields": "[]", "sort": "[]", "count": True, } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args["count"] = False records = ParsedInstance.query_mongo(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml", ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = ParsedInstance.query_mongo(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]["gps"].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_xform_delete_cascades_mongo_instances(self): initial_mongo_count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, "{}", "[]", "{}", count=True )[0]["count"] # submit instance for i in range(len(self.surveys)): self._submit_transport_instance(i) # check mongo record exists mongo_count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, "{}", "[]", "{}", count=True )[0]["count"] self.assertEqual(mongo_count, initial_mongo_count + len(self.surveys)) # delete form xform_delete_url = reverse( delete_xform, kwargs={"username": self.user.username, "id_string": self.xform.id_string} ) self.client.post(xform_delete_url) mongo_count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, "{}", "[]", "{}", count=True )[0]["count"] self.assertEqual(mongo_count, initial_mongo_count)
def _query_mongo(self, query='{}', start=0, limit=ParsedInstance.DEFAULT_LIMIT, fields='[]', count=False): # ParsedInstance.query_mongo takes params as json strings # so we dumps the fields dictionary count_args = { 'username': self.username, 'id_string': self.id_string, 'query': query, 'fields': '[]', 'sort': '{}', 'count': True } count_object = ParsedInstance.query_mongo(**count_args) record_count = count_object[0]["count"] if record_count == 0: raise NoRecordsFoundError("No records found for your query") # if count was requested, return the count if count: return record_count else: query_args = { 'username': self.username, 'id_string': self.id_string, 'query': query, 'fields': fields, # TODO: we might want to add this in for the user # to sepcify a sort order 'sort': '{}', 'start': start, 'limit': limit, 'count': False } # use ParsedInstance.query_mongo cursor = ParsedInstance.query_mongo(**query_args) return cursor
def _query_data(self, query='{}', start=0, limit=ParsedInstance.DEFAULT_LIMIT, fields='[]', count=False): # ParsedInstance.query_mongo takes params as json strings # so we dumps the fields dictionary count_args = { 'xform': self.xform, 'query': query, 'start': self.start, 'end': self.end, 'fields': '[]', 'sort': '{}', 'count': True } count_object = list(ParsedInstance.query_data(**count_args)) record_count = count_object[0]["count"] if record_count < 1: raise NoRecordsFoundError("No records found for your query") # if count was requested, return the count if count: return record_count else: query_args = { 'xform': self.xform, 'query': query, 'fields': fields, 'start': self.start, 'end': self.end, # TODO: we might want to add this in for the user # to sepcify a sort order 'sort': 'id', 'start_index': start, 'limit': limit, 'count': False } # use ParsedInstance.query_mongo cursor = ParsedInstance.query_data(**query_args) return cursor
def bulk_delete(self, request, *args, **kwargs): """ Bulk delete instances """ xform = self.get_object() postgres_query, mongo_query = self.__build_db_queries(xform, request.data) # Disconnect redundant parsed instance pre_delete signal pre_delete.disconnect(_remove_from_mongo, sender=ParsedInstance) # Delete Postgres & Mongo all_count, results = Instance.objects.filter(**postgres_query).delete() identifier = f'{Instance._meta.app_label}.Instance' deleted_records_count = results[identifier] ParsedInstance.bulk_delete(mongo_query) # Pre_delete signal needs to be re-enabled for parsed instance pre_delete.connect(_remove_from_mongo, sender=ParsedInstance) return Response({ 'detail': t('{} submissions have been deleted').format( deleted_records_count) }, status.HTTP_200_OK)
def _get_form_data(self, xform, **kwargs): query = kwargs.get('query', {}) query = query if query is not None else {} if xform: query[ParsedInstance.USERFORM_ID] =\ u'%s_%s' % (xform.user.username, xform.id_string) query = json.dumps(query) if isinstance(query, dict) else query margs = { 'query': query, 'fields': kwargs.get('fields', None), 'sort': kwargs.get('sort', None) } cursor = ParsedInstance.query_mongo_minimal(**margs) records = list(record for record in cursor) return records
def test_edited_submissions_in_exports(self): self._publish_transportation_form() initial_count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]['count'] instance_name = 'transport_2011-07-25_19-05-36' path = _main_fixture_path(instance_name) self._make_submission(path) count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]['count'] self.assertEqual(count, initial_count + 1) # make edited submission - simulating what enketo would return instance_name = 'transport_2011-07-25_19-05-36-edited' path = _main_fixture_path(instance_name) self._make_submission(path) count = ParsedInstance.query_mongo( self.user.username, self.xform.id_string, '{}', '[]', '{}', count=True)[0]['count'] self.assertEqual(count, initial_count + 1) # create the export csv_export_url = reverse( 'csv_export', kwargs={"username": self.user.username, "id_string": self.xform.id_string}) response = self.client.get(csv_export_url) self.assertEqual(response.status_code, 200) f = StringIO.StringIO(self._get_response_content(response)) csv_reader = csv.DictReader(f) data = [row for row in csv_reader] f.close() num_rows = len(data) # number of rows == initial_count + 1 self.assertEqual(num_rows, initial_count + 1) key = 'transport/loop_over_transport_types_frequency/ambulance/'\ 'frequency_to_referral_facility' self.assertEqual(data[initial_count][key], "monthly")
def to_native(self, obj): request = self.context.get('request') query_params = (request and request.QUERY_PARAMS) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.xform.user.username, obj.xform.id_string), u'_id': obj.pk } query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) records = list(record for record in cursor) return (len(records) and records[0]) or records
def to_representation(self, obj): request = self.context.get('request') if not isinstance(obj, XForm): return super(DataListSerializer, self).to_representation(obj) query_params = (request and request.query_params) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.user.username, obj.id_string) } limit = query_params.get('limit', False) start = query_params.get('start', False) count = query_params.get('count', False) try: query.update(json.loads(query_params.get('query', '{}'))) except ValueError: raise ParseError( _("Invalid query: %(query)s" % {'query': query_params.get('query')})) query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } # if we want the count, we don't kwow to paginate the records. # start and limit are useless then. if count: query_kwargs['count'] = True else: if limit: query_kwargs['limit'] = int(limit) if start: query_kwargs['start'] = int(start) cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) # if we want the count, we only need the first index of the list. if count: return cursor[0] else: return [MongoHelper.to_readable_dict(record) for record in cursor]
def to_representation(self, obj): request = self.context.get('request') if not isinstance(obj, XForm): return super(DataListSerializer, self).to_representation(obj) query_params = (request and request.query_params) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.user.username, obj.id_string) } limit = query_params.get('limit', False) start = query_params.get('start', False) count = query_params.get('count', False) try: query.update(json.loads(query_params.get('query', '{}'))) except ValueError: raise ParseError(_("Invalid query: %(query)s" % {'query': query_params.get('query')})) query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } # if we want the count, we don't kwow to paginate the records. # start and limit are useless then. if count: query_kwargs['count'] = True else: if limit: query_kwargs['limit'] = int(limit) if start: query_kwargs['start'] = int(start) cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) # if we want the count, we only need the first index of the list. if count: return cursor[0] else: return [MongoHelper.to_readable_dict(record) for record in cursor]
def api(request, username=None, id_string=None): """ Returns all results as JSON. If a parameter string is passed, it takes the 'query' parameter, converts this string to a dictionary, an that is then used as a MongoDB query string. NOTE: only a specific set of operators are allow, currently $or and $and. Please send a request if you'd like another operator to be enabled. NOTE: Your query must be valid JSON, double check it here, http://json.parser.online.fr/ E.g. api?query='{"last_name": "Smith"}' """ if request.method == "OPTIONS": response = HttpResponse() add_cors_headers(response) return response helper_auth_helper(request) helper_auth_helper(request) xform, owner = check_and_set_user_and_form(username, id_string, request) if not xform: return HttpResponseForbidden(_(u"Not shared.")) try: args = { "username": username, "id_string": id_string, "query": request.GET.get("query"), "fields": request.GET.get("fields"), "sort": request.GET.get("sort"), } if "start" in request.GET: args["start"] = int(request.GET.get("start")) if "limit" in request.GET: args["limit"] = int(request.GET.get("limit")) if "count" in request.GET: args["count"] = True if int(request.GET.get("count")) > 0 else False cursor = ParsedInstance.query_mongo(**args) except ValueError, e: return HttpResponseBadRequest(e.__str__())
def to_representation(self, obj): if not hasattr(obj, 'xform'): return super(DataInstanceSerializer, self).to_representation(obj) request = self.context.get('request') query_params = (request and request.query_params) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.xform.user.username, obj.xform.id_string), u'_id': obj.pk } query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) records = list(record for record in cursor) return (len(records) and records[0]) or records
def test_owner_can_delete(self): # Test if Form owner can delete # check record exist before delete and after delete count = Instance.objects.filter(deleted_at=None).count() instance = Instance.objects.filter( xform=self.xform).latest('date_created') self.assertEqual(instance.deleted_at, None) # delete params = {'id': instance.id} response = self.client.post(self.delete_url, params) self.assertEqual(response.status_code, 200) self.assertEqual( Instance.objects.filter(deleted_at=None).count(), count - 1) instance = Instance.objects.get(id=instance.id) self.assertTrue(isinstance(instance.deleted_at, datetime)) self.assertNotEqual(instance.deleted_at, None) query = '{"_id": %s}' % instance.id self.data_args.update({"query": query}) after = [r for r in ParsedInstance.query_data(**self.data_args)] self.assertEqual(len(after), count - 1)
def to_native(self, obj): if obj is None: return super(DataInstanceSerializer, self).to_native(obj) request = self.context.get('request') query_params = (request and request.QUERY_PARAMS) or {} query = { ParsedInstance.USERFORM_ID: u'%s_%s' % (obj.xform.user.username, obj.xform.id_string), u'_id': obj.pk } query_kwargs = { 'query': json.dumps(query), 'fields': query_params.get('fields'), 'sort': query_params.get('sort') } cursor = ParsedInstance.query_mongo_minimal(**query_kwargs) records = list(record for record in cursor) return (len(records) and records[0]) or records
def generate_external_export( export_type, username, id_string, export_id=None, token=None, filter_query=None, meta=None, data_id=None): xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].get_dict() if inst else {}] else: instances = ParsedInstance.query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e) ) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string) ) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def generate_external_export(export_type, username, id_string, export_id=None, token=None, filter_query=None, meta=None, data_id=None): xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].get_dict() if inst else {}] else: instances = ParsedInstance.query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e)) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string)) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def test_edited_submission_require_auth(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") # require authentication self.user.profile.require_auth = True self.user.profile.save() num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = ParsedInstance.query_data(**query_args) num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # create a new user alice = self._create_user('alice', 'alice') UserProfile.objects.create(user=alice) auth = DigestAuth('alice', 'alice') # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user assign_perm('report_xform', alice, self.xform) assign_perm('logger.change_xform', alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def _get_data(self): cursor = ParsedInstance.query_mongo(**self.mongo_args) records = list(record for record in cursor) return records
def __build_db_queries(xform_, payload): """ Gets instance ids based on the request payload. Useful to narrow down set of instances for bulk actions Args: xform_ (XForm) payload (dict) Returns: tuple(<dict>, <dict>): PostgreSQL filters, Mongo filters. They are meant to be used respectively with Django Queryset and PyMongo query. """ mongo_query = ParsedInstance.get_base_query(xform_.user.username, xform_.id_string) postgres_query = {'xform_id': xform_.id} instance_ids = None ################################################### # Submissions can be retrieve in 3 different ways # ################################################### # First of all, users can't mix `query` and `submission_ids` in `payload` if all(key_ in payload for key_ in ('query', 'submission_ids')): raise ValidationError({ 'payload': _("`query` and `instance_ids` can't be used together") }) # First scenario / Get submissions based on user's query try: query = payload['query'] except KeyError: pass else: try: query.update(mongo_query) # Overrides `_userform_id` if exists except AttributeError: raise ValidationError({ 'payload': _('Invalid query: %(query)s') % { 'query': json.dumps(query) } }) query_kwargs = {'query': json.dumps(query), 'fields': '["_id"]'} cursor = ParsedInstance.query_mongo_no_paging(**query_kwargs) instance_ids = [record.get('_id') for record in list(cursor)] # Second scenario / Get submissions based on list of ids try: submission_ids = payload['submission_ids'] except KeyError: pass else: try: # Use int() to test if list of integers is valid. instance_ids = [ int(submission_id) for submission_id in submission_ids ] except ValueError: raise ValidationError({ 'payload': _('Invalid submission ids: %(submission_ids)s') % { 'submission_ids': json.dumps(payload['submission_ids']) } }) if instance_ids is not None: # Narrow down queries with list of ids. postgres_query.update({'id__in': instance_ids}) mongo_query.update({'_id': {'$in': instance_ids}}) elif payload.get('confirm', False) is not True: # Third scenario / get all submissions in form, # but confirmation param must be among payload raise NoConfirmationProvidedException() return postgres_query, mongo_query
def test_edited_submission_require_auth(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) # require authentication self.user.profile.require_auth = True self.user.profile.save() num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = ParsedInstance.query_data(**query_args) num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # create a new user alice = self._create_user('alice', 'alice') UserProfile.objects.create(user=alice) auth = DigestAuth('alice', 'alice') # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user assign_perm('report_xform', alice, self.xform) assign_perm('logger.change_xform', alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, start=None, end=None, remove_group_name=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', } xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) records = ParsedInstance.query_data(xform, query=filter_query, start=start, end=end) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = remove_group_name export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__(temp_file.name, records, username, id_string, filter_query, start=start, end=end) except NoRecordsFoundError: pass # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None and start is None and end is None: export.save() return export
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, start=None, end=None, remove_group_name=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', } xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) records = ParsedInstance.query_data(xform, query=filter_query, start=start, end=end) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = remove_group_name export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( temp_file.name, records, username, id_string, filter_query, start=start, end=end ) except NoRecordsFoundError: pass # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None and start is None and end is None: export.save() return export
def modify(self, request, *args, **kwargs): xform = self.get_object() http_status = status.HTTP_200_OK response = {} if request.user.has_perm("validate_xform", xform): owner = xform.user userform_id = "{}_{}".format(owner.username, xform.id_string) query = {ParsedInstance.USERFORM_ID: userform_id} # Query used for MongoDB filter_ = {"xform_id": xform.id} # Filter for Django ORM payload = {} try: payload = json.loads(request.data.get("payload", "{}")) except ValueError: http_status = status.HTTP_400_BAD_REQUEST response = {"detail": _("Invalid payload")} if http_status == status.HTTP_200_OK: new_validation_status_uid = payload.get("validation_status.uid") if new_validation_status_uid is None: http_status = status.HTTP_400_BAD_REQUEST response = {"detail": _("No validation_status.uid provided")} else: # Create new validation_status object new_validation_status = get_validation_status( new_validation_status_uid, xform, request.user.username) # 3 scenarios to update submissions # First scenario / Modify submissions based on user's query if payload.get("query"): # Validate if query is valid. try: query.update(payload.get("query")) except ValueError: raise ParseError(_("Invalid query: %(query)s" % {'query': json.dumps(payload.get("query"))})) query_kwargs = { "query": json.dumps(query), "fields": '["_id"]' } cursor = ParsedInstance.query_mongo_no_paging(**query_kwargs) submissions_ids = [record.get("_id") for record in list(cursor)] filter_.update({"id__in": submissions_ids}) # Second scenario / Modify submissions based on list of ids elif payload.get("submissions_ids"): try: # Use int() to test if list of integers is valid. submissions_ids = payload.get("submissions_ids", []) or_ = {u"$or": [{u"_id": int(submission_id)} for submission_id in submissions_ids]} query.update(or_) except ValueError: raise ParseError(_("Invalid submissions ids: %(submissions_ids)s" % {'submissions_ids': json.dumps(payload.get("submissions_ids"))})) filter_.update({"id__in": submissions_ids}) # Third scenario / Modify all submissions in form, but confirmation param must be among payload elif payload.get("confirm", False) is not True: http_status = status.HTTP_400_BAD_REQUEST response = {"detail": _("No confirmations provided")} # If everything is OK, submit data to DBs if http_status == status.HTTP_200_OK: # Update Postgres & Mongo updated_records_count = Instance.objects.\ filter(**filter_).update(validation_status=new_validation_status) ParsedInstance.bulk_update_validation_statuses(query, new_validation_status) response = {"detail": _("{} submissions have been updated").format(updated_records_count)} return Response(response, http_status) else: raise PermissionDenied(_(u"You do not have validate permissions."))
def modify(self, request, *args, **kwargs): xform = self.get_object() http_status = status.HTTP_200_OK response = {} if request.user.has_perm("validate_xform", xform): owner = xform.user userform_id = "{}_{}".format(owner.username, xform.id_string) query = {ParsedInstance.USERFORM_ID: userform_id} # Query used for MongoDB filter_ = {"xform_id": xform.id} # Filter for Django ORM payload = {} try: payload = json.loads(request.data.get("payload", "{}")) except ValueError: http_status = status.HTTP_400_BAD_REQUEST response = {"detail": _("Invalid payload")} if http_status == status.HTTP_200_OK: if request.data.get("reset"): new_validation_status_uid = {} else: new_validation_status_uid = payload.get("validation_status.uid") if new_validation_status_uid is None: http_status = status.HTTP_400_BAD_REQUEST response = {"detail": _("No validation_status.uid provided")} else: # Create new validation_status object new_validation_status = get_validation_status( new_validation_status_uid, xform, request.user.username) # 3 scenarios to update submissions # First scenario / Modify submissions based on user's query if payload.get("query"): # Validate if query is valid. try: query.update(payload.get("query")) except ValueError: raise ParseError(_("Invalid query: %(query)s" % {'query': json.dumps(payload.get("query"))})) query_kwargs = { "query": json.dumps(query), "fields": '["_id"]' } cursor = ParsedInstance.query_mongo_no_paging(**query_kwargs) submissions_ids = [record.get("_id") for record in list(cursor)] filter_.update({"id__in": submissions_ids}) # Second scenario / Modify submissions based on list of ids elif payload.get("submissions_ids"): try: # Use int() to test if list of integers is valid. submissions_ids = payload.get("submissions_ids", []) or_ = {u"$or": [{u"_id": int(submission_id)} for submission_id in submissions_ids]} query.update(or_) except ValueError: raise ParseError(_("Invalid submissions ids: %(submissions_ids)s" % {'submissions_ids': json.dumps(payload.get("submissions_ids"))})) filter_.update({"id__in": submissions_ids}) # Third scenario / Modify all submissions in form, but confirmation param must be among payload elif payload.get("confirm", False) is not True: http_status = status.HTTP_400_BAD_REQUEST response = {"detail": _("No confirmations provided")} # If everything is OK, submit data to DBs if http_status == status.HTTP_200_OK: # Update Postgres & Mongo updated_records_count = Instance.objects.\ filter(**filter_).update(validation_status=new_validation_status) ParsedInstance.bulk_update_validation_statuses(query, new_validation_status) response = {"detail": _("{} submissions have been updated").format(updated_records_count)} return Response(response, http_status) else: raise PermissionDenied(_(u"You do not have validate permissions."))