def test_edit_updated_geopoint_cache(self): query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args['count'] = False records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]['gps'].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_edit_updated_geopoint_cache(self): query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args['count'] = False records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = ParsedInstance.query_data(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]['gps'].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_edited_submission(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = [r for r in ParsedInstance.query_data(**query_args)] num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) client = DigestClient() client.set_authorization('bob', 'bob', 'Digest') self._make_submission(xml_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def test_edited_submission(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = [r for r in ParsedInstance.query_data(**query_args)] num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") client = DigestClient() client.set_authorization('bob', 'bob', 'Digest') self._make_submission(xml_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def _get_data(self, query, fields, sort, start, limit, is_public_request): try: where, where_params = ParsedInstance._get_where_clause(query) except ValueError as e: raise ParseError(unicode(e)) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (sort or limit or start or fields) and not is_public_request: if self.object_list.count(): xform = self.object_list[0].xform self.object_list = \ ParsedInstance.query_data( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) if not isinstance(self.object_list, types.GeneratorType): page = self.paginate_queryset(self.object_list) serializer = self.get_pagination_serializer(page) else: serializer = self.get_serializer(self.object_list, many=True) page = None return Response(serializer.data)
def _query_data(self, query='{}', start=0, limit=ParsedInstance.DEFAULT_LIMIT, fields='[]', count=False): # ParsedInstance.query_mongo takes params as json strings # so we dumps the fields dictionary count_args = { 'xform': self.xform, 'query': query, 'start': self.start, 'end': self.end, 'fields': '[]', 'sort': '{}', 'count': True } count_object = list(ParsedInstance.query_data(**count_args)) record_count = count_object[0]["count"] if record_count < 1: raise NoRecordsFoundError("No records found for your query") # if count was requested, return the count if count: return record_count else: query_args = { 'xform': self.xform, 'query': query, 'fields': fields, 'start': self.start, 'end': self.end, # TODO: we might want to add this in for the user # to sepcify a sort order 'sort': 'id', 'start_index': start, 'limit': limit, 'count': False } # use ParsedInstance.query_mongo cursor = ParsedInstance.query_data(**query_args) return cursor
def test_owner_can_delete(self): # Test if Form owner can delete # check record exist before delete and after delete count = Instance.objects.filter(deleted_at=None).count() instance = Instance.objects.filter( xform=self.xform).latest('date_created') self.assertEqual(instance.deleted_at, None) # delete params = {'id': instance.id} response = self.client.post(self.delete_url, params) self.assertEqual(response.status_code, 200) self.assertEqual( Instance.objects.filter(deleted_at=None).count(), count - 1) instance = Instance.objects.get(id=instance.id) self.assertTrue(isinstance(instance.deleted_at, datetime)) self.assertNotEqual(instance.deleted_at, None) query = '{"_id": %s}' % instance.id self.data_args.update({"query": query}) after = [r for r in ParsedInstance.query_data(**self.data_args)] self.assertEqual(len(after), count - 1)
def test_edited_submission_require_auth(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") # require authentication self.user.profile.require_auth = True self.user.profile.save() num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = ParsedInstance.query_data(**query_args) num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # create a new user alice = self._create_user('alice', 'alice') UserProfile.objects.create(user=alice) auth = DigestAuth('alice', 'alice') # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user assign_perm('report_xform', alice, self.xform) assign_perm('logger.change_xform', alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def test_edited_submission_require_auth(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) # require authentication self.user.profile.require_auth = True self.user.profile.save() num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = ParsedInstance.query_data(**query_args) num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # create a new user alice = self._create_user('alice', 'alice') UserProfile.objects.create(user=alice) auth = DigestAuth('alice', 'alice') # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user assign_perm('report_xform', alice, self.xform) assign_perm('logger.change_xform', alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 1) cursor = ParsedInstance.query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = ParsedInstance.query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, start=None, end=None, remove_group_name=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', } xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) records = ParsedInstance.query_data(xform, query=filter_query, start=start, end=end) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = remove_group_name export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__(temp_file.name, records, username, id_string, filter_query, start=start, end=end) except NoRecordsFoundError: pass # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None and start is None and end is None: export.save() return export
def generate_external_export(export_type, username, id_string, export_id=None, token=None, filter_query=None, meta=None, data_id=None): xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].get_dict() if inst else {}] else: instances = ParsedInstance.query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e)) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string)) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def generate_export(export_type, extension, username, id_string, export_id=None, filter_query=None, group_delimiter='/', split_select_multiples=True, binary_select_multiples=False, start=None, end=None, remove_group_name=False): """ Create appropriate export object given the export type """ # TODO resolve circular import from onadata.apps.viewer.models.export import Export export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', } xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) records = ParsedInstance.query_data(xform, query=filter_query, start=start, end=end) export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = remove_group_name export_builder.GROUP_DELIMITER = group_delimiter export_builder.SPLIT_SELECT_MULTIPLES = split_select_multiples export_builder.BINARY_SELECT_MULTIPLES = binary_select_multiples export_builder.set_survey(xform.data_dictionary().survey) temp_file = NamedTemporaryFile(suffix=("." + extension)) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( temp_file.name, records, username, id_string, filter_query, start=start, end=end ) except NoRecordsFoundError: pass # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S")) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? storage = get_storage_class()() # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export(xform=xform, export_type=export_type) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # dont persist exports that have a filter if filter_query is None and start is None and end is None: export.save() return export
def generate_external_export( export_type, username, id_string, export_id=None, token=None, filter_query=None, meta=None, data_id=None): xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].get_dict() if inst else {}] else: instances = ParsedInstance.query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e) ) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string) ) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export = Export.objects.create(xform=xform, export_type=export_type) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def _get_data(self): cursor = ParsedInstance.query_data(**self.data_args) records = list(record for record in cursor) return records