def test_edit_updated_geopoint_cache(self): query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args['count'] = False records = query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = query_data(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]['gps'].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_query_filter_by_integer(self): self._publish_transportation_form() self._make_submissions() oldest = Instance.objects.filter(xform=self.xform).first().pk data = [ i.get('_id') for i in query_data(self.xform, query='[{"_id": %s}]' % (oldest)) ] self.assertEqual(len(data), 1) self.assertEqual(data, [oldest]) # with fields data = [ i.get('_id') for i in query_data( self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]') ] self.assertEqual(len(data), 1) self.assertEqual(data, [oldest]) # mongo $gt data = [ i.get('_id') for i in query_data(self.xform, query='{"_id": {"$gt": %s}}' % (oldest), fields='["_id"]') ] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3)
def set_object_list_and_total_count(self, query, fields, sort, start, limit, is_public_request): try: if not is_public_request: xform = self.get_object() where, where_params = get_where_clause(query) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list) self.object_list = \ self.object_list.order_by('pk')[start: limit] self.total_count = self.object_list.count() elif (sort or limit or start or fields) and not is_public_request: try: query = \ filter_queryset_xform_meta_perms_sql(self.get_object(), self.request.user, query) self.object_list = query_data(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) self.total_count = query_data(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields, count=True)[0].get('count') except NoRecordsPermission: self.object_list = [] self.total_count = 0 else: self.total_count = self.object_list.count() if self.total_count and isinstance(self.object_list, QuerySet): self.etag_hash = get_etag_hash_from_query(self.object_list) elif self.total_count: sql, params, records = get_sql_with_params(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) self.etag_hash = get_etag_hash_from_query(records, sql, params) except ValueError as e: raise ParseError(unicode(e)) except DataError as e: raise ParseError(unicode(e))
def test_edit_updated_geopoint_cache(self): query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) # query mongo for the _geopoint field query_args['count'] = False records = query_data(**query_args) self.assertEqual(len(records), 1) # submit the edited instance xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) records = query_data(**query_args) self.assertEqual(len(records), 1) cached_geopoint = records[0][GEOLOCATION] # the cached geopoint should equal the gps field gps = records[0]['gps'].split(" ") self.assertEqual(float(gps[0]), float(cached_geopoint[0])) self.assertEqual(float(gps[1]), float(cached_geopoint[1]))
def test_query_filter_by_datetime_field(self, mock_time): self._publish_transportation_form() now = datetime(2014, 1, 1, tzinfo=utc) times = [ now, now + timedelta(seconds=1), now + timedelta(seconds=2), now + timedelta(seconds=3) ] mock_time.side_effect = times self._make_submissions() atime = None for i in self.xform.instances.all().order_by('-pk'): i.date_created = times.pop() i.save() if atime is None: atime = i.date_created.strftime(MONGO_STRFTIME) # mongo $gt data = [ i.get('_submission_time') for i in query_data(self.xform, query='{"_submission_time": {"$lt": "%s"}}' % (atime), fields='["_submission_time"]') ] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3) self.assertNotIn(atime, data)
def set_object_list(self, query, fields, sort, start, limit, is_public_request): try: enable_etag = True if not is_public_request: xform = self.get_object() self.data_count = xform.num_of_submissions enable_etag = self.data_count <\ SUBMISSION_RETRIEVAL_THRESHOLD where, where_params = get_where_clause(query) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list) self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: query = \ filter_queryset_xform_meta_perms_sql(self.get_object(), self.request.user, query) self.object_list = query_data( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields, json_only=not self.kwargs.get('format') == 'xml') except NoRecordsPermission: self.object_list = [] # ETags are Disabled for XForms with Submissions that surpass # the configured SUBMISSION_RETRIEVAL_THRESHOLD setting if enable_etag: if isinstance(self.object_list, QuerySet): self.etag_hash = get_etag_hash_from_query(self.object_list) else: sql, params, records = get_sql_with_params( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) self.etag_hash = get_etag_hash_from_query( records, sql, params) except ValueError as e: raise ParseError(text(e)) except DataError as e: raise ParseError(text(e))
def _query_data(self, query='{}', start=0, limit=ParsedInstance.DEFAULT_LIMIT, fields='[]', count=False): # query_data takes params as json strings # so we dumps the fields dictionary count_args = { 'xform': self.xform, 'query': query, 'start': self.start, 'end': self.end, 'fields': '[]', 'sort': '{}', 'count': True } count_object = list(query_data(**count_args)) record_count = count_object[0]["count"] if record_count < 1: raise NoRecordsFoundError("No records found for your query") # if count was requested, return the count if count: return record_count else: query_args = { 'xform': self.xform, 'query': query, 'fields': fields, 'start': self.start, 'end': self.end, # TODO: we might want to add this in for the user # to sepcify a sort order 'sort': 'id', 'start_index': start, 'limit': limit, 'count': False } cursor = query_data(**query_args) return cursor
def test_query_data_sort(self): self._publish_transportation_form() self._make_submissions() latest = Instance.objects.filter(xform=self.xform).latest('pk').pk oldest = Instance.objects.filter(xform=self.xform).first().pk data = [i.get('_id') for i in query_data( self.xform, sort='-_id')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field data = [i.get('_id') for i in query_data( self.xform, sort='{"_id": "-1"}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field data = [i.get('_id') for i in query_data( self.xform, sort='{"_id": -1}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest)
def test_query_data_sort(self): self._publish_transportation_form() self._make_submissions() latest = Instance.objects.filter(xform=self.xform).latest('pk').pk oldest = Instance.objects.filter(xform=self.xform).first().pk data = [i.get('_id') for i in query_data( self.xform, sort='-_id')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field data = [i.get('_id') for i in query_data( self.xform, sort='{"_id": "-1"}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest) # sort with a json field data = [i.get('_id') for i in query_data( self.xform, sort='{"_id": -1}')] self.assertEqual(data[0], latest) self.assertEqual(data[len(data) - 1], oldest)
def test_query_filter_by_integer(self): self._publish_transportation_form() self._make_submissions() oldest = Instance.objects.filter(xform=self.xform).first().pk data = [i.get('_id') for i in query_data( self.xform, query='[{"_id": %s}]' % (oldest))] self.assertEqual(len(data), 1) self.assertEqual(data, [oldest]) # with fields data = [i.get('_id') for i in query_data( self.xform, query='{"_id": %s}' % (oldest), fields='["_id"]')] self.assertEqual(len(data), 1) self.assertEqual(data, [oldest]) # mongo $gt data = [i.get('_id') for i in query_data( self.xform, query='{"_id": {"$gt": %s}}' % (oldest), fields='["_id"]')] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3)
def _query_data(self, query='{}', start=0, limit=ParsedInstance.DEFAULT_LIMIT, fields='[]', count=False): # query_data takes params as json strings # so we dumps the fields dictionary count_args = { 'xform': self.xform, 'query': query, 'start': self.start, 'end': self.end, 'fields': '[]', 'sort': '{}', 'count': True } count_object = list(query_data(**count_args)) record_count = count_object[0]["count"] if record_count < 1: raise NoRecordsFoundError("No records found for your query") # if count was requested, return the count if count: return record_count else: query_args = { 'xform': self.xform, 'query': query, 'fields': fields, 'start': self.start, 'end': self.end, # TODO: we might want to add this in for the user # to sepcify a sort order 'sort': 'id', 'start_index': start, 'limit': limit, 'count': False } cursor = query_data(**query_args) return cursor
def test_owner_can_delete(self): # Test if Form owner can delete # check record exist before delete and after delete count = Instance.objects.filter(deleted_at=None).count() instance = Instance.objects.filter( xform=self.xform).latest('date_created') self.assertEqual(instance.deleted_at, None) # delete params = {'id': instance.id} response = self.client.post(self.delete_url, params) self.assertEqual(response.status_code, 200) self.assertEqual( Instance.objects.filter(deleted_at=None).count(), count - 1) instance = Instance.objects.get(id=instance.id) self.assertTrue(isinstance(instance.deleted_at, datetime)) self.assertNotEqual(instance.deleted_at, None) query = '{"_id": %s}' % instance.id self.data_args.update({"query": query}) after = [r for r in query_data(**self.data_args)] self.assertEqual(len(after), count - 1)
def set_object_list( self, query, fields, sort, start, limit, is_public_request): try: if not is_public_request: xform = self.get_object() where, where_params = get_where_clause(query) if where: self.object_list = self.object_list.extra(where=where, params=where_params) if (start and limit or limit) and (not sort and not fields): start = start if start is not None else 0 limit = limit if start is None or start == 0 else start + limit self.object_list = filter_queryset_xform_meta_perms( self.get_object(), self.request.user, self.object_list) self.object_list = self.object_list[start:limit] elif (sort or limit or start or fields) and not is_public_request: try: query = \ filter_queryset_xform_meta_perms_sql(self.get_object(), self.request.user, query) self.object_list = query_data(xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields) except NoRecordsPermission: self.object_list = [] if isinstance(self.object_list, QuerySet): self.etag_hash = get_etag_hash_from_query(self.object_list) else: sql, params, records = get_sql_with_params( xform, query=query, sort=sort, start_index=start, limit=limit, fields=fields ) self.etag_hash = get_etag_hash_from_query(records, sql, params) except ValueError as e: raise ParseError(text(e)) except DataError as e: raise ParseError(text(e))
def test_query_filter_by_datetime_field(self, mock_time): self._publish_transportation_form() now = datetime(2014, 1, 1, tzinfo=utc) times = [now, now + timedelta(seconds=1), now + timedelta(seconds=2), now + timedelta(seconds=3)] mock_time.side_effect = times self._make_submissions() atime = None for i in self.xform.instances.all().order_by('-pk'): i.date_created = times.pop() i.save() if atime is None: atime = i.date_created.strftime(MONGO_STRFTIME) # mongo $gt data = [i.get('_submission_time') for i in query_data( self.xform, query='{"_submission_time": {"$lt": "%s"}}' % (atime), fields='["_submission_time"]')] self.assertEqual(self.xform.instances.count(), 4) self.assertEqual(len(data), 3) self.assertNotIn(atime, data)
def test_edited_submission(self): """ Test submissions that have been edited """ # Delete all previous instance history objects InstanceHistory.objects.all().delete() xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = [r for r in query_data(**query_args)] num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # Take initial instance from DB initial_instance = self.xform.instances.first() # check that '_last_edited' key is not in the json self.assertIsNone(initial_instance.json.get(LAST_EDITED)) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # edited submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") client = DigestClient() client.set_authorization('bob', 'bob', 'Digest') with catch_signal(process_submission) as handler: self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) instance_history_1 = InstanceHistory.objects.first() edited_instance = self.xform.instances.first() self.assertDictEqual(initial_instance.get_dict(), instance_history_1.get_dict()) handler.assert_called_once_with(instance=edited_instance, sender=Instance, signal=ANY) self.assertNotEqual(edited_instance.uuid, instance_history_1.uuid) # check that instance history's submission_date is equal to instance's # date_created - last_edited by default is null for an instance self.assertEquals(edited_instance.date_created, instance_history_1.submission_date) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_edit_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name) instance_before_second_edit = edited_instance xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml") self._make_submission(xml_edit_submission_file_path) cursor = query_data(**query_args) record = cursor[0] edited_instance = self.xform.instances.first() instance_history_2 = InstanceHistory.objects.last() self.assertEquals(instance_before_second_edit.last_edited, instance_history_2.submission_date) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) self.assertEqual(record['name'], 'Tom and Jerry') self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 2) # submitting original submission is treated as a duplicate # does not add a new record # does not change data self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 202) self.assertEqual(Instance.objects.count(), num_instances + 1) self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 2)
def generate_export(export_type, xform, export_id=None, options=None): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') show_choice_labels: boolean flag language: language labels as in the XLSForm/XForm """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get( "group_delimiter", DEFAULT_GROUP_DELIMITER ) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True ) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False ) export_builder.INCLUDE_LABELS = options.get('include_labels', False) include_reviews = options.get('include_reviews', False) export_builder.INCLUDE_LABELS_ONLY = options.get( 'include_labels_only', False ) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get( "repeat_index_tags", DEFAULT_INDEX_TAGS ) export_builder.SHOW_CHOICE_LABELS = options.get('show_choice_labels', False) export_builder.language = options.get('language') # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.INCLUDE_REVIEWS = include_reviews export_builder.set_survey(xform.survey, xform, include_reviews=include_reviews) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__( temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records ) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % ( id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join( username, 'exports', id_string, export_type, filename) # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export
def generate_export(export_type, xform, export_id=None, options=None, retries=0): """ Create appropriate export object given the export type. param: export_type param: xform params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. binary_select_multiples: boolean flag end: end offset ext: export extension type dataview_pk: dataview pk group_delimiter: "/" or "." query: filter_query for custom queries remove_group_name: boolean flag split_select_multiples: boolean flag index_tag: ('[', ']') or ('_', '_') """ username = xform.user.username id_string = xform.id_string end = options.get("end") extension = options.get("extension", export_type) filter_query = options.get("query") remove_group_name = options.get("remove_group_name", False) start = options.get("start") export_type_func_map = { Export.XLS_EXPORT: 'to_xls_export', Export.CSV_EXPORT: 'to_flat_csv_export', Export.CSV_ZIP_EXPORT: 'to_zipped_csv', Export.SAV_ZIP_EXPORT: 'to_zipped_sav', Export.GOOGLE_SHEETS_EXPORT: 'to_google_sheets', } if xform is None: xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) dataview = None if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) records = dataview.query_data(dataview, all_data=True, filter_query=filter_query) total_records = dataview.query_data(dataview, count=True)[0].get('count') else: records = query_data(xform, query=filter_query, start=start, end=end) if filter_query: total_records = query_data(xform, query=filter_query, start=start, end=end, count=True)[0].get('count') else: total_records = xform.num_of_submissions if isinstance(records, QuerySet): records = records.iterator() export_builder = ExportBuilder() export_builder.TRUNCATE_GROUP_TITLE = True \ if export_type == Export.SAV_ZIP_EXPORT else remove_group_name export_builder.GROUP_DELIMITER = options.get("group_delimiter", DEFAULT_GROUP_DELIMITER) export_builder.SPLIT_SELECT_MULTIPLES = options.get( "split_select_multiples", True) export_builder.BINARY_SELECT_MULTIPLES = options.get( "binary_select_multiples", False) export_builder.INCLUDE_LABELS = options.get('include_labels', False) export_builder.INCLUDE_LABELS_ONLY = options.get('include_labels_only', False) export_builder.INCLUDE_HXL = options.get('include_hxl', False) export_builder.INCLUDE_IMAGES \ = options.get("include_images", settings.EXPORT_WITH_IMAGE_DEFAULT) export_builder.VALUE_SELECT_MULTIPLES = options.get( 'value_select_multiples', False) export_builder.REPEAT_INDEX_TAGS = options.get("repeat_index_tags", DEFAULT_INDEX_TAGS) # 'win_excel_utf8' is only relevant for CSV exports if 'win_excel_utf8' in options and export_type != Export.CSV_EXPORT: del options['win_excel_utf8'] export_builder.set_survey(xform.survey, xform) temp_file = NamedTemporaryFile(suffix=("." + extension)) columns_with_hxl = export_builder.INCLUDE_HXL and get_columns_with_hxl( xform.survey_elements) # get the export function by export type func = getattr(export_builder, export_type_func_map[export_type]) try: func.__call__(temp_file.name, records, username, id_string, filter_query, start=start, end=end, dataview=dataview, xform=xform, options=options, columns_with_hxl=columns_with_hxl, total_records=total_records) except NoRecordsFoundError: pass except SPSSIOError as e: export = get_or_create_export(export_id, xform, export_type, options) export.error_message = str(e) export.internal_status = Export.FAILED export.save() report_exception("SAV Export Failure", e, sys.exc_info()) return export # generate filename basename = "%s_%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f")) if remove_group_name: # add 'remove group name' flag to filename basename = "{}-{}".format(basename, GROUPNAME_REMOVED_FLAG) if dataview: basename = "{}-{}".format(basename, DATAVIEW_EXPORT) filename = basename + "." + extension # check filename is unique while not Export.is_filename_unique(xform, filename): filename = increment_index_in_filename(filename) file_path = os.path.join(username, 'exports', id_string, export_type, filename) # TODO: if s3 storage, make private - how will we protect local storage?? # seek to the beginning as required by storage classes temp_file.seek(0) export_filename = default_storage.save(file_path, File(temp_file, file_path)) temp_file.close() dir_name, basename = os.path.split(export_filename) # get or create export object export = get_or_create_export(export_id, xform, export_type, options) export.filedir = dir_name export.filename = basename export.internal_status = Export.SUCCESSFUL # do not persist exports that have a filter # Get URL of the exported sheet. if export_type == Export.GOOGLE_SHEETS_EXPORT: export.export_url = export_builder.url # if we should create a new export is true, we should not save it if start is None and end is None: export.save() return export
def generate_external_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates external export using ONA data through an external service. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. data_id: instance id query: filter_query for custom queries meta: metadata associated with external export token: authentication key required by external service """ data_id = options.get("data_id") filter_query = options.get("query") meta = options.get("meta") token = options.get("token") if xform is None: xform = XForm.objects.get(user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].json if inst else {}] else: instances = query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e)) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string)) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) export = Export.objects.create(xform=xform, export_type=export_type, options=export_options) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join(os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = { "extension": Export.ZIP_EXPORT, "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}' } filter_query = options.get("query") instance_ids = query_data(self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() filtered_attachments = Attachment.objects.filter( instance__xform_id=self.xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) self.assertNotEqual(Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query options.pop('query') export1 = generate_attachments_zip_export(Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export1.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def _get_data(self): cursor = query_data(**self.data_args) records = list(record for record in cursor) return records
def generate_external_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates external export using ONA data through an external service. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. data_id: instance id query: filter_query for custom queries meta: metadata associated with external export token: authentication key required by external service """ data_id = options.get("data_id") filter_query = options.get("query") meta = options.get("meta") token = options.get("token") if xform is None: xform = XForm.objects.get( user__username__iexact=username, id_string__iexact=id_string) user = User.objects.get(username=username) server, name = _get_server_from_metadata(xform, meta, token) # dissect the url parsed_url = urlparse(server) token = parsed_url.path[5:] ser = parsed_url.scheme + '://' + parsed_url.netloc # Get single submission data if data_id: inst = Instance.objects.filter(xform__user=user, xform__id_string=id_string, deleted_at=None, pk=data_id) instances = [inst[0].json if inst else {}] else: instances = query_data(xform, query=filter_query) records = _get_records(instances) status_code = 0 if records and server: try: client = Client(ser) response = client.xls.create(token, json.dumps(records)) if hasattr(client.xls.conn, 'last_response'): status_code = client.xls.conn.last_response.status_code except Exception as e: raise J2XException( u"J2X client could not generate report. Server -> {0}," u" Error-> {1}".format(server, e) ) else: if not server: raise J2XException(u"External server not set") elif not records: raise J2XException( u"No record to export. Form -> {0}".format(id_string) ) # get or create export object if export_id: export = Export.objects.get(id=export_id) else: export_options = get_export_options(options) export = Export.objects.create(xform=xform, export_type=export_type, options=export_options) export.export_url = response if status_code == 201: export.internal_status = Export.SUCCESSFUL export.filename = name + '-' + response[5:] if name else response[5:] export.export_url = ser + response else: export.internal_status = Export.FAILED export.save() return export
def generate_attachments_zip_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates zip export of attachments. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. ext: File extension of the generated export """ export_type = options.get("extension", export_type) filter_query = options.get("query") if xform is None: xform = XForm.objects.get(user__username=username, id_string=id_string) if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ rec.get('_id') for rec in dataview.query_data( dataview, all_data=True, filter_query=filter_query) ], instance__deleted_at__isnull=True) else: instance_ids = query_data(xform, fields='["_id"]', query=filter_query) attachments = Attachment.objects.filter( instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter(instance__xform_id__in=[ i for i in xform.mergedxform.xforms.filter( deleted_at__isnull=True).values_list('id', flat=True) ]).filter(instance_id__in=[i_id['_id'] for i_id in instance_ids]) else: attachments = attachments.filter( instance__xform_id=xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) filename = "%s_%s.%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), export_type.lower()) file_path = os.path.join(username, 'exports', id_string, export_type, filename) zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: temp_file = builtins.open(zip_file.name, 'rb') filename = default_storage.save(file_path, File(temp_file, file_path)) finally: temp_file.close() finally: if zip_file: zip_file.close() export = get_or_create_export(export_id, xform, export_type, options) export.filedir, export.filename = os.path.split(filename) export.internal_status = Export.SUCCESSFUL export.save() return export
def test_edited_submission_require_auth(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) # require authentication self.user.profile.require_auth = True self.user.profile.save() num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = query_data(**query_args) num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # create a new user alice = self._create_user('alice', 'alice') UserProfile.objects.create(user=alice) auth = DigestAuth('alice', 'alice') # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user assign_perm('report_xform', alice, self.xform) assign_perm('logger.change_xform', alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 1) cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def test_generate_filtered_attachments_zip_export(self): """Test media zip file export filters attachments""" filenames = [ 'OSMWay234134797.osm', 'OSMWay34298972.osm', ] osm_fixtures_dir = os.path.realpath( os.path.join( os.path.dirname(api_tests.__file__), 'fixtures', 'osm')) paths = [ os.path.join(osm_fixtures_dir, filename) for filename in filenames ] xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') self._publish_xls_file_and_set_xform(xlsform_path) submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') count = Attachment.objects.filter(extension='osm').count() self._make_submission_w_attachment(submission_path, paths) self._make_submission_w_attachment(submission_path, paths) self.assertTrue( Attachment.objects.filter(extension='osm').count() > count) options = { "extension": Export.ZIP_EXPORT, "query": u'{"_submission_time": {"$lte": "2019-01-13T00:00:00"}}'} filter_query = options.get("query") instance_ids = query_data( self.xform, fields='["_id"]', query=filter_query) export = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() filtered_attachments = Attachment.objects.filter( instance__xform_id=self.xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) self.assertNotEqual( Attachment.objects.count(), filtered_attachments.count()) for a in filtered_attachments: self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir) # export with no query options.pop('query') export1 = generate_attachments_zip_export( Export.ZIP_EXPORT, self.user.username, self.xform.id_string, None, options) self.assertTrue(export1.is_successful) temp_dir = tempfile.mkdtemp() zip_file = zipfile.ZipFile(default_storage.path(export1.filepath), "r") zip_file.extractall(temp_dir) zip_file.close() for a in Attachment.objects.all(): self.assertTrue( os.path.exists(os.path.join(temp_dir, a.media_file.name))) shutil.rmtree(temp_dir)
def test_edited_submission(self): """ Test submissions that have been edited """ # Delete all previous instance history objects InstanceHistory.objects.all().delete() xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml" ) num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = [r for r in query_data(**query_args)] num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # Take initial instance from DB initial_instance = self.xform.instances.first() # check that '_last_edited' key is not in the json self.assertIsNone(initial_instance.json.get(LAST_EDITED)) # no new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # edited submission xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml" ) client = DigestClient() client.set_authorization('bob', 'bob', 'Digest') with catch_signal(process_submission) as handler: self._make_submission(xml_edit_submission_file_path, client=client) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 1) instance_history_1 = InstanceHistory.objects.first() edited_instance = self.xform.instances.first() self.assertDictEqual(initial_instance.get_dict(), instance_history_1.get_dict()) handler.assert_called_once_with(instance=edited_instance, sender=Instance, signal=ANY) self.assertNotEqual(edited_instance.uuid, instance_history_1.uuid) # check that instance history's submission_date is equal to instance's # date_created - last_edited by default is null for an instance self.assertEquals(edited_instance.date_created, instance_history_1.submission_date) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_edit_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(r"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name) instance_before_second_edit = edited_instance xml_edit_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited_again.xml" ) self._make_submission(xml_edit_submission_file_path) cursor = query_data(**query_args) record = cursor[0] edited_instance = self.xform.instances.first() instance_history_2 = InstanceHistory.objects.last() self.assertEquals(instance_before_second_edit.last_edited, instance_history_2.submission_date) # check that '_last_edited' key is not in the json self.assertIn(LAST_EDITED, edited_instance.json) self.assertEqual(record['name'], 'Tom and Jerry') self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 2) # submitting original submission is treated as a duplicate # does not add a new record # does not change data self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 202) self.assertEqual(Instance.objects.count(), num_instances + 1) self.assertEqual( InstanceHistory.objects.count(), num_instances_history + 2)
def test_edited_submission_require_auth(self): """ Test submissions that have been edited """ xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid.xml") # require authentication self.user.profile.require_auth = True self.user.profile.save() num_instances_history = InstanceHistory.objects.count() num_instances = Instance.objects.count() query_args = { 'xform': self.xform, 'query': '{}', 'fields': '[]', 'count': True } cursor = query_data(**query_args) num_data_instances = cursor[0]['count'] # make first submission self._make_submission(xml_submission_file_path) self.assertEqual(self.response.status_code, 201) self.assertEqual(Instance.objects.count(), num_instances + 1) # no new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history) # check count of mongo instances after first submission cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # create a new user alice = self._create_user('alice', 'alice') UserProfile.objects.create(user=alice) auth = DigestAuth('alice', 'alice') # edited submission xml_submission_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "fixtures", "tutorial", "instances", "tutorial_2012-06-27_11-27-53_w_uuid_edited.xml") self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 403) # assign report perms to user assign_perm('report_xform', alice, self.xform) assign_perm('logger.change_xform', alice, self.xform) self._make_submission(xml_submission_file_path, auth=auth) self.assertEqual(self.response.status_code, 201) # we must have the same number of instances self.assertEqual(Instance.objects.count(), num_instances + 1) # should be a new record in instances history self.assertEqual(InstanceHistory.objects.count(), num_instances_history + 1) cursor = query_data(**query_args) self.assertEqual(cursor[0]['count'], num_data_instances + 1) # make sure we edited the mongo db record and NOT added a new row query_args['count'] = False cursor = query_data(**query_args) record = cursor[0] with open(xml_submission_file_path, "r") as f: xml_str = f.read() xml_str = clean_and_parse_xml(xml_str).toxml() edited_name = re.match(ur"^.+?<name>(.+?)</name>", xml_str).groups()[0] self.assertEqual(record['name'], edited_name)
def generate_attachments_zip_export(export_type, username, id_string, export_id=None, options=None, xform=None): """ Generates zip export of attachments. param: export_type params: username: logged in username params: id_string: xform id_string params: export_id: ID of export object associated with the request param: options: additional parameters required for the lookup. ext: File extension of the generated export """ export_type = options.get("extension", export_type) filter_query = options.get("query") if xform is None: xform = XForm.objects.get(user__username=username, id_string=id_string) if options.get("dataview_pk"): dataview = DataView.objects.get(pk=options.get("dataview_pk")) attachments = Attachment.objects.filter( instance_id__in=[ rec.get('_id') for rec in dataview.query_data( dataview, all_data=True, filter_query=filter_query)], instance__deleted_at__isnull=True) else: instance_ids = query_data(xform, fields='["_id"]', query=filter_query) attachments = Attachment.objects.filter( instance__deleted_at__isnull=True) if xform.is_merged_dataset: attachments = attachments.filter( instance__xform_id__in=[ i for i in xform.mergedxform.xforms.filter( deleted_at__isnull=True).values_list( 'id', flat=True)]).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) else: attachments = attachments.filter( instance__xform_id=xform.pk).filter( instance_id__in=[i_id['_id'] for i_id in instance_ids]) filename = "%s_%s.%s" % (id_string, datetime.now().strftime("%Y_%m_%d_%H_%M_%S"), export_type.lower()) file_path = os.path.join( username, 'exports', id_string, export_type, filename) zip_file = None try: zip_file = create_attachments_zipfile(attachments) try: temp_file = builtins.open(zip_file.name, 'rb') filename = default_storage.save( file_path, File(temp_file, file_path)) finally: temp_file.close() finally: if zip_file: zip_file.close() export = get_or_create_export(export_id, xform, export_type, options) export.filedir, export.filename = os.path.split(filename) export.internal_status = Export.SUCCESSFUL export.save() return export