示例#1
0
 def test_delete_oldest_export_on_limit(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     # create first export
     first_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
         self.xform.id_string)
     self.assertIsNotNone(first_export.pk)
     # create exports that exceed set limit
     for i in range(Export.MAX_EXPORTS):
         generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
             self.xform.id_string)
     # first export should be deleted
     exports = Export.objects.filter(id=first_export.id)
     self.assertEqual(len(exports), 0)
示例#2
0
def create_csv_export(username, id_string, export_id, query=None,
                      group_delimiter='/', split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_EXPORT, 'csv', username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except NoRecordsFoundError:
        # not much we can do but we don't want to report this as the user
        # should not even be on this page if the survey has no records
        export.internal_status = Export.FAILED
        export.save()
    except Exception as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("CSV Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e, sys.exc_info())
        raise
    else:
        return gen_export.id
示例#3
0
 def test_csv_export_output(self):
     
     
     #time.sleep(3)
     
     path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
     self._publish_xls_file_and_set_xform(path)
     
     # For some reason we need to sleep here so everything gets cleaned up
     time.sleep(3)
     
     path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
     self._make_submission(
         path, forced_submission_time=self._submission_time)
     # test csv
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              'tutorial_w_repeats')
     storage = get_storage_class()()
     self.assertTrue(storage.exists(export.filepath))
     path, ext = os.path.splitext(export.filename)
     self.assertEqual(ext, '.csv')
     with open(os.path.join(
             self.fixture_dir, 'tutorial_w_repeats.csv')) as f1:
         with storage.open(export.filepath) as f2:
             expected_content = f1.read()
             actual_content = f2.read()
             self.assertEquals(actual_content, expected_content)
示例#4
0
def create_csv_export(username,
                      id_string,
                      export_id,
                      query=None,
                      group_delimiter='/',
                      split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(Export.CSV_EXPORT, 'csv', username,
                                     id_string, export_id, query,
                                     group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception(
            "CSV Export Exception: Export ID - "
            "%(export_id)s, /%(username)s/%(id_string)s" % details, e,
            sys.exc_info())
        raise
    else:
        return gen_export.id
示例#5
0
 def test_csv_nested_repeat_output(self):
     path = os.path.join(self.fixture_dir, 'double_repeat.xls')
     self._publish_xls_file(path)
     path = os.path.join(self.fixture_dir, 'instance.xml')
     self._make_submission(path,
                           forced_submission_time=self._submission_time)
     self.maxDiff = None
     dd = DataDictionary.objects.all()[0]
     xpaths = [
         u'/double_repeat/bed_net[1]/member[1]/name',
         u'/double_repeat/bed_net[1]/member[2]/name',
         u'/double_repeat/bed_net[2]/member[1]/name',
         u'/double_repeat/bed_net[2]/member[2]/name',
         u'/double_repeat/meta/instanceID'
     ]
     self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths)
     # test csv
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              'double_repeat')
     storage = get_storage_class()()
     self.assertTrue(storage.exists(export.filepath))
     path, ext = os.path.splitext(export.filename)
     self.assertEqual(ext, '.csv')
     with open(os.path.join(self.fixture_dir, 'export.csv')) as f1:
         with storage.open(export.filepath) as f2:
             expected_content = f1.read()
             actual_content = f2.read()
             self.assertEquals(actual_content, expected_content)
示例#6
0
def create_xls_export(username, id_string, export_id, query=None,
                      force_xlsx=True, group_delimiter='/',
                      split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    ext = 'xls' if not force_xlsx else 'xlsx'

    export = Export.objects.get(id=export_id)
    # though export is not available when for has 0 submissions, we
    # catch this since it potentially stops celery
    try:
        gen_export = generate_export(
            Export.XLS_EXPORT, ext, username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception("XLS Export Exception: Export ID - "
                         "%(export_id)s, /%(username)s/%(id_string)s"
                         % details, e, sys.exc_info())
        #raise for now to let celery know we failed
        # - doesnt seem to break celery`
        raise
    else:
        return gen_export.id
示例#7
0
 def test_csv_nested_repeat_output(self):
     path = os.path.join(self.fixture_dir, 'double_repeat.xls')
     self._publish_xls_file(path)
     path = os.path.join(self.fixture_dir, 'instance.xml')
     self._make_submission(
         path, forced_submission_time=self._submission_time)
     self.maxDiff = None
     dd = DataDictionary.objects.all()[0]
     xpaths = [
         u'/double_repeat/bed_net[1]/member[1]/name',
         u'/double_repeat/bed_net[1]/member[2]/name',
         u'/double_repeat/bed_net[2]/member[1]/name',
         u'/double_repeat/bed_net[2]/member[2]/name',
         u'/double_repeat/meta/instanceID'
     ]
     self.assertEquals(dd.xpaths(repeat_iterations=2), xpaths)
     # test csv
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              'double_repeat')
     storage = get_storage_class()()
     self.assertTrue(storage.exists(export.filepath))
     path, ext = os.path.splitext(export.filename)
     self.assertEqual(ext, '.csv')
     with open(os.path.join(self.fixture_dir, 'export.csv')) as f1:
         with storage.open(export.filepath) as f2:
             expected_content = f1.read()
             actual_content = f2.read()
             self.assertEquals(actual_content, expected_content)
示例#8
0
def create_csv_zip_export(username,
                          id_string,
                          export_id,
                          query=None,
                          group_delimiter='/',
                          split_select_multiples=True):
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(Export.CSV_ZIP_EXPORT, 'zip', username,
                                     id_string, export_id, query,
                                     group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception(
            "CSV ZIP Export Exception: Export ID - "
            "%(export_id)s, /%(username)s/%(id_string)s" % details, e,
            sys.exc_info())
        raise
    else:
        return gen_export.id
示例#9
0
文件: views.py 项目: rjawahar/formhub
def data_export(request, username, id_string, export_type):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    helper_auth_helper(request)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    extension = export_type

    # check if we should force xlsx
    force_xlsx = request.GET.get('xls') != 'true'
    if export_type == Export.XLS_EXPORT and force_xlsx:
        extension = 'xlsx'

    audit = {
        "xform": xform.id_string,
        "export_type": export_type
    }
    # check if we need to re-generate,
    # we always re-generate if a filter is specified
    if should_create_new_export(xform, export_type) or query:
        try:
            export = generate_export(
                export_type, extension, username, id_string, None, query)
            audit_log(
                Actions.EXPORT_CREATED, request.user, owner,
                _("Created %(export_type)s export on '%(id_string)s'.") %
                {
                    'id_string': xform.id_string,
                    'export_type': export_type.upper()
                }, audit, request)
        except NoRecordsFoundError:
            return HttpResponseNotFound(_("No records found to export"))
    else:
        export = newset_export_for(xform, export_type)

    # log download as well
    audit_log(
        Actions.EXPORT_DOWNLOADED, request.user, owner,
        _("Downloaded %(export_type)s export on '%(id_string)s'.") %
        {
            'id_string': xform.id_string,
            'export_type': export_type.upper()
        }, audit, request)

    if not export.filename:
        # tends to happen when using newset_export_for.
        return HttpResponseNotFound("File does not exist!")
    # get extension from file_path, exporter could modify to
    # xlsx if it exceeds limits
    path, ext = os.path.splitext(export.filename)
    ext = ext[1:]
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name(
        Export.EXPORT_MIMES[ext], id_string, extension=ext,
        file_path=export.filepath)
    return response
示例#10
0
 def test_delete_file_on_export_delete(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
                              self.xform.id_string)
     storage = get_storage_class()()
     self.assertTrue(storage.exists(export.filepath))
     # delete export object
     export.delete()
     self.assertFalse(storage.exists(export.filepath))
示例#11
0
 def test_export_progress_output(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     # create exports
     for i in range(2):
         generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
             self.xform.id_string)
     self.assertEqual(Export.objects.count(), 2)
     # progress for multiple exports
     progress_url = reverse(export_progress, kwargs={
         'username': self.user.username,
         'id_string': self.xform.id_string,
         'export_type': 'xls'
     })
     get_data = {'export_ids': [e.id for e in Export.objects.all()]}
     response = self.client.get(progress_url, get_data)
     content = json.loads(response.content)
     self.assertEqual(len(content), 2)
     self.assertEqual(sorted(['url', 'export_id', 'complete', 'filename']),
         sorted(content[0].keys()))
示例#12
0
 def test_last_submission_time_empty(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     # create export
     export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
         self.xform.id_string)
     # set time of last submission to None
     export.time_of_last_submission = None
     export.save()
     self.assertTrue(Export.exports_outdated(xform=self.xform,
         export_type=Export.XLS_EXPORT))
示例#13
0
def create_csv_export(username, id_string, query=None,
                      export_id=None):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        export = generate_export(Export.CSV_EXPORT, 'csv', username, id_string,
            export_id, query)
    except Exception:
        return None
    else:
        return export
示例#14
0
    def test_create_export(self):
        self._publish_transportation_form_and_submit_instance()
        storage = get_storage_class()()
        # test xls
        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
            self.xform.id_string)
        self.assertTrue(storage.exists(export.filepath))
        path, ext = os.path.splitext(export.filename)
        self.assertEqual(ext, '.xls')

        # test csv
        export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
            self.xform.id_string)
        self.assertTrue(storage.exists(export.filepath))
        path, ext = os.path.splitext(export.filename)
        self.assertEqual(ext, '.csv')

        # test xls with existing export_id
        existing_export = Export.objects.create(xform=self.xform,
            export_type=Export.XLS_EXPORT)
        export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
            self.xform.id_string, existing_export.id)
        self.assertEqual(existing_export.id, export.id)
示例#15
0
 def test_export_download_url(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              self.xform.id_string)
     csv_export_url = reverse(export_download, kwargs={
         "username": self.user.username,
         "id_string": self.xform.id_string,
         "export_type": Export.CSV_EXPORT,
         "filename": export.filename
     })
     response = self.client.get(csv_export_url)
     self.assertEqual(response.status_code, 200)
     # test xls
     export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
         self.xform.id_string)
     xls_export_url = reverse(export_download, kwargs={
         "username": self.user.username,
         "id_string": self.xform.id_string,
         "export_type": Export.XLS_EXPORT,
         "filename": export.filename
     })
     response = self.client.get(xls_export_url)
     self.assertEqual(response.status_code, 200)
示例#16
0
文件: tasks.py 项目: rjawahar/formhub
def create_csv_zip_export(username, id_string, export_id, query=None,
                          group_delimiter='/', split_select_multiples=True):
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_ZIP_EXPORT, 'zip', username, id_string, export_id, query,
            group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        raise
    else:
        return gen_export.id
示例#17
0
def create_csv_export(username, id_string, export_id, query=None):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state

    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_EXPORT, 'csv', username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        raise
    else:
        return gen_export.id
示例#18
0
 def test_allow_export_download_for_basic_auth(self):
     extra = {
         'HTTP_AUTHORIZATION': http_auth_string(self.login_username,
             self.login_password)
     }
     # create export
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              self.xform.id_string)
     self.assertTrue(isinstance(export, Export))
     url = reverse(export_download, kwargs={
         'username': self.user.username,
         'id_string': self.xform.id_string,
         'export_type': export.export_type,
         'filename': export.filename
     })
     response = self.anon.get(url, **extra)
     self.assertEqual(response.status_code, 200)
示例#19
0
 def test_duplicate_export_filename_is_renamed(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     # create an export object in the db
     # TODO: only works if the time we time we generate the basename is exact to the second with the time the 2nd export is created
     basename = "%s_%s" % (self.xform.id_string,
                           datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S"))
     filename = basename + ".csv"
     export = Export.objects.create(xform=self.xform,
         export_type=Export.CSV_EXPORT, filename=filename)
     # 2nd export
     export_2 = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                                self.xform.id_string)
     if export.created_on.timetuple() == export_2.created_on.timetuple():
         new_filename = increment_index_in_filename(filename)
         self.assertEqual(new_filename, export_2.filename)
     else:
         stdout.write("duplicate export filename test skipped because export times differ.")
示例#20
0
 def test_allow_export_download_for_basic_auth(self):
     extra = {
         'HTTP_AUTHORIZATION':
         http_auth_string(self.login_username, self.login_password)
     }
     # create export
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              self.xform.id_string)
     self.assertTrue(isinstance(export, Export))
     url = reverse(export_download,
                   kwargs={
                       'username': self.user.username,
                       'id_string': self.xform.id_string,
                       'export_type': export.export_type,
                       'filename': export.filename
                   })
     response = self.anon.get(url, **extra)
     self.assertEqual(response.status_code, 200)
示例#21
0
 def test_delete_export_url(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     # create export
     export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
         self.xform.id_string)
     exports = Export.objects.filter(id=export.id)
     self.assertEqual(len(exports), 1)
     delete_url = reverse(delete_export, kwargs={
         'username': self.user.username,
         'id_string': self.xform.id_string,
         'export_type': 'xls'
     })
     post_data = {'export_id': export.id}
     response = self.client.post(delete_url, post_data)
     self.assertEqual(response.status_code, 302)
     exports = Export.objects.filter(id=export.id)
     self.assertEqual(len(exports), 0)
示例#22
0
 def test_404_on_export_io_error(self):
     """
     Test that we return a 404 when the response_with_mimetype_and_name encounters an IOError
     """
     self._publish_transportation_form()
     self._submit_transport_instance()
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              self.xform.id_string)
     export_url = reverse(export_download, kwargs={
         "username": self.user.username,
         "id_string": self.xform.id_string,
         "export_type": Export.CSV_EXPORT,
         "filename": export.filename
     })
     # delete the export
     export.delete()
     # access the export
     response = self.client.get(export_url)
     self.assertEqual(response.status_code, 404)
示例#23
0
 def test_csv_export_output(self):
     path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xls')
     self._publish_xls_file_and_set_xform(path)
     path = os.path.join(self.fixture_dir, 'tutorial_w_repeats.xml')
     self._make_submission(path,
                           forced_submission_time=self._submission_time)
     # test csv
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              'tutorial_w_repeats')
     storage = get_storage_class()()
     self.assertTrue(storage.exists(export.filepath))
     path, ext = os.path.splitext(export.filename)
     self.assertEqual(ext, '.csv')
     with open(os.path.join(self.fixture_dir,
                            'tutorial_w_repeats.csv')) as f1:
         with storage.open(export.filepath) as f2:
             expected_content = f1.read()
             actual_content = f2.read()
             self.assertEquals(actual_content, expected_content)
示例#24
0
def csv_export(request, username, id_string):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    ext = Export.CSV_EXPORT

    try:
        export = generate_export(Export.CSV_EXPORT, ext, username, id_string,
            None, query)
    except NoRecordsFoundError:
        return HttpResponse(_("No records found to export"))
    else:
        if request.GET.get('raw'):
            id_string = None
        response = response_with_mimetype_and_name(
            Export.EXPORT_MIMES[ext], id_string, extension=ext,
            file_path=export.filepath)
        return response
示例#25
0
def create_xls_export(username, id_string, export_id, query=None,
                      force_xlsx=False):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    ext = 'xls' if not force_xlsx else 'xlsx'

    export = Export.objects.get(id=export_id)
    # though export is not available when for has 0 submissions, we
    # catch this since it potentially stops celery
    try:
        gen_export = generate_export(
            Export.XLS_EXPORT, ext, username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        #raise for now to let celery know we failed
        # - doesnt seem to break celery
        raise
    else:
        return gen_export.id
示例#26
0
文件: tasks.py 项目: rrfenton/formhub
def create_csv_zip_export(username, id_string, export_id, query=None, group_delimiter="/", split_select_multiples=True):
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when for has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(
            Export.CSV_ZIP_EXPORT, "zip", username, id_string, export_id, query, group_delimiter, split_select_multiples
        )
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {"export_id": export_id, "username": username, "id_string": id_string}
        report_exception(
            "CSV ZIP Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details,
            e,
            sys.exc_info(),
        )
        raise
    else:
        return gen_export.id
示例#27
0
def create_json_export(username, id_string, export_id, query=None):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    export = Export.objects.get(id=export_id)
    try:
        # though export is not available when form has 0 submissions, we
        # catch this since it potentially stops celery
        gen_export = generate_export(Export.JSON_EXPORT, "json", username, id_string, export_id, query)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {"export_id": export_id, "username": username, "id_string": id_string}
        report_exception(
            "JSON Export Exception: Export ID - " "%(export_id)s, /%(username)s/%(id_string)s" % details,
            e,
            sys.exc_info(),
        )
        raise
    else:
        return gen_export.id
示例#28
0
def xls_export(request, username, id_string):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    helper_auth_helper(request)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    force_xlsx = request.GET.get('xlsx') == 'true'
    ext = 'xls' if not force_xlsx else 'xlsx'
    try:
        export = generate_export(
            Export.XLS_EXPORT, ext, username, id_string, None, query)
    except NoRecordsFoundError:
        return HttpResponse(_("No records found to export"))
    else:
        audit = {
            "xform": xform.id_string,
            "export_type": Export.XLS_EXPORT
        }
        audit_log(Actions.EXPORT_CREATED, request.user, owner,
            _("Created XLS export on '%(id_string)s'.") %\
            {
                'id_string': xform.id_string,
            }, audit, request)
        # log download as well
        audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner,
            _("Downloaded XLS export on '%(id_string)s'.") %\
            {
                'id_string': xform.id_string,
            }, audit, request)
        # get extension from file_path, exporter could modify to
        # xlsx if it exceeds limits
        path, ext = os.path.splitext(export.filename)
        ext = ext[1:]
        if request.GET.get('raw'):
            id_string = None
        response = response_with_mimetype_and_name(
            Export.EXPORT_MIMES[ext], id_string, extension=ext,
            file_path=export.filepath)
        return response
示例#29
0
 def test_dotted_fields_csv_export_output(self):
     path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone',
                         'userone_with_dot_name_fields.xls')
     self._publish_xls_file_and_set_xform(path)
     path = os.path.join(os.path.dirname(__file__), 'fixtures', 'userone',
                         'userone_with_dot_name_fields.xml')
     self._make_submission(
         path, forced_submission_time=self._submission_time)
     # test csv
     export = generate_export(Export.CSV_EXPORT, 'csv', self.user.username,
                              'userone')
     storage = get_storage_class()()
     self.assertTrue(storage.exists(export.filepath))
     path, ext = os.path.splitext(export.filename)
     self.assertEqual(ext, '.csv')
     with open(os.path.join(
             os.path.dirname(__file__), 'fixtures', 'userone',
             'userone_with_dot_name_fields.csv')) as f1:
         with storage.open(export.filepath) as f2:
             expected_content = f1.read()
             actual_content = f2.read()
             self.assertEquals(actual_content, expected_content)
示例#30
0
 def test_graceful_exit_on_export_delete_if_file_doesnt_exist(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
         self.xform.id_string)
     storage = get_storage_class()()
     # delete file
     storage.delete(export.filepath)
     self.assertFalse(storage.exists(export.filepath))
     # clear filename, like it would be in an incomplete export
     export.filename = None
     export.filedir = None
     export.save()
     # delete export record, which should try to delete file as well
     delete_url = reverse(delete_export, kwargs={
         'username': self.user.username,
         'id_string': self.xform.id_string,
         'export_type': 'xls'
     })
     post_data = {'export_id': export.id}
     response = self.client.post(delete_url, post_data)
     self.assertEqual(response.status_code, 302)
示例#31
0
 def test_last_submission_time_on_export(self):
     self._publish_transportation_form()
     self._submit_transport_instance()
     # create export
     xls_export = generate_export(Export.XLS_EXPORT, 'xls', self.user.username,
         self.xform.id_string)
     num_exports = Export.objects.filter(xform=self.xform,
         export_type=Export.XLS_EXPORT).count()
     # check that our function knows there are no more submissions
     self.assertFalse(Export.exports_outdated(xform=self.xform,
         export_type=Export.XLS_EXPORT))
     # force new  last submission date on xform
     last_submission = self.xform.surveys.order_by('-date_created')[0]
     last_submission.date_created += datetime.timedelta(hours=1)
     last_submission.save()
     # check that our function knows data has changed
     self.assertTrue(Export.exports_outdated(xform=self.xform,
         export_type=Export.XLS_EXPORT))
     # check that requesting list url will generate a new export
     export_list_url = reverse(export_list, kwargs={
         'username': self.user.username,
         'id_string': self.xform.id_string,
         'export_type': Export.XLS_EXPORT
     })
     response = self.client.get(export_list_url)
     self.assertEqual(Export.objects.filter(xform=self.xform,
         export_type=Export.XLS_EXPORT).count(), num_exports + 1)
     # make sure another export type causes auto-generation
     num_exports = Export.objects.filter(xform=self.xform,
         export_type=Export.CSV_EXPORT).count()
     export_list_url = reverse(export_list, kwargs={
         'username': self.user.username,
         'id_string': self.xform.id_string,
         'export_type': Export.CSV_EXPORT
     })
     response = self.client.get(export_list_url)
     self.assertEqual(Export.objects.filter(xform=self.xform,
         export_type=Export.CSV_EXPORT).count(), num_exports + 1)
示例#32
0
def xls_export(request, username, id_string):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    force_xlsx = request.GET.get('xlsx') == 'true'
    ext = 'xls' if not force_xlsx else 'xlsx'
    try:
        export = generate_export(Export.XLS_EXPORT, ext, username, id_string,
            None, query)
    except NoRecordsFoundError:
        return HttpResponse(_("No records found to export"))
    else:
        # get extension from file_path, exporter could modify to xlsx if it exceeds limits
        path, ext = os.path.splitext(export.filename)
        ext = ext[1:]
        if request.GET.get('raw'):
            id_string = None
        response = response_with_mimetype_and_name(
            Export.EXPORT_MIMES[ext], id_string, extension=ext,
            file_path=export.filepath)
        return response
    def createOR(self):
        refiner = refine.Refine(server="http://localhost:3333")
        fileobject = generate_export(Export.CSV_EXPORT, 'csv', self.formid.user.username, self.formid.id_string,
            export_id=None, filter_query=None, group_delimiter='~',
            split_select_multiples=False)
        #need to replace the n/as of the formhub exports
        tempcontents = ""
        with open(fileobject.full_filepath, 'rb') as thefile:
            tempcontents = thefile.read()
        thefile.close()
        tempcontents = tempcontents.replace("n/a","")
        with open(fileobject.full_filepath, 'wb') as thefile:
            thefile.write(tempcontents)

        refineproj = refiner.new_project(project_file=fileobject.full_filepath,
            project_name=self.title,
            store_blank_rows=True,
            store_blank_cells_as_nulls=True)
        if refineproj.project_id:
            self.openrefine_projectnumber = refineproj.project_id
            self.save()
            return refineproj
        else:
            return None
示例#34
0
def csv_export(request, username, id_string):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    helper_auth_helper(request)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    ext = Export.CSV_EXPORT

    try:
        export = generate_export(
            Export.CSV_EXPORT, ext, username, id_string, None, query)
    except NoRecordsFoundError:
        return HttpResponse(_("No records found to export"))
    else:
        audit = {
            "xform": xform.id_string,
            "export_type": Export.CSV_EXPORT
        }
        audit_log(Actions.EXPORT_CREATED, request.user, owner,
            _("Created CSV export on '%(id_string)s'.") %\
            {
                'id_string': xform.id_string,
            }, audit, request)
        # log download as well
        audit_log(Actions.EXPORT_DOWNLOADED, request.user, owner,
            _("Downloaded CSV export on '%(id_string)s'.") %\
            {
                'id_string': xform.id_string,
            }, audit, request)
        if request.GET.get('raw'):
            id_string = None
        response = response_with_mimetype_and_name(
            Export.EXPORT_MIMES[ext], id_string, extension=ext,
            file_path=export.filepath)
        return response
示例#35
0
def create_xls_export(username,
                      id_string,
                      export_id,
                      query=None,
                      force_xlsx=True,
                      group_delimiter='/',
                      split_select_multiples=True):
    # we re-query the db instead of passing model objects according to
    # http://docs.celeryproject.org/en/latest/userguide/tasks.html#state
    ext = 'xls' if not force_xlsx else 'xlsx'

    export = Export.objects.get(id=export_id)
    # though export is not available when for has 0 submissions, we
    # catch this since it potentially stops celery
    try:
        gen_export = generate_export(Export.XLS_EXPORT, ext, username,
                                     id_string, export_id, query,
                                     group_delimiter, split_select_multiples)
    except (Exception, NoRecordsFoundError) as e:
        export.internal_status = Export.FAILED
        export.save()
        # mail admins
        details = {
            'export_id': export_id,
            'username': username,
            'id_string': id_string
        }
        report_exception(
            "XLS Export Exception: Export ID - "
            "%(export_id)s, /%(username)s/%(id_string)s" % details, e,
            sys.exc_info())
        #raise for now to let celery know we failed
        # - doesnt seem to break celery`
        raise
    else:
        return gen_export.id
示例#36
0
def data_export(request, username, id_string, export_type):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    helper_auth_helper(request)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    extension = export_type

    # check if we should force xlsx
    force_xlsx = request.GET.get('xls') != 'true'
    if export_type == Export.XLS_EXPORT and force_xlsx:
        extension = 'xlsx'
    elif export_type == Export.CSV_ZIP_EXPORT:
        extension = 'zip'

    audit = {
        "xform": xform.id_string,
        "export_type": export_type
    }
    # check if we need to re-generate,
    # we always re-generate if a filter is specified
    if should_create_new_export(xform, export_type) or query or\
                    'start' in request.GET or 'end' in request.GET:
        format_date_for_mongo = lambda x, datetime: datetime.strptime(
            x, '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S')
        # check for start and end params
        if 'start' in request.GET or 'end' in request.GET:
            if not query:
                query = '{}'
            query = json.loads(query)
            query[SUBMISSION_TIME] = {}
            try:
                if request.GET.get('start'):
                    query[SUBMISSION_TIME]['$gte'] = format_date_for_mongo(
                        request.GET['start'], datetime)
                if request.GET.get('end'):
                    query[SUBMISSION_TIME]['$lte'] = format_date_for_mongo(
                        request.GET['end'], datetime)
            except ValueError:
                return HttpResponseBadRequest(
                    _("Dates must be in the format YY_MM_DD_hh_mm_ss"))
            else:
                query = json.dumps(query)
        try:
            export = generate_export(
                export_type, extension, username, id_string, None, query)
            audit_log(
                Actions.EXPORT_CREATED, request.user, owner,
                _("Created %(export_type)s export on '%(id_string)s'.") %
                {
                    'id_string': xform.id_string,
                    'export_type': export_type.upper()
                }, audit, request)
        except NoRecordsFoundError:
            return HttpResponseNotFound(_("No records found to export"))
    else:
        export = newset_export_for(xform, export_type)

    # log download as well
    audit_log(
        Actions.EXPORT_DOWNLOADED, request.user, owner,
        _("Downloaded %(export_type)s export on '%(id_string)s'.") %
        {
            'id_string': xform.id_string,
            'export_type': export_type.upper()
        }, audit, request)

    if not export.filename:
        # tends to happen when using newset_export_for.
        return HttpResponseNotFound("File does not exist!")
    # get extension from file_path, exporter could modify to
    # xlsx if it exceeds limits
    path, ext = os.path.splitext(export.filename)
    ext = ext[1:]
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name(
        Export.EXPORT_MIMES[ext], id_string, extension=ext,
        file_path=export.filepath)
    return response
示例#37
0
def data_export(request, username, id_string, export_type):
    owner = get_object_or_404(User, username=username)
    xform = get_object_or_404(XForm, id_string=id_string, user=owner)
    helper_auth_helper(request)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden(_(u'Not shared.'))
    query = request.GET.get("query")
    extension = export_type

    # check if we should force xlsx
    force_xlsx = request.GET.get('xls') != 'true'
    if export_type == Export.XLS_EXPORT and force_xlsx:
        extension = 'xlsx'
    elif export_type == Export.CSV_ZIP_EXPORT:
        extension = 'zip'

    audit = {"xform": xform.id_string, "export_type": export_type}
    # check if we need to re-generate,
    # we always re-generate if a filter is specified
    if should_create_new_export(xform, export_type) or query or\
                    'start' in request.GET or 'end' in request.GET:
        format_date_for_mongo = lambda x, datetime: datetime.strptime(
            x, '%y_%m_%d_%H_%M_%S').strftime('%Y-%m-%dT%H:%M:%S')
        # check for start and end params
        if 'start' in request.GET or 'end' in request.GET:
            if not query:
                query = '{}'
            query = json.loads(query)
            query[SUBMISSION_TIME] = {}
            try:
                if request.GET.get('start'):
                    query[SUBMISSION_TIME]['$gte'] = format_date_for_mongo(
                        request.GET['start'], datetime)
                if request.GET.get('end'):
                    query[SUBMISSION_TIME]['$lte'] = format_date_for_mongo(
                        request.GET['end'], datetime)
            except ValueError:
                return HttpResponseBadRequest(
                    _("Dates must be in the format YY_MM_DD_hh_mm_ss"))
            else:
                query = json.dumps(query)
        try:
            export = generate_export(export_type, extension, username,
                                     id_string, None, query)
            audit_log(
                Actions.EXPORT_CREATED, request.user, owner,
                _("Created %(export_type)s export on '%(id_string)s'.") % {
                    'id_string': xform.id_string,
                    'export_type': export_type.upper()
                }, audit, request)
        except NoRecordsFoundError:
            return HttpResponseNotFound(_("No records found to export"))
    else:
        export = newset_export_for(xform, export_type)

    # log download as well
    audit_log(
        Actions.EXPORT_DOWNLOADED, request.user, owner,
        _("Downloaded %(export_type)s export on '%(id_string)s'.") % {
            'id_string': xform.id_string,
            'export_type': export_type.upper()
        }, audit, request)

    if not export.filename:
        # tends to happen when using newset_export_for.
        return HttpResponseNotFound("File does not exist!")
    # get extension from file_path, exporter could modify to
    # xlsx if it exceeds limits
    path, ext = os.path.splitext(export.filename)
    ext = ext[1:]
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name(Export.EXPORT_MIMES[ext],
                                               id_string,
                                               extension=ext,
                                               file_path=export.filepath)
    return response