Example #1
0
 def test_file_save_and_load(self):
     payload = 'something small and simple'
     for name in ['normal', u'हिंदी', None]:
         saved = SavedBasicExport(configuration=_mk_config(name))
         saved.save()
         saved.set_payload(payload)
         self.assertEqual(payload, saved.get_payload())
Example #2
0
def rebuild_export(config, schema, output_dir, last_access_cutoff=None, filter=None):
    if output_dir == "couch":
        saved = get_saved_export_and_delete_copies(config.index)
        if last_access_cutoff and saved and saved.last_accessed and \
                saved.last_accessed < last_access_cutoff:
            # ignore exports that haven't been accessed since last_access_cutoff
            return

    try:
        files = schema.get_export_files(format=config.format, filter=filter)
    except SchemaMismatchException:
        # fire off a delayed force update to prevent this from happening again
        rebuild_schemas.delay(config.index)
        raise ExportRebuildError(u'Schema mismatch for {}. Rebuilding tables...'.format(config.filename))

    with files:
        payload = files.file.payload
        if output_dir == "couch":
            if not saved:
                saved = SavedBasicExport(configuration=config)
            else:
                saved.configuration = config

            if saved.last_accessed is None:
                saved.last_accessed = datetime.utcnow()
            saved.last_updated = datetime.utcnow()
            saved.save()
            saved.set_payload(payload)
        else:
            with open(os.path.join(output_dir, config.filename), "wb") as f:
                f.write(payload)
Example #3
0
 def test_file_save_and_load(self):
     payload = 'something small and simple'
     for name in ['normal', u'हिंदी', None]:
         saved = SavedBasicExport(configuration=_mk_config(name))
         saved.save()
         saved.set_payload(payload)
         self.assertEqual(payload, saved.get_payload())
Example #4
0
def export_for_group(export_id_or_group, output_dir):
    if isinstance(export_id_or_group, basestring):
        try:
            config = GroupExportConfiguration.get(export_id_or_group)
        except ResourceNotFound:
            raise Exception("Couldn't find an export with id %s" %
                            export_id_or_group)
    else:
        config = export_id_or_group

    for config, schema in config.all_exports:
        try:
            files = schema.get_export_files(format=config.format)
        except SchemaMismatchException, e:
            # fire off a delayed force update to prevent this from happening again
            rebuild_schemas.delay(config.index)
            continue

        payload = files.file.payload
        if output_dir == "couch":
            saved = SavedBasicExport.view("couchexport/saved_exports",
                                          key=json.dumps(config.index),
                                          include_docs=True,
                                          reduce=False).one()
            if not saved:
                saved = SavedBasicExport(configuration=config)
            else:
                saved.configuration = config
            saved.last_updated = datetime.utcnow()
            saved.save()
            saved.set_payload(payload)

        else:
            with open(os.path.join(output_dir, config.filename), "wb") as f:
                f.write(payload)
Example #5
0
def export_for_group(export_id_or_group, output_dir):
    if isinstance(export_id_or_group, basestring):
        try:
            config = GroupExportConfiguration.get(export_id_or_group)
        except ResourceNotFound:
            raise Exception("Couldn't find an export with id %s" % export_id_or_group)
    else:
        config = export_id_or_group

    for config, schema in config.all_exports:
        try:
            tmp, _ = schema.get_export_files(format=config.format)
        except SchemaMismatchException, e:
            # fire off a delayed force update to prevent this from happening again
            rebuild_schemas.delay(config.index)
            continue

        payload = Temp(tmp).payload
        if output_dir == "couch":
            saved = SavedBasicExport.view("couchexport/saved_exports", 
                                          key=json.dumps(config.index),
                                          include_docs=True,
                                          reduce=False).one()
            if not saved: 
                saved = SavedBasicExport(configuration=config)
            else:
                saved.configuration = config
            saved.last_updated = datetime.utcnow()
            saved.save()
            saved.set_payload(payload)

        else:
            with open(os.path.join(output_dir, config.filename), "wb") as f:
                f.write(payload)
Example #6
0
    def testFileSaveAndLoad(self):
        def _mk_config(name):
            return ExportConfiguration(index="dummy_index", name=name, format="xlsx")

        payload = "something small and simple"
        for name in ["normal", u"हिंदी", None]:
            saved = SavedBasicExport(configuration=_mk_config(name))
            saved.save()
            saved.set_payload(payload)
            self.assertEqual(payload, saved.get_payload())
Example #7
0
 def test_save_basic_export_to_blobdb(self):
     index = ['single']
     saved_export = SavedBasicExport(configuration=_mk_config(index=index))
     saved_export.save()
     saved_export.set_payload("content")
     name = saved_export.get_attachment_name()
     self.assertTrue(saved_export.has_file())
     self.assertIn(name, saved_export.external_blobs)
     self.assertEqual(saved_export.size, 7)
     with saved_export.get_payload(stream=True) as fh:
         self.assertEqual(fh.read(), "content")
Example #8
0
 def test_save_basic_export_to_blobdb(self):
     index = ['single']
     saved_export = SavedBasicExport(configuration=_mk_config(index=index))
     saved_export.save()
     saved_export.set_payload("content")
     name = saved_export.get_attachment_name()
     self.assertTrue(saved_export.has_file())
     self.assertIn(name, saved_export.external_blobs)
     self.assertEqual(saved_export.size, 7)
     with saved_export.get_payload(stream=True) as fh:
         self.assertEqual(fh.read(), b"content")
Example #9
0
    def testFileSaveAndLoad(self):
        def _mk_config(name):
            return ExportConfiguration(index='dummy_index',
                                       name=name,
                                       format='xlsx')

        payload = 'something small and simple'
        for name in ['normal', u'हिंदी', None]:
            saved = SavedBasicExport(configuration=_mk_config(name))
            saved.save()
            saved.set_payload(payload)
            self.assertEqual(payload, saved.get_payload())
Example #10
0
    def setUp(self):
        with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
            config = settings.S3_BLOB_DB_SETTINGS

        fsdb = TemporaryFilesystemBlobDB()
        assert get_blob_db() is fsdb, (get_blob_db(), fsdb)
        self.migrate_docs = docs = []
        for i in range(self.test_size):
            doc = SavedBasicExport(configuration=_mk_config("config-%s" % i))
            doc.save()
            doc.set_payload(("content %s" % i).encode('utf-8'))
            docs.append(doc)

        s3db = TemporaryS3BlobDB(config)
        self.db = TemporaryMigratingBlobDB(s3db, fsdb)
        assert get_blob_db() is self.db, (get_blob_db(), self.db)
        BaseMigrationTest.discard_migration_state(self.slug)
Example #11
0
def _save_export_payload(files, saved_export, config, is_safe=False):
    payload = files.file.payload
    if not saved_export:
        saved_export = SavedBasicExport(configuration=config)
    else:
        saved_export.configuration = config
    saved_export.is_safe = is_safe

    if saved_export.last_accessed is None:
        saved_export.last_accessed = datetime.utcnow()
    saved_export.last_updated = datetime.utcnow()
    try:
        saved_export.save()
    except ResourceConflict:
        # task was executed concurrently, so let first to finish win and abort the rest
        pass
    else:
        saved_export.set_payload(payload)
Example #12
0
def _save_export_payload(files, saved_export, config, is_safe=False):
    payload = files.file.payload
    if not saved_export:
        saved_export = SavedBasicExport(configuration=config)
    else:
        saved_export.configuration = config
    saved_export.is_safe = is_safe

    if saved_export.last_accessed is None:
        saved_export.last_accessed = datetime.utcnow()
    saved_export.last_updated = datetime.utcnow()
    try:
        saved_export.save()
    except ResourceConflict:
        # task was executed concurrently, so let first to finish win and abort the rest
        pass
    else:
        saved_export.set_payload(payload)
Example #13
0
    def setUp(self):
        with trap_extra_setup(AttributeError,
                              msg="S3_BLOB_DB_SETTINGS not configured"):
            config = settings.S3_BLOB_DB_SETTINGS

        fsdb = TemporaryFilesystemBlobDB()
        assert get_blob_db() is fsdb, (get_blob_db(), fsdb)
        self.migrate_docs = docs = []
        for i in range(self.test_size):
            doc = SavedBasicExport(configuration=_mk_config("config-%s" % i))
            doc.save()
            doc.set_payload(("content %s" % i).encode('utf-8'))
            docs.append(doc)

        s3db = TemporaryS3BlobDB(config)
        self.db = TemporaryMigratingBlobDB(s3db, fsdb)
        assert get_blob_db() is self.db, (get_blob_db(), self.db)
        BaseMigrationTest.discard_migration_state(self.slug)