def test_file_save_and_load(self):
payload = 'something small and simple'
for name in ['normal', u'हिंदी', None]:
saved = SavedBasicExport(configuration=_mk_config(name))
saved.save()
saved.set_payload(payload)
self.assertEqual(payload, saved.get_payload())
def rebuild_export(config, schema, output_dir, last_access_cutoff=None, filter=None):
if output_dir == "couch":
saved = get_saved_export_and_delete_copies(config.index)
if last_access_cutoff and saved and saved.last_accessed and \
saved.last_accessed < last_access_cutoff:
# ignore exports that haven't been accessed since last_access_cutoff
return
try:
files = schema.get_export_files(format=config.format, filter=filter)
except SchemaMismatchException:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
raise ExportRebuildError(u'Schema mismatch for {}. Rebuilding tables...'.format(config.filename))
with files:
payload = files.file.payload
if output_dir == "couch":
if not saved:
saved = SavedBasicExport(configuration=config)
else:
saved.configuration = config
if saved.last_accessed is None:
saved.last_accessed = datetime.utcnow()
saved.last_updated = datetime.utcnow()
saved.save()
saved.set_payload(payload)
else:
with open(os.path.join(output_dir, config.filename), "wb") as f:
f.write(payload)
def test_file_save_and_load(self):
payload = 'something small and simple'
for name in ['normal', u'हिंदी', None]:
saved = SavedBasicExport(configuration=_mk_config(name))
saved.save()
saved.set_payload(payload)
self.assertEqual(payload, saved.get_payload())
def export_for_group(export_id_or_group, output_dir):
if isinstance(export_id_or_group, basestring):
try:
config = GroupExportConfiguration.get(export_id_or_group)
except ResourceNotFound:
raise Exception("Couldn't find an export with id %s" %
export_id_or_group)
else:
config = export_id_or_group
for config, schema in config.all_exports:
try:
files = schema.get_export_files(format=config.format)
except SchemaMismatchException, e:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
continue
payload = files.file.payload
if output_dir == "couch":
saved = SavedBasicExport.view("couchexport/saved_exports",
key=json.dumps(config.index),
include_docs=True,
reduce=False).one()
if not saved:
saved = SavedBasicExport(configuration=config)
else:
saved.configuration = config
saved.last_updated = datetime.utcnow()
saved.save()
saved.set_payload(payload)
else:
with open(os.path.join(output_dir, config.filename), "wb") as f:
f.write(payload)
def export_for_group(export_id_or_group, output_dir):
if isinstance(export_id_or_group, basestring):
try:
config = GroupExportConfiguration.get(export_id_or_group)
except ResourceNotFound:
raise Exception("Couldn't find an export with id %s" % export_id_or_group)
else:
config = export_id_or_group
for config, schema in config.all_exports:
try:
tmp, _ = schema.get_export_files(format=config.format)
except SchemaMismatchException, e:
# fire off a delayed force update to prevent this from happening again
rebuild_schemas.delay(config.index)
continue
payload = Temp(tmp).payload
if output_dir == "couch":
saved = SavedBasicExport.view("couchexport/saved_exports",
key=json.dumps(config.index),
include_docs=True,
reduce=False).one()
if not saved:
saved = SavedBasicExport(configuration=config)
else:
saved.configuration = config
saved.last_updated = datetime.utcnow()
saved.save()
saved.set_payload(payload)
else:
with open(os.path.join(output_dir, config.filename), "wb") as f:
f.write(payload)
def testFileSaveAndLoad(self):
def _mk_config(name):
return ExportConfiguration(index="dummy_index", name=name, format="xlsx")
payload = "something small and simple"
for name in ["normal", u"हिंदी", None]:
saved = SavedBasicExport(configuration=_mk_config(name))
saved.save()
saved.set_payload(payload)
self.assertEqual(payload, saved.get_payload())
def test_save_basic_export_to_blobdb(self):
index = ['single']
saved_export = SavedBasicExport(configuration=_mk_config(index=index))
saved_export.save()
saved_export.set_payload("content")
name = saved_export.get_attachment_name()
self.assertTrue(saved_export.has_file())
self.assertIn(name, saved_export.external_blobs)
self.assertEqual(saved_export.size, 7)
with saved_export.get_payload(stream=True) as fh:
self.assertEqual(fh.read(), "content")
def test_save_basic_export_to_blobdb(self):
index = ['single']
saved_export = SavedBasicExport(configuration=_mk_config(index=index))
saved_export.save()
saved_export.set_payload("content")
name = saved_export.get_attachment_name()
self.assertTrue(saved_export.has_file())
self.assertIn(name, saved_export.external_blobs)
self.assertEqual(saved_export.size, 7)
with saved_export.get_payload(stream=True) as fh:
self.assertEqual(fh.read(), b"content")
def testFileSaveAndLoad(self):
def _mk_config(name):
return ExportConfiguration(index='dummy_index',
name=name,
format='xlsx')
payload = 'something small and simple'
for name in ['normal', u'हिंदी', None]:
saved = SavedBasicExport(configuration=_mk_config(name))
saved.save()
saved.set_payload(payload)
self.assertEqual(payload, saved.get_payload())
def setUp(self):
with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
config = settings.S3_BLOB_DB_SETTINGS
fsdb = TemporaryFilesystemBlobDB()
assert get_blob_db() is fsdb, (get_blob_db(), fsdb)
self.migrate_docs = docs = []
for i in range(self.test_size):
doc = SavedBasicExport(configuration=_mk_config("config-%s" % i))
doc.save()
doc.set_payload(("content %s" % i).encode('utf-8'))
docs.append(doc)
s3db = TemporaryS3BlobDB(config)
self.db = TemporaryMigratingBlobDB(s3db, fsdb)
assert get_blob_db() is self.db, (get_blob_db(), self.db)
BaseMigrationTest.discard_migration_state(self.slug)
def _save_export_payload(files, saved_export, config, is_safe=False):
payload = files.file.payload
if not saved_export:
saved_export = SavedBasicExport(configuration=config)
else:
saved_export.configuration = config
saved_export.is_safe = is_safe
if saved_export.last_accessed is None:
saved_export.last_accessed = datetime.utcnow()
saved_export.last_updated = datetime.utcnow()
try:
saved_export.save()
except ResourceConflict:
# task was executed concurrently, so let first to finish win and abort the rest
pass
else:
saved_export.set_payload(payload)
def _save_export_payload(files, saved_export, config, is_safe=False):
payload = files.file.payload
if not saved_export:
saved_export = SavedBasicExport(configuration=config)
else:
saved_export.configuration = config
saved_export.is_safe = is_safe
if saved_export.last_accessed is None:
saved_export.last_accessed = datetime.utcnow()
saved_export.last_updated = datetime.utcnow()
try:
saved_export.save()
except ResourceConflict:
# task was executed concurrently, so let first to finish win and abort the rest
pass
else:
saved_export.set_payload(payload)
def setUp(self):
with trap_extra_setup(AttributeError,
msg="S3_BLOB_DB_SETTINGS not configured"):
config = settings.S3_BLOB_DB_SETTINGS
fsdb = TemporaryFilesystemBlobDB()
assert get_blob_db() is fsdb, (get_blob_db(), fsdb)
self.migrate_docs = docs = []
for i in range(self.test_size):
doc = SavedBasicExport(configuration=_mk_config("config-%s" % i))
doc.save()
doc.set_payload(("content %s" % i).encode('utf-8'))
docs.append(doc)
s3db = TemporaryS3BlobDB(config)
self.db = TemporaryMigratingBlobDB(s3db, fsdb)
assert get_blob_db() is self.db, (get_blob_db(), self.db)
BaseMigrationTest.discard_migration_state(self.slug)
