def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
try:
current_seq = int(db.info()["update_seq"])
except ValueError:
pass # we must be on bigcouch, so comparing seqs is useless
else:
if previous_export and not previous_export.is_bigcouch \
and int(previous_export.seq) > current_seq:
# something weird happened (like the couch database changing)
# better rebuild from scratch
previous_export = None
config = ExportConfiguration(db, schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def testComputedRemoved(self):
db = get_db()
db.save_doc({
'#export_tag': 'tag',
'tag': 'computed-test',
'p1': 'v1',
'computed_': {
'ignore': 'this stuff'
}
},
**get_safe_write_kwargs()
)
config = ExportConfiguration(db, ['computed-test'], cleanup_fn=None)
schema = config.get_latest_schema()
self.assertTrue('computed_' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertTrue('computed_' in docs[0])
# check that it works both explicitly and by default
explicit_config = ExportConfiguration(db, ['computed-test'], cleanup_fn=clear_computed)
default_config = ExportConfiguration(db, ['computed-test'])
for config in (explicit_config, default_config):
schema = config.get_latest_schema()
self.assertFalse('computed_' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertFalse('computed_' in docs[0])
def download_data(self, format="", previous_export=None, filter=None):
"""
If there is data, return an HTTPResponse with the appropriate data.
If there is not data returns None.
"""
from couchexport.shortcuts import export_response
from couchexport.export import get_writer, get_schema_new, format_tables, create_intermediate_tables
if not format:
format = self.default_format or Format.XLS_2007
from couchexport.export import ExportConfiguration
database = get_db()
config = ExportConfiguration(database, self.index, previous_export, util.intersect_filters(self.filter, filter))
# get and checkpoint the latest schema
updated_schema = get_schema_new(config)
export_schema_checkpoint = ExportSchema(
seq=config.current_seq, schema=updated_schema, index=config.schema_index
)
export_schema_checkpoint.save()
# transform docs onto output and save
writer = get_writer(format)
# open the doc and the headers
formatted_headers = self.get_table_headers()
tmp = StringIO()
writer.open(formatted_headers, tmp)
for doc in config.get_docs():
writer.write(self.trim(format_tables(create_intermediate_tables(doc, updated_schema), separator=".")))
writer.close()
return export_response(tmp, format, self.name)
def testAttachmentsRemoved(self):
db = get_db()
res = db.save_doc({
'#export_tag': 'tag',
'tag': 'attachments-test',
'p1': 'v1',
},
**get_safe_write_kwargs()
)
doc = db.get(res['id'])
doc['_attachments'] = {
"attach.txt": {
"content_type": "text/plain",
"data": "some content",
},
}
db.save_doc(doc)
config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=None)
schema = config.get_latest_schema()
self.assertTrue('_attachments' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertTrue('_attachments' in docs[0])
# check that it works both explicitly and by default
explicit_config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=clear_attachments)
default_config = ExportConfiguration(db, ['attachments-test'])
for config in (explicit_config, default_config):
schema = config.get_latest_schema()
self.assertFalse('_attachments' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertFalse('_attachments' in docs[0])
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
try:
current_seq = int(db.info()["update_seq"])
except ValueError:
pass # we must be on bigcouch, so comparing seqs is useless
else:
if previous_export and not previous_export.is_bigcouch \
and int(previous_export.seq) > current_seq:
# something weird happened (like the couch database changing)
# better rebuild from scratch
previous_export = None
config = ExportConfiguration(db,
schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def handle(self, *args, **options):
db = ExportSchema.get_db()
for index in ExportSchema.get_all_indices():
last = ExportSchema.last(index)
if not last.timestamp:
config = ExportConfiguration(db, index, disable_checkpoints=True)
config.create_new_checkpoint()
assert ExportSchema.last(index).timestamp
print "set timestamp for %s" % index
else:
print "%s all set" % index
def get_export_components(self, previous_export_id=None, filter=None):
from couchexport.export import ExportConfiguration
database = get_db()
config = ExportConfiguration(database, self.index, previous_export_id,
self.filter & filter)
# get and checkpoint the latest schema
updated_schema = config.get_latest_schema()
export_schema_checkpoint = config.create_new_checkpoint()
return config, updated_schema, export_schema_checkpoint
def handle(self, *args, **options):
db = ExportSchema.get_db()
for index in ExportSchema.get_all_indices():
last = ExportSchema.last(index)
if not last.timestamp:
config = ExportConfiguration(db,
index,
disable_checkpoints=True)
config.create_new_checkpoint()
assert ExportSchema.last(index).timestamp
print "set timestamp for %s" % index
else:
print "%s all set" % index
def rebuild_schemas(index):
"""
Resets the schema for all checkpoints to the latest version based off the
current document structure. Returns the number of checkpoints updated.
"""
db = ExportSchema.get_db()
all_checkpoints = ExportSchema.get_all_checkpoints(index)
config = ExportConfiguration(db, index, disable_checkpoints=True)
latest = config.create_new_checkpoint()
for cp in all_checkpoints:
cp.schema = latest.schema
cp.save()
return len(all_checkpoints)
def get_export_components(self, previous_export_id=None, filter=None):
from couchexport.export import ExportConfiguration
database = get_db()
config = ExportConfiguration(database, self.index,
previous_export_id,
self.filter & filter)
# get and checkpoint the latest schema
updated_schema = config.get_latest_schema()
export_schema_checkpoint = config.create_new_checkpoint()
return config, updated_schema, export_schema_checkpoint
def rebuild_schemas(index):
"""
Resets the schema for all checkpoints to the latest version based off the
current document structure. Returns the number of checkpoints updated.
"""
db = ExportSchema.get_db()
all_checkpoints = ExportSchema.get_all_checkpoints(index)
config = ExportConfiguration(db, index, disable_checkpoints=True)
latest = config.create_new_checkpoint()
for cp in all_checkpoints:
cp.schema = latest.schema
cp.save()
return len(all_checkpoints)
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(db, schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def testAttachmentsRemoved(self):
db = get_db()
res = db.save_doc(
{
'#export_tag': 'tag',
'tag': 'attachments-test',
'p1': 'v1',
}, **get_safe_write_kwargs())
doc = db.get(res['id'])
doc['_attachments'] = {
"attach.txt": {
"content_type": "text/plain",
"data": "some content",
},
}
db.save_doc(doc)
config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=None)
schema = config.get_latest_schema()
self.assertTrue('_attachments' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertTrue('_attachments' in docs[0])
# check that it works both explicitly and by default
explicit_config = ExportConfiguration(db, ['attachments-test'],
cleanup_fn=clear_attachments)
default_config = ExportConfiguration(db, ['attachments-test'])
for config in (explicit_config, default_config):
schema = config.get_latest_schema()
self.assertFalse('_attachments' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertFalse('_attachments' in docs[0])
def testComputedRemoved(self):
db = get_db()
db.save_doc(
{
'#export_tag': 'tag',
'tag': 'computed-test',
'p1': 'v1',
'computed_': {
'ignore': 'this stuff'
}
}, **get_safe_write_kwargs())
config = ExportConfiguration(db, ['computed-test'], cleanup_fn=None)
schema = config.get_latest_schema()
self.assertTrue('computed_' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertTrue('computed_' in docs[0])
# check that it works both explicitly and by default
explicit_config = ExportConfiguration(db, ['computed-test'],
cleanup_fn=clear_computed)
default_config = ExportConfiguration(db, ['computed-test'])
for config in (explicit_config, default_config):
schema = config.get_latest_schema()
self.assertFalse('computed_' in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertFalse('computed_' in docs[0])
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(db,
schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def to_export_config(self):
"""
Return an ExportConfiguration object that represents this.
"""
# confusingly, the index isn't the actual index property,
# but is the index appended with the id to this document.
# this is to avoid conflicts among multiple exports
index = "%s-%s" % (self.index, self._id) if isinstance(self.index, six.string_types) else \
self.index + [self._id] # self.index required to be a string or list
return ExportConfiguration(index=index, name=self.name,
format=self.default_format)
def testAttachmentsRemoved(self):
db = get_db()
res = db.save_doc({"#export_tag": "tag", "tag": "attachments-test", "p1": "v1"}, **get_safe_write_kwargs())
doc = db.get(res["id"])
db.put_attachment(doc, "some content", "attach.txt")
config = ExportConfiguration(db, ["attachments-test"], cleanup_fn=None)
schema = config.get_latest_schema()
self.assertTrue("_attachments" in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertTrue("_attachments" in docs[0])
# check that it works both explicitly and by default
explicit_config = ExportConfiguration(db, ["attachments-test"], cleanup_fn=clear_attachments)
default_config = ExportConfiguration(db, ["attachments-test"])
for config in (explicit_config, default_config):
schema = config.get_latest_schema()
self.assertFalse("_attachments" in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertFalse("_attachments" in docs[0])
def testComputedRemoved(self):
db = get_db()
db.save_doc(
{"#export_tag": "tag", "tag": "computed-test", "p1": "v1", "computed_": {"ignore": "this stuff"}},
**get_safe_write_kwargs()
)
config = ExportConfiguration(db, ["computed-test"], cleanup_fn=None)
schema = config.get_latest_schema()
self.assertTrue("computed_" in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertTrue("computed_" in docs[0])
# check that it works both explicitly and by default
explicit_config = ExportConfiguration(db, ["computed-test"], cleanup_fn=clear_computed)
default_config = ExportConfiguration(db, ["computed-test"])
for config in (explicit_config, default_config):
schema = config.get_latest_schema()
self.assertFalse("computed_" in schema)
docs = list(config.get_docs())
self.assertEqual(1, len(docs))
self.assertFalse("computed_" in docs[0])
def _clear_docs(self):
config = ExportConfiguration(
XFormInstance.get_db(),
[DOMAIN, "http://www.commcarehq.org/export/test"])
for form in config.get_docs():
XFormInstance.wrap(form).delete()
def _clear_docs(self):
config = ExportConfiguration(XFormInstance.get_db(),
[DOMAIN, "http://www.commcarehq.org/export/test"])
for form in config.get_docs():
XFormInstance.wrap(form).delete()
