def create_new_checkpoint(self):
checkpoint = ExportSchema(seq=str(self.current_seq),
schema=self.get_latest_schema(),
timestamp=self.timestamp,
index=self.schema_index)
checkpoint.save()
return checkpoint
def create_new_checkpoint(self):
checkpoint = ExportSchema(
schema=self.get_latest_schema(),
timestamp=self.timestamp,
index=self.schema_index,
)
checkpoint.save()
return checkpoint
def create_basic_form_checkpoint(index):
checkpoint = ExportSchema(
schema=BASIC_FORM_SCHEMA,
timestamp=datetime(1970, 1, 1),
index=index,
)
checkpoint.save()
return checkpoint
def create_basic_form_checkpoint(index):
checkpoint = ExportSchema(
schema=BASIC_FORM_SCHEMA,
timestamp=datetime(1970, 1, 1),
index=index,
)
checkpoint.save()
return checkpoint
def testSaveAndLoad(self):
index = ["foo", 2]
schema = ExportSchema(seq=5, index=index)
inner = {"dict": {"bar": 1, "baz": [2,3]},
"list": ["foo", "bar"],
"dictlist": [{"bip": 1, "bop": "blah"},
{"bip": 2, "bop": "blah2"}],
"item": "yoyoyo"}
schema.schema = inner
schema.save()
back = ExportSchema.get(schema.get_id)
self.assertEqual(inner, back.schema)
self.assertEqual(index, back.index)
def test_get_all_checkpoints(self):
index = ["mydomain", "myxmlns"]
self.addCleanup(lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)])
schema1 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema1.save()
schema1_prime, = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
schema2 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema2.save()
schema1_prime, schema2_prime = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
self.assert_docs_equal(schema2_prime, schema2)
def testSaveAndLoad(self):
index = ["foo", 2]
schema = ExportSchema(seq="5", index=index, timestamp=datetime.now())
inner = {"dict": {"bar": 1, "baz": [2,3]},
"list": ["foo", "bar"],
"dictlist": [{"bip": 1, "bop": "blah"},
{"bip": 2, "bop": "blah2"}],
"item": "yoyoyo"}
schema.schema = inner
schema.save()
back = ExportSchema.get(schema.get_id)
self.assertEqual(inner, back.schema)
self.assertEqual(index, back.index)
def test_get_all_checkpoints(self):
index = ["mydomain", "myxmlns"]
self.addCleanup(lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)])
schema1 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema1.save()
schema1_prime, = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
schema2 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema2.save()
schema1_prime, schema2_prime = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
self.assert_docs_equal(schema2_prime, schema2)
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
current_seq = db.info()["update_seq"]
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(get_db(), schema_index,
previous_export=previous_export)
schema = get_schema_new(config)
if not schema:
return None
updated_checkpoint = ExportSchema(seq=current_seq, schema=schema,
index=schema_index)
updated_checkpoint.save()
return updated_checkpoint
def test_get_last(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.addCleanup(
lambda idx:
[cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)],
index)
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assert_docs_equal(schema1, ExportSchema.last(index))
schema2 = ExportSchema(index=index,
timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assert_docs_equal(schema2, ExportSchema.last(index))
schema3 = ExportSchema(index=index,
timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
# still schema2 (which has a later date than schema3)
self.assert_docs_equal(schema2, ExportSchema.last(index))
def export(
schema_index,
file,
format=Format.XLS_2007,
previous_export_id=None,
filter=None,
max_column_size=2000,
separator="|",
):
"""
Exports data from couch documents matching a given tag to a file.
Returns true if it finds data, otherwise nothing
"""
previous_export = ExportSchema.get(previous_export_id) if previous_export_id else None
database = get_db()
config = ExportConfiguration(database, schema_index, previous_export, filter)
# handle empty case
if not config.potentially_relevant_ids:
return None
# get and checkpoint the latest schema
updated_schema = get_schema_new(config)
export_schema_checkpoint = ExportSchema(seq=config.current_seq, schema=updated_schema, index=config.schema_index)
export_schema_checkpoint.save()
# transform docs onto output and save
writer = get_writer(format)
# open the doc and the headers
formatted_headers = get_headers(updated_schema, separator=separator)
writer.open(formatted_headers, file)
for doc in config.get_docs():
writer.write(
format_tables(create_intermediate_tables(doc, updated_schema), include_headers=False, separator=separator)
)
writer.close()
return export_schema_checkpoint
def testGetLast(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assertEqual(schema1._id, ExportSchema.last(index)._id)
schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def testGetLast(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assertEqual(schema1._id, ExportSchema.last(index)._id)
schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def testGetLast(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
# by design, if something has a timestamp it always wins out, even
# if something has a higher seq
dt = datetime.utcnow()
schema1 = ExportSchema(seq="2", index=index, timestamp=dt)
schema1.save(**save_args)
self.assertEqual(schema1._id, ExportSchema.last(index)._id)
schema2 = ExportSchema(seq="1", index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
schema3 = ExportSchema(seq="3", index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def test_get_last(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.addCleanup(
lambda idx: [cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)],
index
)
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assert_docs_equal(schema1, ExportSchema.last(index))
schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assert_docs_equal(schema2, ExportSchema.last(index))
schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
# still schema2 (which has a later date than schema3)
self.assert_docs_equal(schema2, ExportSchema.last(index))
def create_basic_form_checkpoint(index):
checkpoint = ExportSchema(seq="0", schema=BASIC_FORM_SCHEMA, timestamp=datetime.utcnow(), index=index)
checkpoint.save()
return checkpoint