def create_new_checkpoint(self):
checkpoint = ExportSchema(seq=str(self.current_seq),
schema=self.get_latest_schema(),
timestamp=self.timestamp,
index=self.schema_index)
checkpoint.save()
return checkpoint
def test_wrap_datetime_min(self):
schema_bad = ExportSchema(
schema={},
timestamp=datetime.min,
index='index',
)
schema_good = ExportSchema.wrap(schema_bad.to_json())
self.assertEqual(schema_good.timestamp, datetime(1970, 1, 1))
def create_basic_form_checkpoint(index):
checkpoint = ExportSchema(
schema=BASIC_FORM_SCHEMA,
timestamp=datetime(1970, 1, 1),
index=index,
)
checkpoint.save()
return checkpoint
def test_wrap_datetime_hippy(self):
schema1 = ExportSchema(
schema={},
timestamp=datetime(1970, 1, 2),
index='index',
)
schema2 = ExportSchema.wrap(schema1.to_json())
self.assertEqual(schema2.timestamp, datetime(1970, 1, 2))
def create_basic_form_checkpoint(index):
checkpoint = ExportSchema(
schema=BASIC_FORM_SCHEMA,
timestamp=datetime(1970, 1, 1),
index=index,
)
checkpoint.save()
return checkpoint
def create_new_checkpoint(self):
checkpoint = ExportSchema(
schema=self.get_latest_schema(),
timestamp=self.timestamp,
index=self.schema_index,
)
checkpoint.save()
return checkpoint
def test_wrap_datetime_hippy(self):
schema1 = ExportSchema(
schema={},
timestamp=datetime(1970, 1, 2),
index='index',
)
schema2 = ExportSchema.wrap(schema1.to_json())
self.assertEqual(schema2.timestamp, datetime(1970, 1, 2))
def test_wrap_datetime_min(self):
schema_bad = ExportSchema(
schema={},
timestamp=datetime.min,
index='index',
)
schema_good = ExportSchema.wrap(schema_bad.to_json())
self.assertEqual(schema_good.timestamp, datetime(1970, 1, 1))
def handle(self, *args, **options):
db = ExportSchema.get_db()
for index in ExportSchema.get_all_indices():
last = ExportSchema.last(index)
if not last.timestamp:
config = ExportConfiguration(db, index, disable_checkpoints=True)
config.create_new_checkpoint()
assert ExportSchema.last(index).timestamp
print "set timestamp for %s" % index
else:
print "%s all set" % index
def rebuild_schemas(index):
"""
Resets the schema for all checkpoints to the latest version based off the
current document structure. Returns the number of checkpoints updated.
"""
db = ExportSchema.get_db()
all_checkpoints = ExportSchema.get_all_checkpoints(index)
config = ExportConfiguration(db, index, disable_checkpoints=True)
latest = config.create_new_checkpoint()
for cp in all_checkpoints:
cp.schema = latest.schema
cp.save()
return len(all_checkpoints)
def rebuild_schemas(index):
"""
Resets the schema for all checkpoints to the latest version based off the
current document structure. Returns the number of checkpoints updated.
"""
db = ExportSchema.get_db()
all_checkpoints = ExportSchema.get_all_checkpoints(index)
config = ExportConfiguration(db, index, disable_checkpoints=True)
latest = config.create_new_checkpoint()
for cp in all_checkpoints:
cp.schema = latest.schema
cp.save()
return len(all_checkpoints)
def testSaveAndLoad(self):
index = ["foo", 2]
schema = ExportSchema(seq=5, index=index)
inner = {"dict": {"bar": 1, "baz": [2,3]},
"list": ["foo", "bar"],
"dictlist": [{"bip": 1, "bop": "blah"},
{"bip": 2, "bop": "blah2"}],
"item": "yoyoyo"}
schema.schema = inner
schema.save()
back = ExportSchema.get(schema.get_id)
self.assertEqual(inner, back.schema)
self.assertEqual(index, back.index)
def handle(self, *args, **options):
db = ExportSchema.get_db()
for index in ExportSchema.get_all_indices():
last = ExportSchema.last(index)
if not last.timestamp:
config = ExportConfiguration(db,
index,
disable_checkpoints=True)
config.create_new_checkpoint()
assert ExportSchema.last(index).timestamp
print "set timestamp for %s" % index
else:
print "%s all set" % index
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
try:
current_seq = int(db.info()["update_seq"])
except ValueError:
pass # we must be on bigcouch, so comparing seqs is useless
else:
if previous_export and not previous_export.is_bigcouch \
and int(previous_export.seq) > current_seq:
# something weird happened (like the couch database changing)
# better rebuild from scratch
previous_export = None
config = ExportConfiguration(db, schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
try:
current_seq = int(db.info()["update_seq"])
except ValueError:
pass # we must be on bigcouch, so comparing seqs is useless
else:
if previous_export and not previous_export.is_bigcouch \
and int(previous_export.seq) > current_seq:
# something weird happened (like the couch database changing)
# better rebuild from scratch
previous_export = None
config = ExportConfiguration(db,
schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def test_get_all_checkpoints(self):
index = ["mydomain", "myxmlns"]
self.addCleanup(lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)])
schema1 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema1.save()
schema1_prime, = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
schema2 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema2.save()
schema1_prime, schema2_prime = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
self.assert_docs_equal(schema2_prime, schema2)
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
current_seq = db.info()["update_seq"]
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(get_db(), schema_index,
previous_export=previous_export)
schema = get_schema_new(config)
if not schema:
return None
updated_checkpoint = ExportSchema(seq=current_seq, schema=schema,
index=schema_index)
updated_checkpoint.save()
return updated_checkpoint
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
if post_data['presave'] and not self.allow_daily_saved:
raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports"))
if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard:
raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard"))
if custom_export_json["is_safe"] and not self.allow_deid:
raise BadExportConfiguration(_("This user does not have permission to create a de-identified export"))
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
if post_data['presave'] and not self.allow_daily_saved:
raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports"))
if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard:
raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard"))
if custom_export_json["is_safe"] and not self.allow_deid:
raise BadExportConfiguration(_("This user does not have permission to create a de-identified export"))
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def handle(self, *args, **options):
if len(args) < 1: raise CommandError('Please specify %s.' % self.label)
index_in = args[0]
if index_in == "all":
to_update = ExportSchema.get_all_indices()
else:
to_update = [json.loads(index_in)]
for index in to_update:
processed = rebuild_schemas(index)
print "processed %s checkpoints matching %s" % (processed, index)
def export(
schema_index,
file,
format=Format.XLS_2007,
previous_export_id=None,
filter=None,
max_column_size=2000,
separator="|",
):
"""
Exports data from couch documents matching a given tag to a file.
Returns true if it finds data, otherwise nothing
"""
previous_export = ExportSchema.get(previous_export_id) if previous_export_id else None
database = get_db()
config = ExportConfiguration(database, schema_index, previous_export, filter)
# handle empty case
if not config.potentially_relevant_ids:
return None
# get and checkpoint the latest schema
updated_schema = get_schema_new(config)
export_schema_checkpoint = ExportSchema(seq=config.current_seq, schema=updated_schema, index=config.schema_index)
export_schema_checkpoint.save()
# transform docs onto output and save
writer = get_writer(format)
# open the doc and the headers
formatted_headers = get_headers(updated_schema, separator=separator)
writer.open(formatted_headers, file)
for doc in config.get_docs():
writer.write(
format_tables(create_intermediate_tables(doc, updated_schema), include_headers=False, separator=separator)
)
writer.close()
return export_schema_checkpoint
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
schema = ExportSchema.get(self.request.POST["schema"])
self.custom_export.index = schema.index
self.custom_export.schema_id = self.request.POST["schema"]
self.custom_export.name = self.request.POST["name"]
self.custom_export.default_format = self.request.POST["format"] or Format.XLS_2007
self.custom_export.is_safe = bool(self.request.POST.get('is_safe'))
self.presave = bool(self.request.POST.get('presave'))
table = self.request.POST["table"]
cols = self.request.POST['order'].strip().split()
@list
@inline
def export_cols():
for col in cols:
transform = self.request.POST.get('%s transform' % col) or None
if transform:
transform = SerializableFunction.loads(transform)
yield ExportColumn(
index=col,
display=self.request.POST["%s display" % col],
transform=transform
)
export_table = ExportTable(index=table, display=self.request.POST["name"], columns=export_cols)
self.custom_export.tables = [export_table]
self.custom_export.order = cols
table_dict = dict([t.index, t] for t in self.custom_export.tables)
if table in table_dict:
table_dict[table].columns = export_cols
else:
self.custom_export.tables.append(ExportTable(index=table,
display=self.custom_export.name,
columns=export_cols))
if self.export_type == 'form':
self.custom_export.include_errors = bool(self.request.POST.get("include-errors"))
self.custom_export.app_id = self.request.POST.get('app_id')
self.custom_export.save()
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(db, schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def update_custom_export(self):
"""
Updates custom_export object from the request
and saves to the db
"""
post_data = self.post_data
custom_export_json = post_data['custom_export']
SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates')
for key in SAFE_KEYS:
self.custom_export[key] = custom_export_json[key]
# update the custom export index (to stay in sync)
schema_id = self.custom_export.schema_id
schema = ExportSchema.get(schema_id)
self.custom_export.index = schema.index
self.presave = post_data['presave']
self.export_stock = post_data['export_stock']
self.custom_export.tables = [
ExportTable.wrap(table)
for table in custom_export_json['tables']
]
table_dict = dict((t.index, t) for t in self.custom_export.tables)
for table in self.custom_export.tables:
if table.index in table_dict:
table_dict[table.index].columns = table.columns
else:
self.custom_export.tables.append(
ExportTable(
index=table.index,
display=self.custom_export.name,
columns=table.columns
)
)
self.update_custom_params()
self.custom_export.custom_validate()
self.custom_export.save()
touch_exports(self.domain)
if self.presave:
HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id)
else:
HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
return self.custom_export.get_id
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(db,
schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def testExportTokenMigration(self):
c = Client()
c.login(**{'username': '******', 'password': '******'})
submit_form()
time.sleep(1)
resp = get_export_response(c)
self.assertEqual(200, resp.status_code)
self.assertTrue(resp.content is not None)
self.assertTrue("X-CommCareHQ-Export-Token" in resp)
# blow away the timestamp property to ensure we're testing the
# migration case
prev_token = resp["X-CommCareHQ-Export-Token"]
prev_checkpoint = ExportSchema.get(prev_token)
assert prev_checkpoint.timestamp
prev_checkpoint.timestamp = None
prev_checkpoint.save()
prev_checkpoint = ExportSchema.get(prev_token)
assert not prev_checkpoint.timestamp
# data but no new data = redirect
resp = get_export_response(c, prev_token)
self.assertEqual(302, resp.status_code)
submit_form()
time.sleep(1)
resp = get_export_response(c, prev_token)
self.assertEqual(200, resp.status_code)
self.assertTrue(resp.content is not None)
self.assertTrue("X-CommCareHQ-Export-Token" in resp)
prev_token = resp["X-CommCareHQ-Export-Token"]
full_data = get_export_response(c).content
partial_data = get_export_response(c, prev_token).content
self.assertTrue(len(full_data) > len(partial_data))
def testExportTokenMigration(self):
c = Client()
c.login(**{'username': '******', 'password': '******'})
_submit_form()
time.sleep(1)
resp = get_export_response(c)
self.assertEqual(200, resp.status_code)
self.assertTrue(_content(resp) is not None)
self.assertTrue("X-CommCareHQ-Export-Token" in resp)
# blow away the timestamp property to ensure we're testing the
# migration case
prev_token = resp["X-CommCareHQ-Export-Token"]
prev_checkpoint = ExportSchema.get(prev_token)
assert prev_checkpoint.timestamp
def testExportTokenMigration(self):
c = Client()
c.login(**{'username': '******', 'password': '******'})
self._submit_form()
time.sleep(1)
resp = get_export_response(c)
self.assertEqual(200, resp.status_code)
self.assertTrue(_content(resp) is not None)
self.assertTrue("X-CommCareHQ-Export-Token" in resp)
# blow away the timestamp property to ensure we're testing the
# migration case
prev_token = resp["X-CommCareHQ-Export-Token"]
prev_checkpoint = ExportSchema.get(prev_token)
assert prev_checkpoint.timestamp
def get_export_components(schema_index, previous_export_id=None, filter=None):
"""
Get all the components needed to build an export file.
"""
previous_export = ExportSchema.get(previous_export_id) if previous_export_id else None
database = get_db()
config = ExportConfiguration(database, schema_index, previous_export, filter)
# handle empty case
if not config.potentially_relevant_ids:
return None, None, None
# get and checkpoint the latest schema
updated_schema = config.get_latest_schema()
export_schema_checkpoint = config.create_new_checkpoint()
return config, updated_schema, export_schema_checkpoint
def testSaveAndLoad(self):
index = ["foo", 2]
schema = ExportSchema(seq="5", index=index, timestamp=datetime.now())
inner = {"dict": {"bar": 1, "baz": [2,3]},
"list": ["foo", "bar"],
"dictlist": [{"bip": 1, "bop": "blah"},
{"bip": 2, "bop": "blah2"}],
"item": "yoyoyo"}
schema.schema = inner
schema.save()
back = ExportSchema.get(schema.get_id)
self.assertEqual(inner, back.schema)
self.assertEqual(index, back.index)
def get_export_components(schema_index, previous_export_id=None, filter=None):
"""
Get all the components needed to build an export file.
"""
previous_export = ExportSchema.get(previous_export_id)\
if previous_export_id else None
database = get_db()
config = ExportConfiguration(database, schema_index, previous_export,
filter)
# handle empty case
if not config.potentially_relevant_ids:
return None, None, None
# get and checkpoint the latest schema
updated_schema = config.get_latest_schema()
export_schema_checkpoint = config.create_new_checkpoint()
return config, updated_schema, export_schema_checkpoint
def test_get_all_checkpoints(self):
index = ["mydomain", "myxmlns"]
self.addCleanup(
lambda:
[cp.delete() for cp in ExportSchema.get_all_checkpoints(index)])
schema1 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema1.save()
schema1_prime, = list(ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
schema2 = ExportSchema(index=index, timestamp=datetime.utcnow())
schema2.save()
schema1_prime, schema2_prime = list(
ExportSchema.get_all_checkpoints(index))
self.assert_docs_equal(schema1_prime, schema1)
self.assert_docs_equal(schema2_prime, schema2)
def test_get_last(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.addCleanup(
lambda idx:
[cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)],
index)
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assert_docs_equal(schema1, ExportSchema.last(index))
schema2 = ExportSchema(index=index,
timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assert_docs_equal(schema2, ExportSchema.last(index))
schema3 = ExportSchema(index=index,
timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
# still schema2 (which has a later date than schema3)
self.assert_docs_equal(schema2, ExportSchema.last(index))
def testGetLast(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
# by design, if something has a timestamp it always wins out, even
# if something has a higher seq
dt = datetime.utcnow()
schema1 = ExportSchema(seq="2", index=index, timestamp=dt)
schema1.save(**save_args)
self.assertEqual(schema1._id, ExportSchema.last(index)._id)
schema2 = ExportSchema(seq="1", index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
schema3 = ExportSchema(seq="3", index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def last_checkpoint(self):
return self.previous_export or ExportSchema.last(self.schema_index)
def last_checkpoint(self):
return None if self.disable_checkpoints else ExportSchema.last(self.schema_index)
def test_get_last(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.addCleanup(
lambda idx: [cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)],
index
)
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assert_docs_equal(schema1, ExportSchema.last(index))
schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assert_docs_equal(schema2, ExportSchema.last(index))
schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
# still schema2 (which has a later date than schema3)
self.assert_docs_equal(schema2, ExportSchema.last(index))
def testGetLast(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assertEqual(schema1._id, ExportSchema.last(index)._id)
schema2 = ExportSchema(index=index,
timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
schema3 = ExportSchema(index=index,
timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def testGetLast(self):
indices = ["a string", ["a", "list"]]
save_args = get_safe_write_kwargs()
for index in indices:
self.assertEqual(None, ExportSchema.last(index))
dt = datetime.utcnow()
schema1 = ExportSchema(index=index, timestamp=dt)
schema1.save(**save_args)
self.assertEqual(schema1._id, ExportSchema.last(index)._id)
schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
schema2.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
schema3.save(**save_args)
self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def last_checkpoint(self):
return None if self.disable_checkpoints else ExportSchema.last(
self.schema_index)
def create_basic_form_checkpoint(index):
checkpoint = ExportSchema(seq="0", schema=BASIC_FORM_SCHEMA, timestamp=datetime.utcnow(), index=index)
checkpoint.save()
return checkpoint