def create_new_checkpoint(self): checkpoint = ExportSchema(seq=str(self.current_seq), schema=self.get_latest_schema(), timestamp=self.timestamp, index=self.schema_index) checkpoint.save() return checkpoint
def test_wrap_datetime_min(self): schema_bad = ExportSchema( schema={}, timestamp=datetime.min, index='index', ) schema_good = ExportSchema.wrap(schema_bad.to_json()) self.assertEqual(schema_good.timestamp, datetime(1970, 1, 1))
def create_basic_form_checkpoint(index): checkpoint = ExportSchema( schema=BASIC_FORM_SCHEMA, timestamp=datetime(1970, 1, 1), index=index, ) checkpoint.save() return checkpoint
def test_wrap_datetime_hippy(self): schema1 = ExportSchema( schema={}, timestamp=datetime(1970, 1, 2), index='index', ) schema2 = ExportSchema.wrap(schema1.to_json()) self.assertEqual(schema2.timestamp, datetime(1970, 1, 2))
def create_new_checkpoint(self): checkpoint = ExportSchema( schema=self.get_latest_schema(), timestamp=self.timestamp, index=self.schema_index, ) checkpoint.save() return checkpoint
def handle(self, *args, **options): db = ExportSchema.get_db() for index in ExportSchema.get_all_indices(): last = ExportSchema.last(index) if not last.timestamp: config = ExportConfiguration(db, index, disable_checkpoints=True) config.create_new_checkpoint() assert ExportSchema.last(index).timestamp print "set timestamp for %s" % index else: print "%s all set" % index
def rebuild_schemas(index): """ Resets the schema for all checkpoints to the latest version based off the current document structure. Returns the number of checkpoints updated. """ db = ExportSchema.get_db() all_checkpoints = ExportSchema.get_all_checkpoints(index) config = ExportConfiguration(db, index, disable_checkpoints=True) latest = config.create_new_checkpoint() for cp in all_checkpoints: cp.schema = latest.schema cp.save() return len(all_checkpoints)
def testSaveAndLoad(self): index = ["foo", 2] schema = ExportSchema(seq=5, index=index) inner = {"dict": {"bar": 1, "baz": [2,3]}, "list": ["foo", "bar"], "dictlist": [{"bip": 1, "bop": "blah"}, {"bip": 2, "bop": "blah2"}], "item": "yoyoyo"} schema.schema = inner schema.save() back = ExportSchema.get(schema.get_id) self.assertEqual(inner, back.schema) self.assertEqual(index, back.index)
def build_latest_schema(schema_index): """ Build a schema, directly from the index. Also creates a saved checkpoint. """ from couchexport.export import ExportConfiguration db = Database(settings.COUCH_DATABASE) previous_export = ExportSchema.last(schema_index) try: current_seq = int(db.info()["update_seq"]) except ValueError: pass # we must be on bigcouch, so comparing seqs is useless else: if previous_export and not previous_export.is_bigcouch \ and int(previous_export.seq) > current_seq: # something weird happened (like the couch database changing) # better rebuild from scratch previous_export = None config = ExportConfiguration(db, schema_index, previous_export=previous_export) schema = config.get_latest_schema() if not schema: return None updated_checkpoint = config.create_new_checkpoint() return updated_checkpoint
def test_get_all_checkpoints(self): index = ["mydomain", "myxmlns"] self.addCleanup(lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)]) schema1 = ExportSchema(index=index, timestamp=datetime.utcnow()) schema1.save() schema1_prime, = list(ExportSchema.get_all_checkpoints(index)) self.assert_docs_equal(schema1_prime, schema1) schema2 = ExportSchema(index=index, timestamp=datetime.utcnow()) schema2.save() schema1_prime, schema2_prime = list(ExportSchema.get_all_checkpoints(index)) self.assert_docs_equal(schema1_prime, schema1) self.assert_docs_equal(schema2_prime, schema2)
def build_latest_schema(schema_index): """ Build a schema, directly from the index. Also creates a saved checkpoint. """ from couchexport.export import ExportConfiguration db = Database(settings.COUCH_DATABASE) current_seq = db.info()["update_seq"] previous_export = ExportSchema.last(schema_index) config = ExportConfiguration(get_db(), schema_index, previous_export=previous_export) schema = get_schema_new(config) if not schema: return None updated_checkpoint = ExportSchema(seq=current_seq, schema=schema, index=schema_index) updated_checkpoint.save() return updated_checkpoint
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ post_data = self.post_data custom_export_json = post_data['custom_export'] if post_data['presave'] and not self.allow_daily_saved: raise BadExportConfiguration(_("This user does not have permission to create Daily Saved Exports")) if custom_export_json['default_format'] == "html" and not self.allow_excel_dashboard: raise BadExportConfiguration(_("This user does not have permission to create an excel dashboard")) if custom_export_json["is_safe"] and not self.allow_deid: raise BadExportConfiguration(_("This user does not have permission to create a de-identified export")) SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates') for key in SAFE_KEYS: self.custom_export[key] = custom_export_json[key] # update the custom export index (to stay in sync) schema_id = self.custom_export.schema_id schema = ExportSchema.get(schema_id) self.custom_export.index = schema.index self.presave = post_data['presave'] self.export_stock = post_data['export_stock'] self.custom_export.tables = [ ExportTable.wrap(table) for table in custom_export_json['tables'] ] table_dict = dict((t.index, t) for t in self.custom_export.tables) for table in self.custom_export.tables: if table.index in table_dict: table_dict[table.index].columns = table.columns else: self.custom_export.tables.append( ExportTable( index=table.index, display=self.custom_export.name, columns=table.columns ) ) self.update_custom_params() self.custom_export.custom_validate() self.custom_export.save() touch_exports(self.domain) if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id) return self.custom_export.get_id
def handle(self, *args, **options): if len(args) < 1: raise CommandError('Please specify %s.' % self.label) index_in = args[0] if index_in == "all": to_update = ExportSchema.get_all_indices() else: to_update = [json.loads(index_in)] for index in to_update: processed = rebuild_schemas(index) print "processed %s checkpoints matching %s" % (processed, index)
def export( schema_index, file, format=Format.XLS_2007, previous_export_id=None, filter=None, max_column_size=2000, separator="|", ): """ Exports data from couch documents matching a given tag to a file. Returns true if it finds data, otherwise nothing """ previous_export = ExportSchema.get(previous_export_id) if previous_export_id else None database = get_db() config = ExportConfiguration(database, schema_index, previous_export, filter) # handle empty case if not config.potentially_relevant_ids: return None # get and checkpoint the latest schema updated_schema = get_schema_new(config) export_schema_checkpoint = ExportSchema(seq=config.current_seq, schema=updated_schema, index=config.schema_index) export_schema_checkpoint.save() # transform docs onto output and save writer = get_writer(format) # open the doc and the headers formatted_headers = get_headers(updated_schema, separator=separator) writer.open(formatted_headers, file) for doc in config.get_docs(): writer.write( format_tables(create_intermediate_tables(doc, updated_schema), include_headers=False, separator=separator) ) writer.close() return export_schema_checkpoint
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ schema = ExportSchema.get(self.request.POST["schema"]) self.custom_export.index = schema.index self.custom_export.schema_id = self.request.POST["schema"] self.custom_export.name = self.request.POST["name"] self.custom_export.default_format = self.request.POST["format"] or Format.XLS_2007 self.custom_export.is_safe = bool(self.request.POST.get('is_safe')) self.presave = bool(self.request.POST.get('presave')) table = self.request.POST["table"] cols = self.request.POST['order'].strip().split() @list @inline def export_cols(): for col in cols: transform = self.request.POST.get('%s transform' % col) or None if transform: transform = SerializableFunction.loads(transform) yield ExportColumn( index=col, display=self.request.POST["%s display" % col], transform=transform ) export_table = ExportTable(index=table, display=self.request.POST["name"], columns=export_cols) self.custom_export.tables = [export_table] self.custom_export.order = cols table_dict = dict([t.index, t] for t in self.custom_export.tables) if table in table_dict: table_dict[table].columns = export_cols else: self.custom_export.tables.append(ExportTable(index=table, display=self.custom_export.name, columns=export_cols)) if self.export_type == 'form': self.custom_export.include_errors = bool(self.request.POST.get("include-errors")) self.custom_export.app_id = self.request.POST.get('app_id') self.custom_export.save() if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id)
def build_latest_schema(schema_index): """ Build a schema, directly from the index. Also creates a saved checkpoint. """ from couchexport.export import ExportConfiguration db = Database(settings.COUCH_DATABASE) previous_export = ExportSchema.last(schema_index) config = ExportConfiguration(db, schema_index, previous_export=previous_export) schema = config.get_latest_schema() if not schema: return None updated_checkpoint = config.create_new_checkpoint() return updated_checkpoint
def update_custom_export(self): """ Updates custom_export object from the request and saves to the db """ post_data = self.post_data custom_export_json = post_data['custom_export'] SAFE_KEYS = ('default_format', 'is_safe', 'name', 'schema_id', 'transform_dates') for key in SAFE_KEYS: self.custom_export[key] = custom_export_json[key] # update the custom export index (to stay in sync) schema_id = self.custom_export.schema_id schema = ExportSchema.get(schema_id) self.custom_export.index = schema.index self.presave = post_data['presave'] self.export_stock = post_data['export_stock'] self.custom_export.tables = [ ExportTable.wrap(table) for table in custom_export_json['tables'] ] table_dict = dict((t.index, t) for t in self.custom_export.tables) for table in self.custom_export.tables: if table.index in table_dict: table_dict[table.index].columns = table.columns else: self.custom_export.tables.append( ExportTable( index=table.index, display=self.custom_export.name, columns=table.columns ) ) self.update_custom_params() self.custom_export.custom_validate() self.custom_export.save() touch_exports(self.domain) if self.presave: HQGroupExportConfiguration.add_custom_export(self.domain, self.custom_export.get_id) else: HQGroupExportConfiguration.remove_custom_export(self.domain, self.custom_export.get_id) return self.custom_export.get_id
def testExportTokenMigration(self): c = Client() c.login(**{'username': '******', 'password': '******'}) submit_form() time.sleep(1) resp = get_export_response(c) self.assertEqual(200, resp.status_code) self.assertTrue(resp.content is not None) self.assertTrue("X-CommCareHQ-Export-Token" in resp) # blow away the timestamp property to ensure we're testing the # migration case prev_token = resp["X-CommCareHQ-Export-Token"] prev_checkpoint = ExportSchema.get(prev_token) assert prev_checkpoint.timestamp prev_checkpoint.timestamp = None prev_checkpoint.save() prev_checkpoint = ExportSchema.get(prev_token) assert not prev_checkpoint.timestamp # data but no new data = redirect resp = get_export_response(c, prev_token) self.assertEqual(302, resp.status_code) submit_form() time.sleep(1) resp = get_export_response(c, prev_token) self.assertEqual(200, resp.status_code) self.assertTrue(resp.content is not None) self.assertTrue("X-CommCareHQ-Export-Token" in resp) prev_token = resp["X-CommCareHQ-Export-Token"] full_data = get_export_response(c).content partial_data = get_export_response(c, prev_token).content self.assertTrue(len(full_data) > len(partial_data))
def testExportTokenMigration(self): c = Client() c.login(**{'username': '******', 'password': '******'}) _submit_form() time.sleep(1) resp = get_export_response(c) self.assertEqual(200, resp.status_code) self.assertTrue(_content(resp) is not None) self.assertTrue("X-CommCareHQ-Export-Token" in resp) # blow away the timestamp property to ensure we're testing the # migration case prev_token = resp["X-CommCareHQ-Export-Token"] prev_checkpoint = ExportSchema.get(prev_token) assert prev_checkpoint.timestamp
def testExportTokenMigration(self): c = Client() c.login(**{'username': '******', 'password': '******'}) self._submit_form() time.sleep(1) resp = get_export_response(c) self.assertEqual(200, resp.status_code) self.assertTrue(_content(resp) is not None) self.assertTrue("X-CommCareHQ-Export-Token" in resp) # blow away the timestamp property to ensure we're testing the # migration case prev_token = resp["X-CommCareHQ-Export-Token"] prev_checkpoint = ExportSchema.get(prev_token) assert prev_checkpoint.timestamp
def get_export_components(schema_index, previous_export_id=None, filter=None): """ Get all the components needed to build an export file. """ previous_export = ExportSchema.get(previous_export_id) if previous_export_id else None database = get_db() config = ExportConfiguration(database, schema_index, previous_export, filter) # handle empty case if not config.potentially_relevant_ids: return None, None, None # get and checkpoint the latest schema updated_schema = config.get_latest_schema() export_schema_checkpoint = config.create_new_checkpoint() return config, updated_schema, export_schema_checkpoint
def testSaveAndLoad(self): index = ["foo", 2] schema = ExportSchema(seq="5", index=index, timestamp=datetime.now()) inner = {"dict": {"bar": 1, "baz": [2,3]}, "list": ["foo", "bar"], "dictlist": [{"bip": 1, "bop": "blah"}, {"bip": 2, "bop": "blah2"}], "item": "yoyoyo"} schema.schema = inner schema.save() back = ExportSchema.get(schema.get_id) self.assertEqual(inner, back.schema) self.assertEqual(index, back.index)
def get_export_components(schema_index, previous_export_id=None, filter=None): """ Get all the components needed to build an export file. """ previous_export = ExportSchema.get(previous_export_id)\ if previous_export_id else None database = get_db() config = ExportConfiguration(database, schema_index, previous_export, filter) # handle empty case if not config.potentially_relevant_ids: return None, None, None # get and checkpoint the latest schema updated_schema = config.get_latest_schema() export_schema_checkpoint = config.create_new_checkpoint() return config, updated_schema, export_schema_checkpoint
def test_get_all_checkpoints(self): index = ["mydomain", "myxmlns"] self.addCleanup( lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)]) schema1 = ExportSchema(index=index, timestamp=datetime.utcnow()) schema1.save() schema1_prime, = list(ExportSchema.get_all_checkpoints(index)) self.assert_docs_equal(schema1_prime, schema1) schema2 = ExportSchema(index=index, timestamp=datetime.utcnow()) schema2.save() schema1_prime, schema2_prime = list( ExportSchema.get_all_checkpoints(index)) self.assert_docs_equal(schema1_prime, schema1) self.assert_docs_equal(schema2_prime, schema2)
def test_get_last(self): indices = ["a string", ["a", "list"]] save_args = get_safe_write_kwargs() for index in indices: self.addCleanup( lambda idx: [cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)], index) for index in indices: self.assertEqual(None, ExportSchema.last(index)) dt = datetime.utcnow() schema1 = ExportSchema(index=index, timestamp=dt) schema1.save(**save_args) self.assert_docs_equal(schema1, ExportSchema.last(index)) schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1)) schema2.save(**save_args) self.assert_docs_equal(schema2, ExportSchema.last(index)) schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1)) schema3.save(**save_args) # still schema2 (which has a later date than schema3) self.assert_docs_equal(schema2, ExportSchema.last(index))
def testGetLast(self): indices = ["a string", ["a", "list"]] save_args = get_safe_write_kwargs() for index in indices: self.assertEqual(None, ExportSchema.last(index)) # by design, if something has a timestamp it always wins out, even # if something has a higher seq dt = datetime.utcnow() schema1 = ExportSchema(seq="2", index=index, timestamp=dt) schema1.save(**save_args) self.assertEqual(schema1._id, ExportSchema.last(index)._id) schema2 = ExportSchema(seq="1", index=index, timestamp=dt + timedelta(seconds=1)) schema2.save(**save_args) self.assertEqual(schema2._id, ExportSchema.last(index)._id) schema3 = ExportSchema(seq="3", index=index, timestamp=dt - timedelta(seconds=1)) schema3.save(**save_args) self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def last_checkpoint(self): return self.previous_export or ExportSchema.last(self.schema_index)
def last_checkpoint(self): return None if self.disable_checkpoints else ExportSchema.last(self.schema_index)
def test_get_last(self): indices = ["a string", ["a", "list"]] save_args = get_safe_write_kwargs() for index in indices: self.addCleanup( lambda idx: [cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)], index ) for index in indices: self.assertEqual(None, ExportSchema.last(index)) dt = datetime.utcnow() schema1 = ExportSchema(index=index, timestamp=dt) schema1.save(**save_args) self.assert_docs_equal(schema1, ExportSchema.last(index)) schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1)) schema2.save(**save_args) self.assert_docs_equal(schema2, ExportSchema.last(index)) schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1)) schema3.save(**save_args) # still schema2 (which has a later date than schema3) self.assert_docs_equal(schema2, ExportSchema.last(index))
def testGetLast(self): indices = ["a string", ["a", "list"]] save_args = get_safe_write_kwargs() for index in indices: self.assertEqual(None, ExportSchema.last(index)) dt = datetime.utcnow() schema1 = ExportSchema(index=index, timestamp=dt) schema1.save(**save_args) self.assertEqual(schema1._id, ExportSchema.last(index)._id) schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1)) schema2.save(**save_args) self.assertEqual(schema2._id, ExportSchema.last(index)._id) schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1)) schema3.save(**save_args) self.assertEqual(schema2._id, ExportSchema.last(index)._id)
def last_checkpoint(self): return None if self.disable_checkpoints else ExportSchema.last( self.schema_index)
def create_basic_form_checkpoint(index): checkpoint = ExportSchema(seq="0", schema=BASIC_FORM_SCHEMA, timestamp=datetime.utcnow(), index=index) checkpoint.save() return checkpoint