def build_latest_schema(schema_index): """ Build a schema, directly from the index. Also creates a saved checkpoint. """ from couchexport.export import ExportConfiguration db = Database(settings.COUCH_DATABASE) previous_export = ExportSchema.last(schema_index) try: current_seq = int(db.info()["update_seq"]) except ValueError: pass # we must be on bigcouch, so comparing seqs is useless else: if previous_export and not previous_export.is_bigcouch \ and int(previous_export.seq) > current_seq: # something weird happened (like the couch database changing) # better rebuild from scratch previous_export = None config = ExportConfiguration(db, schema_index, previous_export=previous_export) schema = config.get_latest_schema() if not schema: return None updated_checkpoint = config.create_new_checkpoint() return updated_checkpoint
def testComputedRemoved(self): db = get_db() db.save_doc({ '#export_tag': 'tag', 'tag': 'computed-test', 'p1': 'v1', 'computed_': { 'ignore': 'this stuff' } }, **get_safe_write_kwargs() ) config = ExportConfiguration(db, ['computed-test'], cleanup_fn=None) schema = config.get_latest_schema() self.assertTrue('computed_' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertTrue('computed_' in docs[0]) # check that it works both explicitly and by default explicit_config = ExportConfiguration(db, ['computed-test'], cleanup_fn=clear_computed) default_config = ExportConfiguration(db, ['computed-test']) for config in (explicit_config, default_config): schema = config.get_latest_schema() self.assertFalse('computed_' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertFalse('computed_' in docs[0])
def download_data(self, format="", previous_export=None, filter=None): """ If there is data, return an HTTPResponse with the appropriate data. If there is not data returns None. """ from couchexport.shortcuts import export_response from couchexport.export import get_writer, get_schema_new, format_tables, create_intermediate_tables if not format: format = self.default_format or Format.XLS_2007 from couchexport.export import ExportConfiguration database = get_db() config = ExportConfiguration(database, self.index, previous_export, util.intersect_filters(self.filter, filter)) # get and checkpoint the latest schema updated_schema = get_schema_new(config) export_schema_checkpoint = ExportSchema( seq=config.current_seq, schema=updated_schema, index=config.schema_index ) export_schema_checkpoint.save() # transform docs onto output and save writer = get_writer(format) # open the doc and the headers formatted_headers = self.get_table_headers() tmp = StringIO() writer.open(formatted_headers, tmp) for doc in config.get_docs(): writer.write(self.trim(format_tables(create_intermediate_tables(doc, updated_schema), separator="."))) writer.close() return export_response(tmp, format, self.name)
def testAttachmentsRemoved(self): db = get_db() res = db.save_doc({ '#export_tag': 'tag', 'tag': 'attachments-test', 'p1': 'v1', }, **get_safe_write_kwargs() ) doc = db.get(res['id']) doc['_attachments'] = { "attach.txt": { "content_type": "text/plain", "data": "some content", }, } db.save_doc(doc) config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=None) schema = config.get_latest_schema() self.assertTrue('_attachments' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertTrue('_attachments' in docs[0]) # check that it works both explicitly and by default explicit_config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=clear_attachments) default_config = ExportConfiguration(db, ['attachments-test']) for config in (explicit_config, default_config): schema = config.get_latest_schema() self.assertFalse('_attachments' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertFalse('_attachments' in docs[0])
def handle(self, *args, **options): db = ExportSchema.get_db() for index in ExportSchema.get_all_indices(): last = ExportSchema.last(index) if not last.timestamp: config = ExportConfiguration(db, index, disable_checkpoints=True) config.create_new_checkpoint() assert ExportSchema.last(index).timestamp print "set timestamp for %s" % index else: print "%s all set" % index
def get_export_components(self, previous_export_id=None, filter=None): from couchexport.export import ExportConfiguration database = get_db() config = ExportConfiguration(database, self.index, previous_export_id, self.filter & filter) # get and checkpoint the latest schema updated_schema = config.get_latest_schema() export_schema_checkpoint = config.create_new_checkpoint() return config, updated_schema, export_schema_checkpoint
def rebuild_schemas(index): """ Resets the schema for all checkpoints to the latest version based off the current document structure. Returns the number of checkpoints updated. """ db = ExportSchema.get_db() all_checkpoints = ExportSchema.get_all_checkpoints(index) config = ExportConfiguration(db, index, disable_checkpoints=True) latest = config.create_new_checkpoint() for cp in all_checkpoints: cp.schema = latest.schema cp.save() return len(all_checkpoints)
def build_latest_schema(schema_index): """ Build a schema, directly from the index. Also creates a saved checkpoint. """ from couchexport.export import ExportConfiguration db = Database(settings.COUCH_DATABASE) previous_export = ExportSchema.last(schema_index) config = ExportConfiguration(db, schema_index, previous_export=previous_export) schema = config.get_latest_schema() if not schema: return None updated_checkpoint = config.create_new_checkpoint() return updated_checkpoint
def testAttachmentsRemoved(self): db = get_db() res = db.save_doc( { '#export_tag': 'tag', 'tag': 'attachments-test', 'p1': 'v1', }, **get_safe_write_kwargs()) doc = db.get(res['id']) doc['_attachments'] = { "attach.txt": { "content_type": "text/plain", "data": "some content", }, } db.save_doc(doc) config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=None) schema = config.get_latest_schema() self.assertTrue('_attachments' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertTrue('_attachments' in docs[0]) # check that it works both explicitly and by default explicit_config = ExportConfiguration(db, ['attachments-test'], cleanup_fn=clear_attachments) default_config = ExportConfiguration(db, ['attachments-test']) for config in (explicit_config, default_config): schema = config.get_latest_schema() self.assertFalse('_attachments' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertFalse('_attachments' in docs[0])
def testComputedRemoved(self): db = get_db() db.save_doc( { '#export_tag': 'tag', 'tag': 'computed-test', 'p1': 'v1', 'computed_': { 'ignore': 'this stuff' } }, **get_safe_write_kwargs()) config = ExportConfiguration(db, ['computed-test'], cleanup_fn=None) schema = config.get_latest_schema() self.assertTrue('computed_' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertTrue('computed_' in docs[0]) # check that it works both explicitly and by default explicit_config = ExportConfiguration(db, ['computed-test'], cleanup_fn=clear_computed) default_config = ExportConfiguration(db, ['computed-test']) for config in (explicit_config, default_config): schema = config.get_latest_schema() self.assertFalse('computed_' in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertFalse('computed_' in docs[0])
def to_export_config(self): """ Return an ExportConfiguration object that represents this. """ # confusingly, the index isn't the actual index property, # but is the index appended with the id to this document. # this is to avoid conflicts among multiple exports index = "%s-%s" % (self.index, self._id) if isinstance(self.index, six.string_types) else \ self.index + [self._id] # self.index required to be a string or list return ExportConfiguration(index=index, name=self.name, format=self.default_format)
def testAttachmentsRemoved(self): db = get_db() res = db.save_doc({"#export_tag": "tag", "tag": "attachments-test", "p1": "v1"}, **get_safe_write_kwargs()) doc = db.get(res["id"]) db.put_attachment(doc, "some content", "attach.txt") config = ExportConfiguration(db, ["attachments-test"], cleanup_fn=None) schema = config.get_latest_schema() self.assertTrue("_attachments" in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertTrue("_attachments" in docs[0]) # check that it works both explicitly and by default explicit_config = ExportConfiguration(db, ["attachments-test"], cleanup_fn=clear_attachments) default_config = ExportConfiguration(db, ["attachments-test"]) for config in (explicit_config, default_config): schema = config.get_latest_schema() self.assertFalse("_attachments" in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertFalse("_attachments" in docs[0])
def testComputedRemoved(self): db = get_db() db.save_doc( {"#export_tag": "tag", "tag": "computed-test", "p1": "v1", "computed_": {"ignore": "this stuff"}}, **get_safe_write_kwargs() ) config = ExportConfiguration(db, ["computed-test"], cleanup_fn=None) schema = config.get_latest_schema() self.assertTrue("computed_" in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertTrue("computed_" in docs[0]) # check that it works both explicitly and by default explicit_config = ExportConfiguration(db, ["computed-test"], cleanup_fn=clear_computed) default_config = ExportConfiguration(db, ["computed-test"]) for config in (explicit_config, default_config): schema = config.get_latest_schema() self.assertFalse("computed_" in schema) docs = list(config.get_docs()) self.assertEqual(1, len(docs)) self.assertFalse("computed_" in docs[0])
def _clear_docs(self): config = ExportConfiguration( XFormInstance.get_db(), [DOMAIN, "http://www.commcarehq.org/export/test"]) for form in config.get_docs(): XFormInstance.wrap(form).delete()
def _clear_docs(self): config = ExportConfiguration(XFormInstance.get_db(), [DOMAIN, "http://www.commcarehq.org/export/test"]) for form in config.get_docs(): XFormInstance.wrap(form).delete()