Example #1
0
 def create_new_checkpoint(self):
     checkpoint = ExportSchema(seq=str(self.current_seq),
                               schema=self.get_latest_schema(),
                               timestamp=self.timestamp,
                               index=self.schema_index)
     checkpoint.save()
     return checkpoint
Example #2
0
 def create_new_checkpoint(self):
     checkpoint = ExportSchema(
         schema=self.get_latest_schema(),
         timestamp=self.timestamp,
         index=self.schema_index,
     )
     checkpoint.save()
     return checkpoint
Example #3
0
def create_basic_form_checkpoint(index):
    checkpoint = ExportSchema(
        schema=BASIC_FORM_SCHEMA,
        timestamp=datetime(1970, 1, 1),
        index=index,
    )
    checkpoint.save()
    return checkpoint
Example #4
0
def create_basic_form_checkpoint(index):
    checkpoint = ExportSchema(
        schema=BASIC_FORM_SCHEMA,
        timestamp=datetime(1970, 1, 1),
        index=index,
    )
    checkpoint.save()
    return checkpoint
Example #5
0
 def testSaveAndLoad(self):
     index = ["foo", 2]
     schema = ExportSchema(seq=5, index=index)
     inner = {"dict": {"bar": 1, "baz": [2,3]},
              "list": ["foo", "bar"],
              "dictlist": [{"bip": 1, "bop": "blah"},
                           {"bip": 2, "bop": "blah2"}],
              "item": "yoyoyo"}
     schema.schema = inner
     schema.save()
     back = ExportSchema.get(schema.get_id)
     self.assertEqual(inner, back.schema)
     self.assertEqual(index, back.index)
Example #6
0
    def test_get_all_checkpoints(self):
        index = ["mydomain", "myxmlns"]
        self.addCleanup(lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)])

        schema1 = ExportSchema(index=index, timestamp=datetime.utcnow())
        schema1.save()
        schema1_prime, = list(ExportSchema.get_all_checkpoints(index))
        self.assert_docs_equal(schema1_prime, schema1)
        schema2 = ExportSchema(index=index, timestamp=datetime.utcnow())
        schema2.save()
        schema1_prime, schema2_prime = list(ExportSchema.get_all_checkpoints(index))
        self.assert_docs_equal(schema1_prime, schema1)
        self.assert_docs_equal(schema2_prime, schema2)
Example #7
0
 def testSaveAndLoad(self):
     index = ["foo", 2]
     schema = ExportSchema(seq="5", index=index, timestamp=datetime.now())
     inner = {"dict": {"bar": 1, "baz": [2,3]},
              "list": ["foo", "bar"],
              "dictlist": [{"bip": 1, "bop": "blah"},
                           {"bip": 2, "bop": "blah2"}],
              "item": "yoyoyo"}
     schema.schema = inner
     schema.save()
     back = ExportSchema.get(schema.get_id)
     self.assertEqual(inner, back.schema)
     self.assertEqual(index, back.index)
Example #8
0
    def test_get_all_checkpoints(self):
        index = ["mydomain", "myxmlns"]
        self.addCleanup(lambda: [cp.delete() for cp in ExportSchema.get_all_checkpoints(index)])

        schema1 = ExportSchema(index=index, timestamp=datetime.utcnow())
        schema1.save()
        schema1_prime, = list(ExportSchema.get_all_checkpoints(index))
        self.assert_docs_equal(schema1_prime, schema1)
        schema2 = ExportSchema(index=index, timestamp=datetime.utcnow())
        schema2.save()
        schema1_prime, schema2_prime = list(ExportSchema.get_all_checkpoints(index))
        self.assert_docs_equal(schema1_prime, schema1)
        self.assert_docs_equal(schema2_prime, schema2)
Example #9
0
def build_latest_schema(schema_index):
    """
    Build a schema, directly from the index. Also creates a saved checkpoint.
    """
    from couchexport.export import ExportConfiguration
    db = Database(settings.COUCH_DATABASE)
    current_seq = db.info()["update_seq"]
    previous_export = ExportSchema.last(schema_index)
    config = ExportConfiguration(get_db(), schema_index, 
                                 previous_export=previous_export)
    schema = get_schema_new(config)
    if not schema:
        return None
    updated_checkpoint = ExportSchema(seq=current_seq, schema=schema, 
                                      index=schema_index)
    updated_checkpoint.save()
    return updated_checkpoint
Example #10
0
    def test_get_last(self):
        indices = ["a string", ["a", "list"]]
        save_args = get_safe_write_kwargs()

        for index in indices:
            self.addCleanup(
                lambda idx:
                [cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)],
                index)

        for index in indices:
            self.assertEqual(None, ExportSchema.last(index))
            dt = datetime.utcnow()
            schema1 = ExportSchema(index=index, timestamp=dt)
            schema1.save(**save_args)
            self.assert_docs_equal(schema1, ExportSchema.last(index))
            schema2 = ExportSchema(index=index,
                                   timestamp=dt + timedelta(seconds=1))
            schema2.save(**save_args)
            self.assert_docs_equal(schema2, ExportSchema.last(index))
            schema3 = ExportSchema(index=index,
                                   timestamp=dt - timedelta(seconds=1))
            schema3.save(**save_args)
            # still schema2 (which has a later date than schema3)
            self.assert_docs_equal(schema2, ExportSchema.last(index))
Example #11
0
def export(
    schema_index,
    file,
    format=Format.XLS_2007,
    previous_export_id=None,
    filter=None,
    max_column_size=2000,
    separator="|",
):
    """
    Exports data from couch documents matching a given tag to a file. 
    Returns true if it finds data, otherwise nothing
    """

    previous_export = ExportSchema.get(previous_export_id) if previous_export_id else None
    database = get_db()
    config = ExportConfiguration(database, schema_index, previous_export, filter)

    # handle empty case
    if not config.potentially_relevant_ids:
        return None

    # get and checkpoint the latest schema
    updated_schema = get_schema_new(config)
    export_schema_checkpoint = ExportSchema(seq=config.current_seq, schema=updated_schema, index=config.schema_index)
    export_schema_checkpoint.save()

    # transform docs onto output and save
    writer = get_writer(format)

    # open the doc and the headers
    formatted_headers = get_headers(updated_schema, separator=separator)
    writer.open(formatted_headers, file)

    for doc in config.get_docs():
        writer.write(
            format_tables(create_intermediate_tables(doc, updated_schema), include_headers=False, separator=separator)
        )
    writer.close()
    return export_schema_checkpoint
Example #12
0
    def testGetLast(self):
        indices = ["a string", ["a", "list"]]
        save_args = get_safe_write_kwargs()

        for index in indices:
            self.assertEqual(None, ExportSchema.last(index))
            dt = datetime.utcnow()
            schema1 = ExportSchema(index=index, timestamp=dt)
            schema1.save(**save_args)
            self.assertEqual(schema1._id, ExportSchema.last(index)._id)
            schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
            schema2.save(**save_args)
            self.assertEqual(schema2._id, ExportSchema.last(index)._id)
            schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
            schema3.save(**save_args)
            self.assertEqual(schema2._id, ExportSchema.last(index)._id)
Example #13
0
    def testGetLast(self):
        indices = ["a string", ["a", "list"]]
        save_args = get_safe_write_kwargs()

        for index in indices:
            self.assertEqual(None, ExportSchema.last(index))
            dt = datetime.utcnow()
            schema1 = ExportSchema(index=index, timestamp=dt)
            schema1.save(**save_args)
            self.assertEqual(schema1._id, ExportSchema.last(index)._id)
            schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
            schema2.save(**save_args)
            self.assertEqual(schema2._id, ExportSchema.last(index)._id)
            schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
            schema3.save(**save_args)
            self.assertEqual(schema2._id, ExportSchema.last(index)._id)
Example #14
0
    def testGetLast(self):
        indices = ["a string", ["a", "list"]]
        save_args = get_safe_write_kwargs()

        for index in indices:
            self.assertEqual(None, ExportSchema.last(index))
            # by design, if something has a timestamp it always wins out, even
            # if something has a higher seq

            dt = datetime.utcnow()
            schema1 = ExportSchema(seq="2", index=index, timestamp=dt)
            schema1.save(**save_args)
            self.assertEqual(schema1._id, ExportSchema.last(index)._id)
            schema2 = ExportSchema(seq="1", index=index, timestamp=dt + timedelta(seconds=1))
            schema2.save(**save_args)
            self.assertEqual(schema2._id, ExportSchema.last(index)._id)
            schema3 = ExportSchema(seq="3", index=index, timestamp=dt - timedelta(seconds=1))
            schema3.save(**save_args)
            self.assertEqual(schema2._id, ExportSchema.last(index)._id)
Example #15
0
    def test_get_last(self):
        indices = ["a string", ["a", "list"]]
        save_args = get_safe_write_kwargs()

        for index in indices:
            self.addCleanup(
                lambda idx: [cp.delete() for cp in ExportSchema.get_all_checkpoints(idx)],
                index
            )

        for index in indices:
            self.assertEqual(None, ExportSchema.last(index))
            dt = datetime.utcnow()
            schema1 = ExportSchema(index=index, timestamp=dt)
            schema1.save(**save_args)
            self.assert_docs_equal(schema1, ExportSchema.last(index))
            schema2 = ExportSchema(index=index, timestamp=dt + timedelta(seconds=1))
            schema2.save(**save_args)
            self.assert_docs_equal(schema2, ExportSchema.last(index))
            schema3 = ExportSchema(index=index, timestamp=dt - timedelta(seconds=1))
            schema3.save(**save_args)
            # still schema2 (which has a later date than schema3)
            self.assert_docs_equal(schema2, ExportSchema.last(index))
Example #16
0
def create_basic_form_checkpoint(index):
    checkpoint = ExportSchema(seq="0", schema=BASIC_FORM_SCHEMA, timestamp=datetime.utcnow(), index=index)
    checkpoint.save()
    return checkpoint