def test_update_primary_key_concurrent(self, data, book_cls, rating_cls): """Test sync updates primary_key and then sync in concurrent mode.""" document = { 'index': 'testdb', 'nodes': [{ 'table': 'book', 'columns': ['isbn', 'title'], 'children': [{ 'table': 'rating', 'columns': ['id', 'value'], 'relationship': { 'variant': 'object', 'type': 'one_to_one' } }] }] } sync = Sync(document) sync.sync() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [{ u'_meta': { 'rating': { 'id': [1] } }, u'isbn': u'abc', u'rating': { 'id': 1, 'value': 1.1 }, u'title': u'The Tiger Club' }, { u'_meta': { 'rating': { 'id': [2] } }, u'isbn': u'def', u'rating': { 'id': 2, 'value': 2.2 }, u'title': u'The Lion Club' }, { u'_meta': { 'rating': { 'id': [3] } }, u'isbn': u'ghi', u'rating': { 'id': 3, 'value': 3.3 }, u'title': u'The Rabbit Club' }] session = sync.session def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): try: session.execute(book_cls.__table__.insert().values( isbn='xyz', title='Milli and the Ants', )) session.execute(rating_cls.__table__.update().where( rating_cls.__table__.c.book_isbn == 'ghi').values( book_isbn='xyz')) session.commit() except Exception: session.rollback() raise with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis): with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db): with mock.patch('pgsync.sync.Sync.pull', side_effect=pull): with mock.patch( 'pgsync.sync.Sync.truncate_slots', side_effect=truncate_slots, ): sync.receive() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [{ u'_meta': { 'rating': { 'id': [1] } }, u'isbn': u'abc', u'rating': { 'id': 1, 'value': 1.1 }, u'title': u'The Tiger Club' }, { u'_meta': { 'rating': { 'id': [2] } }, u'isbn': u'def', u'rating': { 'id': 2, 'value': 2.2 }, u'title': u'The Lion Club' }, { '_meta': {}, 'isbn': 'ghi', 'rating': None, 'title': 'The Rabbit Club', }, { u'_meta': { 'rating': { 'id': [3] } }, u'isbn': u'xyz', u'rating': { 'id': 3, 'value': 3.3 }, u'title': u'Milli and the Ants' }] assert_resync_empty( sync, document.get('nodes', []), document.get('index'), )
def test_delete_concurrent(self, data, book_cls, publisher_cls): """Test sync delete and then sync in concurrent mode.""" document = { 'index': 'testdb', 'nodes': [{ 'table': 'book', 'columns': ['isbn', 'title'], 'children': [{ 'table': 'publisher', 'columns': ['id', 'name'], 'relationship': { 'variant': 'object', 'type': 'one_to_one' } }] }] } sync = Sync(document) sync.sync() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [{ u'_meta': { 'publisher': { 'id': [1] } }, u'isbn': u'abc', u'publisher': { 'id': 1, 'name': 'Tiger publishing' }, u'title': u'The Tiger Club' }, { u'_meta': { 'publisher': { 'id': [2] } }, u'isbn': u'def', u'publisher': { 'id': 2, 'name': 'Lion publishing' }, u'title': u'The Lion Club' }, { u'_meta': { 'publisher': { 'id': [3] } }, u'isbn': u'ghi', u'publisher': { 'id': 3, 'name': 'Hop Bunny publishing' }, u'title': u'The Rabbit Club' }] session = sync.session def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): try: session.execute(publisher_cls.__table__.insert().values( id=99, name='Rabbit publishers')) session.execute(book_cls.__table__.update().where( book_cls.__table__.c.publisher_id == 3).values( publisher_id=99)) session.execute(publisher_cls.__table__.delete().where( publisher_cls.__table__.c.id == 3)) session.commit() except Exception: session.rollback() raise with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis): with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db): with mock.patch('pgsync.sync.Sync.pull', side_effect=pull): with mock.patch( 'pgsync.sync.Sync.truncate_slots', side_effect=truncate_slots, ): sync.receive() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [{ u'_meta': { 'publisher': { 'id': [1] } }, u'isbn': u'abc', u'publisher': { 'id': 1, 'name': 'Tiger publishing' }, u'title': u'The Tiger Club' }, { u'_meta': { 'publisher': { 'id': [2] } }, u'isbn': u'def', u'publisher': { 'id': 2, 'name': 'Lion publishing' }, u'title': u'The Lion Club' }, { u'_meta': { 'publisher': { 'id': [99] } }, u'isbn': u'ghi', u'publisher': { 'id': 99, 'name': 'Rabbit publishers' }, u'title': u'The Rabbit Club' }] assert_resync_empty( sync, document.get('nodes', []), document.get('index'), )
def test_delete_concurrent(self, data, book_cls): """Test sync delete and then sync in concurrent mode.""" document = { 'index': 'testdb', 'nodes': [{ 'table': 'book', 'columns': ['isbn', 'title'] }] } sync = Sync(document) sync.sync() sync.es.refresh('testdb') session = sync.session docs = search(sync.es, 'testdb') assert docs == [ { u'_meta': {}, u'isbn': u'abc', u'title': u'The Tiger Club' }, { u'_meta': {}, u'isbn': u'def', u'title': u'The Lion Club' }, { u'_meta': {}, u'isbn': u'ghi', u'title': u'The Rabbit Club' } ] def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): with subtransactions(session): session.execute( book_cls.__table__.delete().where( book_cls.__table__.c.isbn == 'abc' ) ) session.commit() with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis): with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db): with mock.patch('pgsync.sync.Sync.pull', side_effect=pull): with mock.patch( 'pgsync.sync.Sync.truncate_slots', side_effect=truncate_slots, ): sync.receive() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [ { u'_meta': {}, u'isbn': u'def', u'title': u'The Lion Club' }, { u'_meta': {}, u'isbn': u'ghi', u'title': u'The Rabbit Club' } ] assert_resync_empty( sync, document.get('nodes', []), document.get('index'), )
def test_update_primary_key_concurrent(self, data, book_cls, publisher_cls): """Test sync updates primary_key and then sync in concurrent mode.""" document = { "index": "testdb", "nodes": { "table": "book", "columns": ["isbn", "title"], "children": [{ "table": "publisher", "columns": ["id", "name"], "relationship": { "variant": "object", "type": "one_to_one", }, }], }, } sync = Sync(document) sync.sync() sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": { "publisher": { "id": [1] } }, "isbn": "abc", "publisher": { "id": 1, "name": "Tiger publishing" }, "title": "The Tiger Club", }, { "_meta": { "publisher": { "id": [2] } }, "isbn": "def", "publisher": { "id": 2, "name": "Lion publishing" }, "title": "The Lion Club", }, { "_meta": { "publisher": { "id": [3] } }, "isbn": "ghi", "publisher": { "id": 3, "name": "Hop Bunny publishing" }, "title": "The Rabbit Club", }, ] session = sync.session def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): try: session.execute(publisher_cls.__table__.insert().values( id=99, name="Rabbit publishers")) session.execute(book_cls.__table__.update().where( book_cls.__table__.c.publisher_id == 3).values( publisher_id=99)) session.commit() except Exception: session.rollback() raise with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis): with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db): with mock.patch("pgsync.sync.Sync.pull", side_effect=pull): with mock.patch( "pgsync.sync.Sync.truncate_slots", side_effect=truncate_slots, ): sync.receive() sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": { "publisher": { "id": [1] } }, "isbn": "abc", "publisher": { "id": 1, "name": "Tiger publishing" }, "title": "The Tiger Club", }, { "_meta": { "publisher": { "id": [2] } }, "isbn": "def", "publisher": { "id": 2, "name": "Lion publishing" }, "title": "The Lion Club", }, { "_meta": { "publisher": { "id": [99] } }, "isbn": "ghi", "publisher": { "id": 99, "name": "Rabbit publishers" }, "title": "The Rabbit Club", }, ] assert_resync_empty(sync, document.get("node", {}))
def test_delete_concurrent(self, data, book_cls): """Test sync delete and then sync in concurrent mode.""" document = { "index": "testdb", "nodes": { "table": "book", "columns": ["isbn", "title"] }, } sync = Sync(document) sync.es.bulk(sync.index, sync.sync()) sync.es.refresh("testdb") session = sync.session docs = search(sync.es, "testdb") assert docs == [ { "_meta": {}, "isbn": "abc", "title": "The Tiger Club" }, { "_meta": {}, "isbn": "def", "title": "The Lion Club" }, { "_meta": {}, "isbn": "ghi", "title": "The Rabbit Club" }, ] def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): with subtransactions(session): session.execute(book_cls.__table__.delete().where( book_cls.__table__.c.isbn == "abc")) session.commit() with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis): with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db): with mock.patch("pgsync.sync.Sync.pull", side_effect=pull): with mock.patch( "pgsync.sync.Sync.truncate_slots", side_effect=noop, ): with mock.patch( "pgsync.sync.Sync.status", side_effect=noop, ): sync.receive() sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": {}, "isbn": "def", "title": "The Lion Club" }, { "_meta": {}, "isbn": "ghi", "title": "The Rabbit Club" }, ] assert_resync_empty(sync, document.get("node", {})) sync.es.close()
def test_delete_concurrent(self, data, book_cls, rating_cls): """Test sync delete and then sync in concurrent mode.""" document = { "index": "testdb", "nodes": { "table": "book", "columns": ["isbn", "title"], "children": [{ "table": "rating", "columns": ["id", "value"], "relationship": { "variant": "object", "type": "one_to_one", }, }], }, } sync = Sync(document) sync.sync() sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": { "rating": { "id": [1] } }, "isbn": "abc", "rating": { "id": 1, "value": 1.1 }, "title": "The Tiger Club", }, { "_meta": { "rating": { "id": [2] } }, "isbn": "def", "rating": { "id": 2, "value": 2.2 }, "title": "The Lion Club", }, { "_meta": { "rating": { "id": [3] } }, "isbn": "ghi", "rating": { "id": 3, "value": 3.3 }, "title": "The Rabbit Club", }, ] session = sync.session def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): try: session.execute(book_cls.__table__.insert().values( isbn="xyz", title="The End of time", )) session.execute(rating_cls.__table__.update().where( rating_cls.__table__.c.id == 3).values(book_isbn="xyz")) session.execute(book_cls.__table__.delete().where( book_cls.__table__.c.isbn == "ghi")) session.commit() except Exception: session.rollback() raise with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis): with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db): with mock.patch("pgsync.sync.Sync.pull", side_effect=pull): with mock.patch( "pgsync.sync.Sync.truncate_slots", side_effect=truncate_slots, ): sync.receive() sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": { "rating": { "id": [1] } }, "isbn": "abc", "rating": { "id": 1, "value": 1.1 }, "title": "The Tiger Club", }, { "_meta": { "rating": { "id": [2] } }, "isbn": "def", "rating": { "id": 2, "value": 2.2 }, "title": "The Lion Club", }, { "_meta": { "rating": { "id": [3] } }, "isbn": "xyz", "rating": { "id": 3, "value": 3.3 }, "title": "The End of time", }, ] assert_resync_empty(sync, document.get("node", {}))
def test_update_non_primary_key_concurrent(self, data, book_cls, rating_cls): """Test sync update and then sync in concurrent mode.""" document = { 'index': 'testdb', 'nodes': { 'table': 'book', 'columns': ['isbn', 'title'], 'children': [{ 'table': 'rating', 'columns': ['id', 'value'], 'relationship': { 'variant': 'object', 'type': 'one_to_one' } }] } } sync = Sync(document) sync.sync() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [{ u'_meta': { 'rating': { 'id': [1] } }, u'isbn': u'abc', u'rating': { 'id': 1, 'value': 1.1 }, u'title': u'The Tiger Club' }, { u'_meta': { 'rating': { 'id': [2] } }, u'isbn': u'def', u'rating': { 'id': 2, 'value': 2.2 }, u'title': u'The Lion Club' }, { u'_meta': { 'rating': { 'id': [3] } }, u'isbn': u'ghi', u'rating': { 'id': 3, 'value': 3.3 }, u'title': u'The Rabbit Club' }] session = sync.session def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): with subtransactions(session): session.execute(rating_cls.__table__.update().where( rating_cls.__table__.c.id == 3).values(value=4.4)) session.commit() with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis): with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db): with mock.patch('pgsync.sync.Sync.pull', side_effect=pull): with mock.patch( 'pgsync.sync.Sync.truncate_slots', side_effect=truncate_slots, ): sync.receive() sync.es.refresh('testdb') docs = search(sync.es, 'testdb') assert docs == [{ u'_meta': { 'rating': { 'id': [1] } }, u'isbn': u'abc', u'rating': { 'id': 1, 'value': 1.1 }, u'title': u'The Tiger Club' }, { u'_meta': { 'rating': { 'id': [2] } }, u'isbn': u'def', u'rating': { 'id': 2, 'value': 2.2 }, u'title': u'The Lion Club' }, { u'_meta': { 'rating': { 'id': [3] } }, u'isbn': u'ghi', u'rating': { 'id': 3, 'value': 4.4 }, u'title': u'The Rabbit Club' }] assert_resync_empty(sync, document.get('node', {}))
def test_update_non_primary_key_concurrent(self, data, book_cls, rating_cls): """Test sync update and then sync in concurrent mode.""" document = { "index": "testdb", "nodes": { "table": "book", "columns": ["isbn", "title"], "children": [{ "table": "rating", "columns": ["id", "value"], "relationship": { "variant": "object", "type": "one_to_one", }, }], }, } sync = Sync(document) sync.es.bulk(sync.index, sync.sync()) sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": { "rating": { "id": [1] } }, "isbn": "abc", "rating": { "id": 1, "value": 1.1 }, "title": "The Tiger Club", }, { "_meta": { "rating": { "id": [2] } }, "isbn": "def", "rating": { "id": 2, "value": 2.2 }, "title": "The Lion Club", }, { "_meta": { "rating": { "id": [3] } }, "isbn": "ghi", "rating": { "id": 3, "value": 3.3 }, "title": "The Rabbit Club", }, ] session = sync.session def pull(): txmin = sync.checkpoint txmax = sync.txid_current sync.logical_slot_changes(txmin=txmin, txmax=txmax) def poll_redis(): return [] def poll_db(): with subtransactions(session): session.execute(rating_cls.__table__.update().where( rating_cls.__table__.c.id == 3).values(value=4.4)) session.commit() with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis): with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db): with mock.patch("pgsync.sync.Sync.pull", side_effect=pull): with mock.patch( "pgsync.sync.Sync.truncate_slots", side_effect=noop, ): with mock.patch( "pgsync.sync.Sync.status", side_effect=noop, ): sync.receive() sync.es.refresh("testdb") docs = search(sync.es, "testdb") assert docs == [ { "_meta": { "rating": { "id": [1] } }, "isbn": "abc", "rating": { "id": 1, "value": 1.1 }, "title": "The Tiger Club", }, { "_meta": { "rating": { "id": [2] } }, "isbn": "def", "rating": { "id": 2, "value": 2.2 }, "title": "The Lion Club", }, { "_meta": { "rating": { "id": [3] } }, "isbn": "ghi", "rating": { "id": 3, "value": 4.4 }, "title": "The Rabbit Club", }, ] assert_resync_empty(sync, document.get("node", {})) sync.es.close()