def sync():
_sync = Sync({
"index": "testdb",
"nodes": {
"table": "book"
},
})
yield _sync
_sync.logical_slot_get_changes(
f"{_sync.database}_testdb",
upto_nchanges=None,
)
_sync.engine.connect().close()
_sync.engine.dispose()
def sync():
_sync = Sync({
'index': 'testdb',
'nodes': {
'table': 'book'
},
})
yield _sync
_sync.logical_slot_get_changes(
f'{_sync.database}_testdb',
upto_nchanges=None,
)
_sync.engine.connect().close()
_sync.engine.dispose()
def test_update_primary_key_non_concurrent(self, data, book_cls,
publisher_cls, engine):
"""
Test sync updates primary_key and then sync in non-concurrent mode.
"""
document = {
'index':
'testdb',
'nodes': [{
'table':
'book',
'columns': ['isbn', 'title'],
'children': [{
'table': 'publisher',
'columns': ['id', 'name'],
'relationship': {
'variant': 'object',
'type': 'one_to_one'
}
}]
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'publisher': {
'id': [1]
}
},
u'isbn': u'abc',
u'publisher': {
'id': 1,
'name': 'Tiger publishing'
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'publisher': {
'id': [2]
}
},
u'isbn': u'def',
u'publisher': {
'id': 2,
'name': 'Lion publishing'
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'publisher': {
'id': [3]
}
},
u'isbn': u'ghi',
u'publisher': {
'id': 3,
'name': 'Hop Bunny publishing'
},
u'title': u'The Rabbit Club'
}]
session = sync.session
try:
session.execute(publisher_cls.__table__.insert().values(
id=99, name='Rabbit publishers'))
session.execute(book_cls.__table__.update().where(
book_cls.__table__.c.publisher_id == 3).values(
publisher_id=99))
session.commit()
except Exception:
session.rollback()
raise
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'publisher': {
'id': [1]
}
},
u'isbn': u'abc',
u'publisher': {
'id': 1,
'name': 'Tiger publishing'
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'publisher': {
'id': [2]
}
},
u'isbn': u'def',
u'publisher': {
'id': 2,
'name': 'Lion publishing'
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'publisher': {
'id': [99]
}
},
u'isbn': u'ghi',
u'publisher': {
'id': 99,
'name': 'Rabbit publishers'
},
u'title': u'The Rabbit Club'
}]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_delete_concurrent(self, data, book_cls, publisher_cls):
"""Test sync delete and then sync in concurrent mode."""
document = {
'index':
'testdb',
'nodes': [{
'table':
'book',
'columns': ['isbn', 'title'],
'children': [{
'table': 'publisher',
'columns': ['id', 'name'],
'relationship': {
'variant': 'object',
'type': 'one_to_one'
}
}]
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'publisher': {
'id': [1]
}
},
u'isbn': u'abc',
u'publisher': {
'id': 1,
'name': 'Tiger publishing'
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'publisher': {
'id': [2]
}
},
u'isbn': u'def',
u'publisher': {
'id': 2,
'name': 'Lion publishing'
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'publisher': {
'id': [3]
}
},
u'isbn': u'ghi',
u'publisher': {
'id': 3,
'name': 'Hop Bunny publishing'
},
u'title': u'The Rabbit Club'
}]
session = sync.session
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
try:
session.execute(publisher_cls.__table__.insert().values(
id=99, name='Rabbit publishers'))
session.execute(book_cls.__table__.update().where(
book_cls.__table__.c.publisher_id == 3).values(
publisher_id=99))
session.execute(publisher_cls.__table__.delete().where(
publisher_cls.__table__.c.id == 3))
session.commit()
except Exception:
session.rollback()
raise
with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis):
with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db):
with mock.patch('pgsync.sync.Sync.pull', side_effect=pull):
with mock.patch(
'pgsync.sync.Sync.truncate_slots',
side_effect=truncate_slots,
):
sync.receive()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'publisher': {
'id': [1]
}
},
u'isbn': u'abc',
u'publisher': {
'id': 1,
'name': 'Tiger publishing'
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'publisher': {
'id': [2]
}
},
u'isbn': u'def',
u'publisher': {
'id': 2,
'name': 'Lion publishing'
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'publisher': {
'id': [99]
}
},
u'isbn': u'ghi',
u'publisher': {
'id': 99,
'name': 'Rabbit publishers'
},
u'title': u'The Rabbit Club'
}]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_insert_non_concurrent(self, data, book_cls):
"""Test sync insert and then sync in non-concurrent mode."""
document = {
'index': 'testdb',
'nodes': {
'table': 'book',
'columns': ['isbn', 'title']
}
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
session = sync.session
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
with subtransactions(session):
session.execute(
book_cls.__table__.insert().values(
isbn='xyz',
title='Encyclopedia'
)
)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
},
{
u'_meta': {},
u'isbn': u'xyz',
u'title': u'Encyclopedia'
}
]
assert_resync_empty(sync, document.get('node', {}))
def test_insert_non_concurrent(self, data, book_cls, publisher_cls):
"""Test sync insert and then sync in non-concurrent mode."""
document = {
"index": "testdb",
"nodes": {
"table":
"book",
"columns": ["isbn", "title"],
"children": [{
"table": "publisher",
"columns": ["id", "name"],
"relationship": {
"variant": "object",
"type": "one_to_one",
},
}],
},
}
sync = Sync(document)
sync.sync()
sync.es.refresh("testdb")
session = sync.session
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"publisher": {
"id": [1]
}
},
"isbn": "abc",
"publisher": {
"id": 1,
"name": "Tiger publishing"
},
"title": "The Tiger Club",
},
{
"_meta": {
"publisher": {
"id": [2]
}
},
"isbn": "def",
"publisher": {
"id": 2,
"name": "Lion publishing"
},
"title": "The Lion Club",
},
{
"_meta": {
"publisher": {
"id": [3]
}
},
"isbn": "ghi",
"publisher": {
"id": 3,
"name": "Hop Bunny publishing"
},
"title": "The Rabbit Club",
},
]
try:
session.execute(publisher_cls.__table__.insert().values(
id=99, name="Rabbit publishers"))
session.execute(book_cls.__table__.insert().values(
isbn="xyz", title="Encyclopedia", publisher_id=99))
session.commit()
except Exception:
session.rollback()
raise
sync.sync()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"publisher": {
"id": [1]
}
},
"isbn": "abc",
"publisher": {
"id": 1,
"name": "Tiger publishing"
},
"title": "The Tiger Club",
},
{
"_meta": {
"publisher": {
"id": [2]
}
},
"isbn": "def",
"publisher": {
"id": 2,
"name": "Lion publishing"
},
"title": "The Lion Club",
},
{
"_meta": {
"publisher": {
"id": [3]
}
},
"isbn": "ghi",
"publisher": {
"id": 3,
"name": "Hop Bunny publishing"
},
"title": "The Rabbit Club",
},
{
"_meta": {
"publisher": {
"id": [99]
}
},
"isbn": "xyz",
"publisher": {
"id": 99,
"name": "Rabbit publishers"
},
"title": "Encyclopedia",
},
]
assert_resync_empty(sync, document.get("node", {}))
def test_update_primary_key_concurrent(self, data, book_cls, rating_cls):
"""Test sync updates primary_key and then sync in concurrent mode."""
document = {
'index':
'testdb',
'nodes': [{
'table':
'book',
'columns': ['isbn', 'title'],
'children': [{
'table': 'rating',
'columns': ['id', 'value'],
'relationship': {
'variant': 'object',
'type': 'one_to_one'
}
}]
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'ghi',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'The Rabbit Club'
}]
session = sync.session
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
try:
session.execute(book_cls.__table__.insert().values(
isbn='xyz',
title='Milli and the Ants',
))
session.execute(rating_cls.__table__.update().where(
rating_cls.__table__.c.book_isbn == 'ghi').values(
book_isbn='xyz'))
session.commit()
except Exception:
session.rollback()
raise
with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis):
with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db):
with mock.patch('pgsync.sync.Sync.pull', side_effect=pull):
with mock.patch(
'pgsync.sync.Sync.truncate_slots',
side_effect=truncate_slots,
):
sync.receive()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
'_meta': {},
'isbn': 'ghi',
'rating': None,
'title': 'The Rabbit Club',
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'xyz',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'Milli and the Ants'
}]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_delete_concurrent(self, data, book_cls):
"""Test sync delete and then sync in concurrent mode."""
document = {
'index': 'testdb',
'nodes': [{
'table': 'book',
'columns': ['isbn', 'title']
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
session = sync.session
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
with subtransactions(session):
session.execute(
book_cls.__table__.delete().where(
book_cls.__table__.c.isbn == 'abc'
)
)
session.commit()
with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis):
with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db):
with mock.patch('pgsync.sync.Sync.pull', side_effect=pull):
with mock.patch(
'pgsync.sync.Sync.truncate_slots',
side_effect=truncate_slots,
):
sync.receive()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_update_primary_key_non_concurrent(self, data, book_cls):
"""
Test sync updates primary_key and then sync in non-concurrent mode.
TODO: Note this test highlights a potential undesired bahaviour
i.e we have a duplicate doc at a point in time.
Note to self. I think this has been fixed. i.e we delete and then
query ibsert
"""
document = {
"index": "testdb",
"nodes": {
"table": "book",
"columns": ["isbn", "title"]
},
}
sync = Sync(document)
sync.es.bulk(sync.index, sync.sync())
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {},
"isbn": "abc",
"title": "The Tiger Club"
},
{
"_meta": {},
"isbn": "def",
"title": "The Lion Club"
},
{
"_meta": {},
"isbn": "ghi",
"title": "The Rabbit Club"
},
]
session = sync.session
with subtransactions(session):
session.execute(book_cls.__table__.update().where(
book_cls.__table__.c.isbn == "abc").values(isbn="cba"))
sync.es.bulk(sync.index, sync.sync())
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {},
"isbn": "abc",
"title": "The Tiger Club"
},
{
"_meta": {},
"isbn": "cba",
"title": "The Tiger Club"
},
{
"_meta": {},
"isbn": "def",
"title": "The Lion Club"
},
{
"_meta": {},
"isbn": "ghi",
"title": "The Rabbit Club"
},
]
assert_resync_empty(sync, document.get("node", {}))
sync.es.close()
def test_update_primary_key_non_concurrent(self, data, book_cls,
rating_cls, engine):
"""
Test sync updates primary_key and then sync in non-concurrent mode.
"""
document = {
"index": "testdb",
"nodes": {
"table":
"book",
"columns": ["isbn", "title"],
"children": [{
"table": "rating",
"columns": ["id", "value"],
"relationship": {
"variant": "object",
"type": "one_to_one",
},
}],
},
}
sync = Sync(document)
sync.sync()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"rating": {
"id": [1]
}
},
"isbn": "abc",
"rating": {
"id": 1,
"value": 1.1
},
"title": "The Tiger Club",
},
{
"_meta": {
"rating": {
"id": [2]
}
},
"isbn": "def",
"rating": {
"id": 2,
"value": 2.2
},
"title": "The Lion Club",
},
{
"_meta": {
"rating": {
"id": [3]
}
},
"isbn": "ghi",
"rating": {
"id": 3,
"value": 3.3
},
"title": "The Rabbit Club",
},
]
session = sync.session
try:
session.execute(book_cls.__table__.insert().values(
isbn="xyz", title="Milli and the Ants"))
session.execute(rating_cls.__table__.update().where(
rating_cls.__table__.c.book_isbn == "ghi").values(
book_isbn="xyz"))
session.commit()
except Exception:
session.rollback()
raise
sync.sync()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"rating": {
"id": [1]
}
},
"isbn": "abc",
"rating": {
"id": 1,
"value": 1.1
},
"title": "The Tiger Club",
},
{
"_meta": {
"rating": {
"id": [2]
}
},
"isbn": "def",
"rating": {
"id": 2,
"value": 2.2
},
"title": "The Lion Club",
},
{
"_meta": {},
"isbn": "ghi",
"rating": None,
"title": "The Rabbit Club",
},
{
"_meta": {
"rating": {
"id": [3]
}
},
"isbn": "xyz",
"rating": {
"id": 3,
"value": 3.3
},
"title": "Milli and the Ants",
},
]
assert_resync_empty(sync, document.get("node", {}))
def test_delete_concurrent(self, data, book_cls, rating_cls):
"""Test sync delete and then sync in concurrent mode."""
document = {
"index": "testdb",
"nodes": {
"table":
"book",
"columns": ["isbn", "title"],
"children": [{
"table": "rating",
"columns": ["id", "value"],
"relationship": {
"variant": "object",
"type": "one_to_one",
},
}],
},
}
sync = Sync(document)
sync.sync()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"rating": {
"id": [1]
}
},
"isbn": "abc",
"rating": {
"id": 1,
"value": 1.1
},
"title": "The Tiger Club",
},
{
"_meta": {
"rating": {
"id": [2]
}
},
"isbn": "def",
"rating": {
"id": 2,
"value": 2.2
},
"title": "The Lion Club",
},
{
"_meta": {
"rating": {
"id": [3]
}
},
"isbn": "ghi",
"rating": {
"id": 3,
"value": 3.3
},
"title": "The Rabbit Club",
},
]
session = sync.session
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
try:
session.execute(book_cls.__table__.insert().values(
isbn="xyz",
title="The End of time",
))
session.execute(rating_cls.__table__.update().where(
rating_cls.__table__.c.id == 3).values(book_isbn="xyz"))
session.execute(book_cls.__table__.delete().where(
book_cls.__table__.c.isbn == "ghi"))
session.commit()
except Exception:
session.rollback()
raise
with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis):
with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db):
with mock.patch("pgsync.sync.Sync.pull", side_effect=pull):
with mock.patch(
"pgsync.sync.Sync.truncate_slots",
side_effect=truncate_slots,
):
sync.receive()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"rating": {
"id": [1]
}
},
"isbn": "abc",
"rating": {
"id": 1,
"value": 1.1
},
"title": "The Tiger Club",
},
{
"_meta": {
"rating": {
"id": [2]
}
},
"isbn": "def",
"rating": {
"id": 2,
"value": 2.2
},
"title": "The Lion Club",
},
{
"_meta": {
"rating": {
"id": [3]
}
},
"isbn": "xyz",
"rating": {
"id": 3,
"value": 3.3
},
"title": "The End of time",
},
]
assert_resync_empty(sync, document.get("node", {}))
def test_update_non_primary_key_concurrent(self, data, book_cls,
rating_cls):
"""Test sync update and then sync in concurrent mode."""
document = {
'index': 'testdb',
'nodes': {
'table':
'book',
'columns': ['isbn', 'title'],
'children': [{
'table': 'rating',
'columns': ['id', 'value'],
'relationship': {
'variant': 'object',
'type': 'one_to_one'
}
}]
}
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'ghi',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'The Rabbit Club'
}]
session = sync.session
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
with subtransactions(session):
session.execute(rating_cls.__table__.update().where(
rating_cls.__table__.c.id == 3).values(value=4.4))
session.commit()
with mock.patch('pgsync.sync.Sync.poll_redis', side_effect=poll_redis):
with mock.patch('pgsync.sync.Sync.poll_db', side_effect=poll_db):
with mock.patch('pgsync.sync.Sync.pull', side_effect=pull):
with mock.patch(
'pgsync.sync.Sync.truncate_slots',
side_effect=truncate_slots,
):
sync.receive()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'ghi',
u'rating': {
'id': 3,
'value': 4.4
},
u'title': u'The Rabbit Club'
}]
assert_resync_empty(sync, document.get('node', {}))
def test_update_non_primary_key_concurrent(self, data, book_cls,
rating_cls):
"""Test sync update and then sync in concurrent mode."""
document = {
"index": "testdb",
"nodes": {
"table":
"book",
"columns": ["isbn", "title"],
"children": [{
"table": "rating",
"columns": ["id", "value"],
"relationship": {
"variant": "object",
"type": "one_to_one",
},
}],
},
}
sync = Sync(document)
sync.es.bulk(sync.index, sync.sync())
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"rating": {
"id": [1]
}
},
"isbn": "abc",
"rating": {
"id": 1,
"value": 1.1
},
"title": "The Tiger Club",
},
{
"_meta": {
"rating": {
"id": [2]
}
},
"isbn": "def",
"rating": {
"id": 2,
"value": 2.2
},
"title": "The Lion Club",
},
{
"_meta": {
"rating": {
"id": [3]
}
},
"isbn": "ghi",
"rating": {
"id": 3,
"value": 3.3
},
"title": "The Rabbit Club",
},
]
session = sync.session
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
with subtransactions(session):
session.execute(rating_cls.__table__.update().where(
rating_cls.__table__.c.id == 3).values(value=4.4))
session.commit()
with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis):
with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db):
with mock.patch("pgsync.sync.Sync.pull", side_effect=pull):
with mock.patch(
"pgsync.sync.Sync.truncate_slots",
side_effect=noop,
):
with mock.patch(
"pgsync.sync.Sync.status",
side_effect=noop,
):
sync.receive()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"rating": {
"id": [1]
}
},
"isbn": "abc",
"rating": {
"id": 1,
"value": 1.1
},
"title": "The Tiger Club",
},
{
"_meta": {
"rating": {
"id": [2]
}
},
"isbn": "def",
"rating": {
"id": 2,
"value": 2.2
},
"title": "The Lion Club",
},
{
"_meta": {
"rating": {
"id": [3]
}
},
"isbn": "ghi",
"rating": {
"id": 3,
"value": 4.4
},
"title": "The Rabbit Club",
},
]
assert_resync_empty(sync, document.get("node", {}))
sync.es.close()
def test_update_primary_key_non_concurrent(self, data, book_cls):
"""
Test sync updates primary_key and then sync in non-concurrent mode.
TODO: Note this test highlights a potential undesired bahaviour
i.e we have a duplicate doc at a point in time.
Note to self. I think this has been fixed. i.e we delete and then
query ibsert
"""
document = {
'index': 'testdb',
'nodes': [{
'table': 'book',
'columns': ['isbn', 'title']
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
session = sync.session
with subtransactions(session):
session.execute(
book_cls.__table__.update().where(
book_cls.__table__.c.isbn == 'abc'
).values(isbn='cba')
)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'cba',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_update_non_concurrent(self, data, book_cls):
"""Test sync update and then sync in non-concurrent mode."""
document = {
"index": "testdb",
"nodes": {
"table": "book",
"columns": ["isbn", "title"]
},
}
sync = Sync(document)
sync.es.bulk(sync.index, sync.sync())
sync.es.refresh("testdb")
session = sync.session
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {},
"isbn": "abc",
"title": "The Tiger Club"
},
{
"_meta": {},
"isbn": "def",
"title": "The Lion Club"
},
{
"_meta": {},
"isbn": "ghi",
"title": "The Rabbit Club"
},
]
with subtransactions(session):
session.execute(book_cls.__table__.update().where(
book_cls.__table__.c.isbn == "abc").values(title="Tiger Club"))
sync.es.bulk(sync.index, sync.sync())
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {},
"isbn": "abc",
"title": "Tiger Club"
},
{
"_meta": {},
"isbn": "def",
"title": "The Lion Club"
},
{
"_meta": {},
"isbn": "ghi",
"title": "The Rabbit Club"
},
]
assert_resync_empty(sync, document.get("node", {}))
sync.es.close()
def test_update_non_concurrent(self, data, book_cls):
"""Test sync update and then sync in non-concurrent mode."""
document = {
'index': 'testdb',
'nodes': [{
'table': 'book',
'columns': ['isbn', 'title']
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
session = sync.session
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'The Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
with subtransactions(session):
session.execute(
book_cls.__table__.update().where(
book_cls.__table__.c.isbn == 'abc'
).values(title='Tiger Club')
)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [
{
u'_meta': {},
u'isbn': u'abc',
u'title': u'Tiger Club'
},
{
u'_meta': {},
u'isbn': u'def',
u'title': u'The Lion Club'
},
{
u'_meta': {},
u'isbn': u'ghi',
u'title': u'The Rabbit Club'
}
]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_delete_concurrent(self, data, book_cls):
"""Test sync delete and then sync in concurrent mode."""
document = {
"index": "testdb",
"nodes": {
"table": "book",
"columns": ["isbn", "title"]
},
}
sync = Sync(document)
sync.es.bulk(sync.index, sync.sync())
sync.es.refresh("testdb")
session = sync.session
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {},
"isbn": "abc",
"title": "The Tiger Club"
},
{
"_meta": {},
"isbn": "def",
"title": "The Lion Club"
},
{
"_meta": {},
"isbn": "ghi",
"title": "The Rabbit Club"
},
]
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
with subtransactions(session):
session.execute(book_cls.__table__.delete().where(
book_cls.__table__.c.isbn == "abc"))
session.commit()
with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis):
with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db):
with mock.patch("pgsync.sync.Sync.pull", side_effect=pull):
with mock.patch(
"pgsync.sync.Sync.truncate_slots",
side_effect=noop,
):
with mock.patch(
"pgsync.sync.Sync.status",
side_effect=noop,
):
sync.receive()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {},
"isbn": "def",
"title": "The Lion Club"
},
{
"_meta": {},
"isbn": "ghi",
"title": "The Rabbit Club"
},
]
assert_resync_empty(sync, document.get("node", {}))
sync.es.close()
def test_update_primary_key_non_concurrent(self, data, book_cls,
rating_cls, engine):
"""
Test sync updates primary_key and then sync in non-concurrent mode.
"""
document = {
'index':
'testdb',
'nodes': [{
'table':
'book',
'columns': ['isbn', 'title'],
'children': [{
'table': 'rating',
'columns': ['id', 'value'],
'relationship': {
'variant': 'object',
'type': 'one_to_one'
}
}]
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'ghi',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'The Rabbit Club'
}]
session = sync.session
try:
session.execute(book_cls.__table__.insert().values(
isbn='xyz', title='Milli and the Ants'))
session.execute(rating_cls.__table__.update().where(
rating_cls.__table__.c.book_isbn == 'ghi').values(
book_isbn='xyz'))
session.commit()
except Exception:
session.rollback()
raise
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club',
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club',
}, {
'_meta': {},
'isbn': 'ghi',
'rating': None,
'title': 'The Rabbit Club',
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'xyz',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'Milli and the Ants',
}]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_update_primary_key_concurrent(self, data, book_cls,
publisher_cls):
"""Test sync updates primary_key and then sync in concurrent mode."""
document = {
"index": "testdb",
"nodes": {
"table":
"book",
"columns": ["isbn", "title"],
"children": [{
"table": "publisher",
"columns": ["id", "name"],
"relationship": {
"variant": "object",
"type": "one_to_one",
},
}],
},
}
sync = Sync(document)
sync.sync()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"publisher": {
"id": [1]
}
},
"isbn": "abc",
"publisher": {
"id": 1,
"name": "Tiger publishing"
},
"title": "The Tiger Club",
},
{
"_meta": {
"publisher": {
"id": [2]
}
},
"isbn": "def",
"publisher": {
"id": 2,
"name": "Lion publishing"
},
"title": "The Lion Club",
},
{
"_meta": {
"publisher": {
"id": [3]
}
},
"isbn": "ghi",
"publisher": {
"id": 3,
"name": "Hop Bunny publishing"
},
"title": "The Rabbit Club",
},
]
session = sync.session
def pull():
txmin = sync.checkpoint
txmax = sync.txid_current
sync.logical_slot_changes(txmin=txmin, txmax=txmax)
def poll_redis():
return []
def poll_db():
try:
session.execute(publisher_cls.__table__.insert().values(
id=99, name="Rabbit publishers"))
session.execute(book_cls.__table__.update().where(
book_cls.__table__.c.publisher_id == 3).values(
publisher_id=99))
session.commit()
except Exception:
session.rollback()
raise
with mock.patch("pgsync.sync.Sync.poll_redis", side_effect=poll_redis):
with mock.patch("pgsync.sync.Sync.poll_db", side_effect=poll_db):
with mock.patch("pgsync.sync.Sync.pull", side_effect=pull):
with mock.patch(
"pgsync.sync.Sync.truncate_slots",
side_effect=truncate_slots,
):
sync.receive()
sync.es.refresh("testdb")
docs = search(sync.es, "testdb")
assert docs == [
{
"_meta": {
"publisher": {
"id": [1]
}
},
"isbn": "abc",
"publisher": {
"id": 1,
"name": "Tiger publishing"
},
"title": "The Tiger Club",
},
{
"_meta": {
"publisher": {
"id": [2]
}
},
"isbn": "def",
"publisher": {
"id": 2,
"name": "Lion publishing"
},
"title": "The Lion Club",
},
{
"_meta": {
"publisher": {
"id": [99]
}
},
"isbn": "ghi",
"publisher": {
"id": 99,
"name": "Rabbit publishers"
},
"title": "The Rabbit Club",
},
]
assert_resync_empty(sync, document.get("node", {}))
def test_insert_non_concurrent(self, data, book_cls, rating_cls):
"""Test sync insert and then sync in non-concurrent mode."""
document = {
'index':
'testdb',
'nodes': [{
'table':
'book',
'columns': ['isbn', 'title'],
'children': [{
'table': 'rating',
'columns': ['id', 'value'],
'relationship': {
'variant': 'object',
'type': 'one_to_one'
}
}]
}]
}
sync = Sync(document)
sync.sync()
sync.es.refresh('testdb')
session = sync.session
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'ghi',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'The Rabbit Club'
}]
try:
session.execute(book_cls.__table__.insert().values(
isbn='xyz',
title='Encyclopedia',
))
session.execute(rating_cls.__table__.insert().values(
id=99,
book_isbn='xyz',
value=4.4,
))
session.commit()
except Exception:
session.rollback()
raise
sync.sync()
sync.es.refresh('testdb')
docs = search(sync.es, 'testdb')
assert docs == [{
u'_meta': {
'rating': {
'id': [1]
}
},
u'isbn': u'abc',
u'rating': {
'id': 1,
'value': 1.1
},
u'title': u'The Tiger Club'
}, {
u'_meta': {
'rating': {
'id': [2]
}
},
u'isbn': u'def',
u'rating': {
'id': 2,
'value': 2.2
},
u'title': u'The Lion Club'
}, {
u'_meta': {
'rating': {
'id': [3]
}
},
u'isbn': u'ghi',
u'rating': {
'id': 3,
'value': 3.3
},
u'title': u'The Rabbit Club'
}, {
u'_meta': {
'rating': {
'id': [99]
}
},
u'isbn': u'xyz',
u'rating': {
'id': 99,
'value': 4.4
},
u'title': u'Encyclopedia'
}]
assert_resync_empty(
sync,
document.get('nodes', []),
document.get('index'),
)
def test_sync_validate(self, mock_es):
with pytest.raises(SchemaError) as excinfo:
sync = Sync(
document={
"index": "testdb",
"nodes": ["foo"],
},
verbose=False,
validate=True,
repl_slots=False,
)
assert "Incompatible schema. Please run v2 schema migration" in str(
excinfo.value
)
sync = Sync(
document={
"index": "testdb",
"nodes": {"table": "book"},
"plugins": ["Hero"],
},
verbose=False,
validate=True,
repl_slots=False,
)
def _side_effect(*args, **kwargs):
if args[0] == 0:
return 0
elif args[0] == "max_replication_slots":
raise RuntimeError(
"Ensure there is at least one replication slot defined "
"by setting max_replication_slots=1"
)
elif args[0] == "wal_level":
raise RuntimeError(
"Enable logical decoding by setting wal_level=logical"
)
elif args[0] == "rds_logical_replication":
raise RDSError("rds.logical_replication is not enabled")
else:
return args[0]
with pytest.raises(RuntimeError) as excinfo:
with patch(
"pgsync.base.Base.pg_settings",
side_effects=_side_effect("max_replication_slots"),
):
sync = Sync(
document={
"index": "testdb",
"nodes": {"table": "book"},
"plugins": ["Hero"],
},
)
assert (
"Ensure there is at least one replication slot defined "
"by setting max_replication_slots=1" in str(excinfo.value)
)
with pytest.raises(RuntimeError) as excinfo:
with patch(
"pgsync.base.Base.pg_settings",
side_effects=_side_effect("wal_level"),
):
sync = Sync(
document={
"index": "testdb",
"nodes": {"table": "book"},
"plugins": ["Hero"],
},
)
assert "Enable logical decoding by setting wal_level=logical" in str(
excinfo.value
)
with pytest.raises(RDSError) as excinfo:
with patch(
"pgsync.base.Base.pg_settings",
side_effects=_side_effect("rds_logical_replication"),
):
sync = Sync(
document={
"index": "testdb",
"nodes": {"table": "book"},
"plugins": ["Hero"],
},
)
assert "rds.logical_replication is not enabled" in str(excinfo.value)
