def test_bogus_etag_change():
'''Assert that sync algorithm is resilient against etag changes if content
didn\'t change.
In this particular case we test a scenario where both etags have been
updated, but only one side actually changed its item content.
'''
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('ASDASD')
href_a, etag_a = a.upload(item)
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {item.raw}
new_item = format_item('ASDASD')
(href_b, etag_b), = b.list()
a.update(href_a, item, etag_a)
b.update(href_b, new_item, etag_b)
b.delete = b.update = b.upload = blow_up
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {new_item.raw}
def test_changed_uids():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(Item('UID:A-ONE'))
href_b, etag_b = b.upload(Item('UID:B-ONE'))
status = {}
sync(a, b, status)
a.update(href_a, Item('UID:A-TWO'), etag_a)
sync(a, b, status)
def test_changed_uids():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(Item(u'UID:A-ONE'))
href_b, etag_b = b.upload(Item(u'UID:B-ONE'))
status = {}
sync(a, b, status)
a.update(href_a, Item(u'UID:A-TWO'), etag_a)
sync(a, b, status)
def test_changed_uids():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(format_item('a1'))
href_b, etag_b = b.upload(format_item('b1'))
status = {}
sync(a, b, status)
a.update(href_a, format_item('a2'), etag_a)
sync(a, b, status)
def test_updated_and_deleted():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(Item(u'UID:1'))
status = {}
sync(a, b, status, force_delete=True)
(href_b, etag_b), = b.list()
b.delete(href_b, etag_b)
a.update(href_a, Item(u'UID:1\nupdated'), etag_a)
sync(a, b, status, force_delete=True)
assert len(list(a.list())) == len(list(b.list())) == 1
def test_updated_and_deleted():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(Item(u"UID:1"))
status = {}
sync(a, b, status, force_delete=True)
(href_b, etag_b), = b.list()
b.delete(href_b, etag_b)
a.update(href_a, Item(u"UID:1\nupdated"), etag_a)
sync(a, b, status, force_delete=True)
assert len(list(a.list())) == len(list(b.list())) == 1
def test_ident_conflict(sync_inbetween):
a = MemoryStorage()
b = MemoryStorage()
status = {}
href_a, etag_a = a.upload(Item('UID:aaa'))
href_b, etag_b = a.upload(Item('UID:bbb'))
if sync_inbetween:
sync(a, b, status)
a.update(href_a, Item('UID:xxx'), etag_a)
a.update(href_b, Item('UID:xxx'), etag_b)
with pytest.raises(IdentConflict):
sync(a, b, status)
def test_updated_and_deleted():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(Item(u'UID:1'))
status = {}
sync(a, b, status, force_delete=True)
(href_b, etag_b), = b.list()
b.delete(href_b, etag_b)
updated = Item(u'UID:1\nupdated')
a.update(href_a, updated, etag_a)
sync(a, b, status, force_delete=True)
assert items(a) == items(b) == {updated.raw}
def test_ident_conflict(sync_inbetween):
a = MemoryStorage()
b = MemoryStorage()
status = {}
href_a, etag_a = a.upload(Item(u'UID:aaa'))
href_b, etag_b = a.upload(Item(u'UID:bbb'))
if sync_inbetween:
sync(a, b, status)
a.update(href_a, Item(u'UID:xxx'), etag_a)
a.update(href_b, Item(u'UID:xxx'), etag_b)
with pytest.raises(IdentConflict):
sync(a, b, status)
def test_updated_and_deleted():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(Item('UID:1'))
status = {}
sync(a, b, status, force_delete=True)
(href_b, etag_b), = b.list()
b.delete(href_b, etag_b)
updated = Item('UID:1\nupdated')
a.update(href_a, updated, etag_a)
sync(a, b, status, force_delete=True)
assert items(a) == items(b) == {updated.raw}
def test_insert_hash():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('1')
href, etag = a.upload(item)
sync(a, b, status)
for d in status['1']:
del d['hash']
a.update(href, format_item('1'), etag) # new item content
sync(a, b, status)
assert 'hash' in status['1'][0] and 'hash' in status['1'][1]
def test_insert_hash():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = Item("UID:1")
href, etag = a.upload(item)
sync(a, b, status)
for d in status["1"]:
del d["hash"]
a.update(href, Item("UID:1\nHAHA:YES"), etag)
sync(a, b, status)
assert "hash" in status["1"][0] and "hash" in status["1"][1]
def test_insert_hash():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = Item('UID:1')
href, etag = a.upload(item)
sync(a, b, status)
for d in status['1']:
del d['hash']
a.update(href, Item('UID:1\nHAHA:YES'), etag)
sync(a, b, status)
assert 'hash' in status['1'][0] and 'hash' in status['1'][1]
def test_insert_hash():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = Item('UID:1')
href, etag = a.upload(item)
sync(a, b, status)
for d in status['1']:
del d['hash']
a.update(href, Item('UID:1\nHAHA:YES'), etag)
sync(a, b, status)
assert 'hash' in status['1'][0] and 'hash' in status['1'][1]
def test_ident_conflict(sync_inbetween):
a = MemoryStorage()
b = MemoryStorage()
status = {}
item_a = format_item('aaa')
item_b = format_item('bbb')
href_a, etag_a = a.upload(item_a)
href_b, etag_b = a.upload(item_b)
if sync_inbetween:
sync(a, b, status)
item_x = format_item('xxx')
a.update(href_a, item_x, etag_a)
a.update(href_b, item_x, etag_b)
with pytest.raises(IdentConflict):
sync(a, b, status)
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = Item("UID:1")
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
item_a = Item("UID:1\nitem a")
item_b = Item("UID:1\nitem b")
a.update(href_a, item_a, etag_a)
b.update(href_b, item_b, etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution=f"{winning_storage} wins")
assert (items(a) == items(b) ==
{item_a.raw if winning_storage == "a" else item_b.raw})
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = Item('UID:1')
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
item_a = Item('UID:1\nitem a')
item_b = Item('UID:1\nitem b')
a.update(href_a, item_a, etag_a)
b.update(href_b, item_b, etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution='{} wins'.format(winning_storage))
assert items(a) == items(b) == {
item_a.raw if winning_storage == 'a' else item_b.raw
}
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = Item(u'UID:1')
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
item_a = Item(u'UID:1\nitem a')
item_b = Item(u'UID:1\nitem b')
a.update(href_a, item_a, etag_a)
b.update(href_b, item_b, etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution='{} wins'.format(winning_storage))
assert items(a) == items(b) == {
item_a.raw if winning_storage == 'a' else item_b.raw
}
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = Item(u'UID:1')
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
a.update(href_a, Item(u'UID:1\nitem a'), etag_a)
b.update(href_b, Item(u'UID:1\nitem b'), etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution='{} wins'.format(winning_storage))
item_a, _ = a.get(href_a)
item_b, _ = b.get(href_b)
assert_item_equals(item_a, item_b)
n = normalize_item(item_a)
assert u'UID:1' in n
assert u'item {}'.format(winning_storage) in n
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = Item(u'UID:1')
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
a.update(href_a, Item(u'UID:1\nitem a'), etag_a)
b.update(href_b, Item(u'UID:1\nitem b'), etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution='{} wins'.format(winning_storage))
item_a, _ = a.get(href_a)
item_b, _ = b.get(href_b)
assert_item_equals(item_a, item_b)
n = normalize_item(item_a)
assert u'UID:1' in n
assert u'item {}'.format(winning_storage) in n
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = Item(u"UID:1")
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
a.update(href_a, Item(u"UID:1\nitem a"), etag_a)
b.update(href_b, Item(u"UID:1\nitem b"), etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution="{} wins".format(winning_storage))
item_a, _ = a.get(href_a)
item_b, _ = b.get(href_b)
assert_item_equals(item_a, item_b)
n = item_a.raw.splitlines()
assert u"UID:1" in n
assert u"item {}".format(winning_storage) in n
def test_partial_sync_ignore2():
a = MemoryStorage()
b = MemoryStorage()
status = {}
href, etag = a.upload(Item('UID:0'))
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {'UID:0'}
b.items.clear()
sync(a, b, status, partial_sync='ignore', force_delete=True)
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(a) == {'UID:0'}
assert not b.items
a.read_only = False
a.update(href, Item('UID:0\nupdated'), etag)
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {'UID:0\nupdated'}
def test_partial_sync_ignore2():
a = MemoryStorage()
b = MemoryStorage()
status = {}
href, etag = a.upload(Item('UID:0'))
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {'UID:0'}
b.items.clear()
sync(a, b, status, partial_sync='ignore', force_delete=True)
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(a) == {'UID:0'}
assert not b.items
a.read_only = False
a.update(href, Item('UID:0\nupdated'), etag)
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {'UID:0\nupdated'}
def test_bogus_etag_change():
'''Assert that sync algorithm is resilient against etag changes if content
didn\'t change.
In this particular case we test a scenario where both etags have been
updated, but only one side actually changed its item content.
'''
a = MemoryStorage()
b = MemoryStorage()
status = {}
href_a, etag_a = a.upload(Item(u'UID:ASDASD'))
sync(a, b, status)
assert len(status) == len(list(a.list())) == len(list(b.list())) == 1
(href_b, etag_b), = b.list()
a.update(href_a, Item(u'UID:ASDASD'), etag_a)
b.update(href_b, Item(u'UID:ASDASD\nACTUALCHANGE:YES'), etag_b)
b.delete = b.update = b.upload = blow_up
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {u'UID:ASDASD\nACTUALCHANGE:YES'}
def test_bogus_etag_change():
'''Assert that sync algorithm is resilient against etag changes if content
didn\'t change.
In this particular case we test a scenario where both etags have been
updated, but only one side actually changed its item content.
'''
a = MemoryStorage()
b = MemoryStorage()
status = {}
href_a, etag_a = a.upload(Item('UID:ASDASD'))
sync(a, b, status)
assert len(status) == len(list(a.list())) == len(list(b.list())) == 1
(href_b, etag_b), = b.list()
a.update(href_a, Item('UID:ASDASD'), etag_a)
b.update(href_b, Item('UID:ASDASD\nACTUALCHANGE:YES'), etag_b)
b.delete = b.update = b.upload = blow_up
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {'UID:ASDASD\nACTUALCHANGE:YES'}
def test_upload_and_update():
a = MemoryStorage(fileext=".a")
b = MemoryStorage(fileext=".b")
status = {}
item = Item(u"UID:1") # new item 1 in a
a.upload(item)
sync(a, b, status)
assert_item_equals(b.get("1.b")[0], item)
item = Item(u"UID:1\nASDF:YES") # update of item 1 in b
b.update("1.b", item, b.get("1.b")[1])
sync(a, b, status)
assert_item_equals(a.get("1.a")[0], item)
item2 = Item(u"UID:2") # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert_item_equals(a.get("2.a")[0], item2)
item2 = Item(u"UID:2\nASDF:YES") # update of item 2 in a
a.update("2.a", item2, a.get("2.a")[1])
sync(a, b, status)
assert_item_equals(b.get("2.b")[0], item2)
def test_partial_sync_ignore2():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('0')
href, etag = a.upload(item)
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {item.raw}
b.items.clear()
sync(a, b, status, partial_sync='ignore', force_delete=True)
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(a) == {item.raw}
assert not b.items
a.read_only = False
new_item = format_item('0')
a.update(href, new_item, etag)
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {new_item.raw}
def test_upload_and_update():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = Item(u'UID:1') # new item 1 in a
a.upload(item)
sync(a, b, status)
assert_item_equals(b.get('1')[0], item)
item = Item(u'UID:1\nASDF:YES') # update of item 1 in b
b.update('1', item, b.get('1')[1])
sync(a, b, status)
assert_item_equals(a.get('1')[0], item)
item2 = Item(u'UID:2') # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert_item_equals(a.get('2')[0], item2)
item2 = Item(u'UID:2\nASDF:YES') # update of item 2 in a
a.update('2', item2, a.get('2')[1])
sync(a, b, status)
assert_item_equals(b.get('2')[0], item2)
def test_upload_and_update():
a = MemoryStorage(fileext='.a')
b = MemoryStorage(fileext='.b')
status = {}
item = Item('UID:1') # new item 1 in a
a.upload(item)
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item = Item('UID:1\nASDF:YES') # update of item 1 in b
b.update('1.b', item, b.get('1.b')[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item2 = Item('UID:2') # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
item2 = Item('UID:2\nASDF:YES') # update of item 2 in a
a.update('2.a', item2, a.get('2.a')[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
def test_upload_and_update():
a = MemoryStorage(fileext=".a")
b = MemoryStorage(fileext=".b")
status = {}
item = Item("UID:1") # new item 1 in a
a.upload(item)
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item = Item("UID:1\nASDF:YES") # update of item 1 in b
b.update("1.b", item, b.get("1.b")[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item2 = Item("UID:2") # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
item2 = Item("UID:2\nASDF:YES") # update of item 2 in a
a.update("2.a", item2, a.get("2.a")[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
def test_upload_and_update():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = Item(u'UID:1') # new item 1 in a
a.upload(item)
sync(a, b, status)
assert_item_equals(b.get('1.txt')[0], item)
item = Item(u'UID:1\nASDF:YES') # update of item 1 in b
b.update('1.txt', item, b.get('1.txt')[1])
sync(a, b, status)
assert_item_equals(a.get('1.txt')[0], item)
item2 = Item(u'UID:2') # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert_item_equals(a.get('2.txt')[0], item2)
item2 = Item(u'UID:2\nASDF:YES') # update of item 2 in a
a.update('2.txt', item2, a.get('2.txt')[1])
sync(a, b, status)
assert_item_equals(b.get('2.txt')[0], item2)
def test_upload_and_update():
a = MemoryStorage(fileext='.a')
b = MemoryStorage(fileext='.b')
status = {}
item = Item(u'UID:1') # new item 1 in a
a.upload(item)
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item = Item(u'UID:1\nASDF:YES') # update of item 1 in b
b.update('1.b', item, b.get('1.b')[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item2 = Item(u'UID:2') # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
item2 = Item(u'UID:2\nASDF:YES') # update of item 2 in a
a.update('2.a', item2, a.get('2.a')[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
def newstorage(self, flaky_etags, null_etag_on_upload):
s = MemoryStorage()
if flaky_etags:
def get(href):
old_etag, item = s.items[href]
etag = _random_string()
s.items[href] = etag, item
return item, etag
s.get = get
if null_etag_on_upload:
_old_upload = s.upload
_old_update = s.update
s.upload = lambda item: (_old_upload(item)[0], 'NULL')
s.update = lambda h, i, e: _old_update(h, i, e) and 'NULL'
return s
def newstorage(self, flaky_etags, null_etag_on_upload):
s = MemoryStorage()
if flaky_etags:
def get(href):
old_etag, item = s.items[href]
etag = _random_string()
s.items[href] = etag, item
return item, etag
s.get = get
if null_etag_on_upload:
_old_upload = s.upload
_old_update = s.update
s.upload = lambda item: (_old_upload(item)[0], 'NULL')
s.update = lambda h, i, e: _old_update(h, i, e) and 'NULL'
return s
