def test_uses_get_multi(monkeypatch): def breakdown(*a, **kw): raise AssertionError('Expected use of get_multi') get_multi_calls = [] old_get = MemoryStorage.get def get_multi(self, hrefs): hrefs = list(hrefs) get_multi_calls.append(hrefs) for href in hrefs: item, etag = old_get(self, href) yield href, item, etag monkeypatch.setattr(MemoryStorage, 'get', breakdown) monkeypatch.setattr(MemoryStorage, 'get_multi', get_multi) a = MemoryStorage() b = MemoryStorage() item = Item(u'UID:1') expected_href, etag = a.upload(item) sync(a, b, {}) assert get_multi_calls == [[expected_href]]
def test_rollback(error_callback): a = MemoryStorage() b = MemoryStorage() status = {} a.items['0'] = ('', Item('UID:0')) b.items['1'] = ('', Item('UID:1')) b.upload = b.update = b.delete = action_failure if error_callback: errors = [] sync(a, b, status=status, conflict_resolution='a wins', error_callback=errors.append) assert len(errors) == 1 assert isinstance(errors[0], ActionIntentionallyFailed) assert len(status) == 1 assert status['1'] else: with pytest.raises(ActionIntentionallyFailed): sync(a, b, status=status, conflict_resolution='a wins')
def test_already_synced(): a = MemoryStorage(fileext='.a') b = MemoryStorage(fileext='.b') item = Item(u'UID:1') a.upload(item) b.upload(item) status = { '1': ({ 'href': '1.a', 'hash': item.hash, 'etag': a.get('1.a')[1] }, { 'href': '1.b', 'hash': item.hash, 'etag': b.get('1.b')[1] }) } old_status = deepcopy(status) a.update = b.update = a.upload = b.upload = \ lambda *a, **kw: pytest.fail('Method shouldn\'t have been called.') for _ in (1, 2): sync(a, b, status) assert status == old_status assert items(a) == items(b) == {item.raw}
def sync(self, status, a, b, force_delete, conflict_resolution): old_items_a = self._get_items(a) old_items_b = self._get_items(b) try: # If one storage is read-only, double-sync because changes don't # get reverted immediately. for _ in range(2 if a.read_only or b.read_only else 1): sync(a, b, status, force_delete=force_delete, conflict_resolution=conflict_resolution) except BothReadOnly: assert a.read_only and b.read_only assume(False) except StorageEmpty: if force_delete: raise else: assert not list(a.list()) or not list(b.list()) return status items_a = self._get_items(a) items_b = self._get_items(b) assert items_a == items_b assert items_a == old_items_a or not a.read_only assert items_b == old_items_b or not b.read_only return status
def sync_collection(config_a, config_b, pair_name, collection, pair_options, general, force_delete): status_name = '_'.join(filter(bool, (pair_name, collection))) pair_description = ' from '.join(filter(bool, (collection, pair_name))) a = storage_instance_from_config(config_a, pair_description) b = storage_instance_from_config(config_b, pair_description) cli_logger.info('Syncing {}'.format(pair_description)) status = load_status(general['status_path'], status_name) try: sync(a, b, status, conflict_resolution=pair_options.get('conflict_resolution', None), force_delete=status_name in force_delete) except exceptions.StorageEmpty as e: side = 'a' if e.empty_storage is a else 'b' storage = e.empty_storage cli_logger.error( '{pair_description}: Storage "{side}" ({storage}) ' 'was completely emptied. Use "--force-delete ' '{status_name}" to synchronize that emptyness to ' 'the other side, or delete the status by yourself to ' 'restore the empty side from the other one.'.format(**locals())) sys.exit(1) save_status(general['status_path'], status_name, status)
def test_irrelevant_status(): a = MemoryStorage() b = MemoryStorage() status = {'1': ('1', 1234, '1.ics', 2345)} sync(a, b, status) assert not status assert empty_storage(a) assert empty_storage(b)
def test_no_uids(): a = MemoryStorage() b = MemoryStorage() a.upload(Item(u'ASDF')) b.upload(Item(u'FOOBAR')) status = {} sync(a, b, status) assert items(a) == items(b) == {u'ASDF', u'FOOBAR'}
def test_both_readonly(): a = MemoryStorage(read_only=True) b = MemoryStorage(read_only=True) assert a.read_only assert b.read_only status = {} with pytest.raises(BothReadOnly): sync(a, b, status)
def test_irrelevant_status(): a = MemoryStorage() b = MemoryStorage() status = {"1": ("1", 1234, "1.ics", 2345)} sync(a, b, status) assert not status assert empty_storage(a) assert empty_storage(b)
def test_irrelevant_status(): a = MemoryStorage() b = MemoryStorage() status = {'1': ('1.txt', 1234, '1.ics', 2345)} sync(a, b, status) assert not status assert empty_storage(a) assert empty_storage(b)
def test_conflict_resolution_new_etags_without_changes(): a = MemoryStorage() b = MemoryStorage() item = Item(u'UID:1') href_a, etag_a = a.upload(item) href_b, etag_b = b.upload(item) status = {'1': (href_a, 'BOGUS_a', href_b, 'BOGUS_b')} sync(a, b, status) assert status == {'1': (href_a, etag_a, href_b, etag_b)}
def test_conflict_resolution_invalid_mode(): a = MemoryStorage() b = MemoryStorage() item_a = Item(u'UID:1\nitem a') item_b = Item(u'UID:1\nitem b') a.upload(item_a) b.upload(item_b) with pytest.raises(ValueError): sync(a, b, {}, conflict_resolution='yolo')
def test_readonly(): a = MemoryStorage() b = MemoryStorage(read_only=True) status = {} href_a, _ = a.upload(Item(u'UID:1')) href_b, _ = b.upload(Item(u'UID:2')) sync(a, b, status) assert len(status) == 2 and a.has(href_a) and not b.has(href_a) sync(a, b, status) assert len(status) == 1 and not a.has(href_a) and not b.has(href_a)
def test_missing_status(): a = MemoryStorage() b = MemoryStorage() status = {} item = Item(u'asdf') a.upload(item) b.upload(item) sync(a, b, status) assert len(status) == 1 assert items(a) == items(b) == {item.raw}
def test_changed_uids(): a = MemoryStorage() b = MemoryStorage() href_a, etag_a = a.upload(Item(u'UID:A-ONE')) href_b, etag_b = b.upload(Item(u'UID:B-ONE')) status = {} sync(a, b, status) a.update(href_a, Item(u'UID:A-TWO'), etag_a) sync(a, b, status)
def test_partial_sync_error(): a = MemoryStorage() b = MemoryStorage() status = {} a.upload(Item('UID:0')) b.read_only = True with pytest.raises(PartialSync): sync(a, b, status, partial_sync='error')
def test_missing_status(): a = MemoryStorage() b = MemoryStorage() status = {} item = Item(u'asdf') href_a, _ = a.upload(item) href_b, _ = b.upload(item) sync(a, b, status) assert len(status) == 1 assert a.has(href_a) assert b.has(href_b)
def test_missing_status(): a = MemoryStorage() b = MemoryStorage() status = {} item = Item(u'UID:1') a.upload(item) b.upload(item) sync(a, b, status) assert len(status) == 1 assert a.has('1.txt') assert b.has('1.txt')
def test_no_uids(): a = MemoryStorage() b = MemoryStorage() href_a, _ = a.upload(Item(u'ASDF')) href_b, _ = b.upload(Item(u'FOOBAR')) status = {} sync(a, b, status) a_items = set(a.get(href)[0].raw for href, etag in a.list()) b_items = set(b.get(href)[0].raw for href, etag in b.list()) assert a_items == b_items == {u'ASDF', u'FOOBAR'}
def sync(self, status, a, b, force_delete, conflict_resolution, with_error_callback, partial_sync): assume(a is not b) old_items_a = items(a) old_items_b = items(b) a.instance_name = 'a' b.instance_name = 'b' errors = [] if with_error_callback: error_callback = errors.append else: error_callback = None try: # If one storage is read-only, double-sync because changes don't # get reverted immediately. for _ in range(2 if a.read_only or b.read_only else 1): sync(a, b, status, force_delete=force_delete, conflict_resolution=conflict_resolution, error_callback=error_callback, partial_sync=partial_sync) for e in errors: raise e except PartialSync: assert partial_sync == 'error' except ActionIntentionallyFailed: pass except BothReadOnly: assert a.read_only and b.read_only assume(False) except StorageEmpty: if force_delete: raise else: assert not list(a.list()) or not list(b.list()) else: items_a = items(a) items_b = items(b) assert items_a == items_b or partial_sync == 'ignore' assert items_a == old_items_a or not a.read_only assert items_b == old_items_b or not b.read_only assert set(a.items) | set(b.items) == set(status) or \ partial_sync == 'ignore'
def test_updated_and_deleted(): a = MemoryStorage() b = MemoryStorage() href_a, etag_a = a.upload(Item(u'UID:1')) status = {} sync(a, b, status, force_delete=True) (href_b, etag_b), = b.list() b.delete(href_b, etag_b) a.update(href_a, Item(u'UID:1\nupdated'), etag_a) sync(a, b, status, force_delete=True) assert len(list(a.list())) == len(list(b.list())) == 1
def test_updated_and_deleted(): a = MemoryStorage() b = MemoryStorage() href_a, etag_a = a.upload(Item(u"UID:1")) status = {} sync(a, b, status, force_delete=True) (href_b, etag_b), = b.list() b.delete(href_b, etag_b) a.update(href_a, Item(u"UID:1\nupdated"), etag_a) sync(a, b, status, force_delete=True) assert len(list(a.list())) == len(list(b.list())) == 1
def test_updated_and_deleted(): a = MemoryStorage() b = MemoryStorage() href_a, etag_a = a.upload(Item(u'UID:1')) status = {} sync(a, b, status, force_delete=True) (href_b, etag_b), = b.list() b.delete(href_b, etag_b) updated = Item(u'UID:1\nupdated') a.update(href_a, updated, etag_a) sync(a, b, status, force_delete=True) assert items(a) == items(b) == {updated.raw}
def test_readonly(): a = MemoryStorage(instance_name='a') b = MemoryStorage(instance_name='b') status = {} href_a, _ = a.upload(Item(u'UID:1')) href_b, _ = b.upload(Item(u'UID:2')) b.read_only = True with pytest.raises(exceptions.ReadOnlyError): b.upload(Item(u'UID:3')) sync(a, b, status) assert len(status) == 2 and a.has(href_a) and not b.has(href_a) sync(a, b, status) assert len(status) == 1 and not a.has(href_a) and not b.has(href_a)
def test_readonly(): a = MemoryStorage(instance_name="a") b = MemoryStorage(instance_name="b") status = {} href_a, _ = a.upload(Item(u"UID:1")) href_b, _ = b.upload(Item(u"UID:2")) b.read_only = True with pytest.raises(exceptions.ReadOnlyError): b.upload(Item(u"UID:3")) sync(a, b, status) assert len(status) == 2 and a.has(href_a) and not b.has(href_a) sync(a, b, status) assert len(status) == 1 and not a.has(href_a) and not b.has(href_a)
def test_ident_conflict(sync_inbetween): a = MemoryStorage() b = MemoryStorage() status = {} href_a, etag_a = a.upload(Item(u'UID:aaa')) href_b, etag_b = a.upload(Item(u'UID:bbb')) if sync_inbetween: sync(a, b, status) a.update(href_a, Item(u'UID:xxx'), etag_a) a.update(href_b, Item(u'UID:xxx'), etag_b) with pytest.raises(IdentConflict): sync(a, b, status)
def test_already_synced(): a = MemoryStorage(fileext=".a") b = MemoryStorage(fileext=".b") item = Item(u"UID:1") a.upload(item) b.upload(item) status = {"1": ({"href": "1.a", "etag": a.get("1.a")[1]}, {"href": "1.b", "etag": b.get("1.b")[1]})} old_status = dict(status) a.update = b.update = a.upload = b.upload = lambda *a, **kw: pytest.fail("Method shouldn't have been called.") for i in (1, 2): sync(a, b, status) assert status == old_status assert a.has("1.a") and b.has("1.b")
def test_missing_status_and_different_items(): a = MemoryStorage() b = MemoryStorage() status = {} item1 = Item(u'UID:1\nhaha') item2 = Item(u'UID:1\nhoho') a.upload(item1) b.upload(item2) with pytest.raises(SyncConflict): sync(a, b, status) assert not status sync(a, b, status, conflict_resolution='a wins') assert_item_equals(item1, b.get('1.txt')[0]) assert_item_equals(item1, a.get('1.txt')[0])
def test_missing_status_and_different_items(): a = MemoryStorage() b = MemoryStorage() status = {} item1 = Item(u'UID:1\nhaha') item2 = Item(u'UID:1\nhoho') a.upload(item1) b.upload(item2) with pytest.raises(SyncConflict): sync(a, b, status) assert not status sync(a, b, status, conflict_resolution='a wins') assert items(a) == items(b) == {item1.raw}
def test_readonly(): a = MemoryStorage(instance_name='a') b = MemoryStorage(instance_name='b') status = {} href_a, _ = a.upload(Item(u'UID:1')) href_b, _ = b.upload(Item(u'UID:2')) b.read_only = True with pytest.raises(exceptions.ReadOnlyError): b.upload(Item(u'UID:3')) sync(a, b, status, partial_sync='revert') assert len(status) == 2 and a.has(href_a) and not b.has(href_a) sync(a, b, status, partial_sync='revert') assert len(status) == 1 and not a.has(href_a) and not b.has(href_a)
def test_missing_status_and_different_items(): a = MemoryStorage() b = MemoryStorage() status = {} item1 = Item(u'UID:1\nhaha') item2 = Item(u'UID:1\nhoho') a.upload(item1) b.upload(item2) with pytest.raises(SyncConflict): sync(a, b, status) assert not status sync(a, b, status, conflict_resolution='a wins') assert_item_equals(item1, b.get('1')[0]) assert_item_equals(item1, a.get('1')[0])
def test_already_synced(): a = MemoryStorage() b = MemoryStorage() item = Item(u'UID:1') a.upload(item) b.upload(item) status = {'1': ('1', a.get('1')[1], '1', b.get('1')[1])} old_status = dict(status) a.update = b.update = a.upload = b.upload = \ lambda *a, **kw: pytest.fail('Method shouldn\'t have been called.') for i in (1, 2): sync(a, b, status) assert status == old_status assert a.has('1') and b.has('1')
def test_insert_hash(): a = MemoryStorage() b = MemoryStorage() status = {} item = Item('UID:1') href, etag = a.upload(item) sync(a, b, status) for d in status['1']: del d['hash'] a.update(href, Item('UID:1\nHAHA:YES'), etag) sync(a, b, status) assert 'hash' in status['1'][0] and 'hash' in status['1'][1]
def test_read_only_and_prefetch(): a = MemoryStorage() b = MemoryStorage() b.read_only = True status = {} item1 = Item(u'UID:1\nhaha') item2 = Item(u'UID:2\nhoho') a.upload(item1) a.upload(item2) sync(a, b, status, force_delete=True) sync(a, b, status, force_delete=True) assert not items(a) and not items(b)
def test_read_only_and_prefetch(): a = MemoryStorage() b = MemoryStorage() b.read_only = True status = {} item1 = Item(u"UID:1\nhaha") item2 = Item(u"UID:2\nhoho") a.upload(item1) a.upload(item2) sync(a, b, status, force_delete=True) sync(a, b, status, force_delete=True) assert list(a.list()) == list(b.list()) == []
def test_conflict_resolution_new_etags_without_changes(): a = MemoryStorage() b = MemoryStorage() item = Item(u"UID:1") href_a, etag_a = a.upload(item) href_b, etag_b = b.upload(item) status = {"1": (href_a, "BOGUS_a", href_b, "BOGUS_b")} sync(a, b, status) (ident, (status_a, status_b)), = status.items() assert ident == "1" assert status_a["href"] == href_a assert status_a["etag"] == etag_a assert status_b["href"] == href_b assert status_b["etag"] == etag_b
def test_conflict_resolution_new_etags_without_changes(): a = MemoryStorage() b = MemoryStorage() item = Item(u'UID:1') href_a, etag_a = a.upload(item) href_b, etag_b = b.upload(item) status = {'1': (href_a, 'BOGUS_a', href_b, 'BOGUS_b')} sync(a, b, status) (ident, (status_a, status_b)), = status.items() assert ident == '1' assert status_a['href'] == href_a assert status_a['etag'] == etag_a assert status_b['href'] == href_b assert status_b['etag'] == etag_b
def test_already_synced(): a = MemoryStorage() b = MemoryStorage() item = Item(u'UID:1') a.upload(item) b.upload(item) status = { '1': ('1.txt', a.get('1.txt')[1], '1.txt', b.get('1.txt')[1]) } old_status = dict(status) a.update = b.update = a.upload = b.upload = \ lambda *a, **kw: pytest.fail('Method shouldn\'t have been called.') for i in (1, 2): sync(a, b, status) assert status == old_status assert a.has('1.txt') and b.has('1.txt')
def test_partial_sync_ignore(): a = MemoryStorage() b = MemoryStorage() status = {} item0 = Item('UID:0\nhehe') a.upload(item0) b.upload(item0) b.read_only = True item1 = Item('UID:1\nhaha') a.upload(item1) sync(a, b, status, partial_sync='ignore') sync(a, b, status, partial_sync='ignore') assert items(a) == {item0.raw, item1.raw} assert items(b) == {item0.raw}
def test_moved_href(): ''' Concrete application: ppl_ stores contact aliases in filenames, which means item's hrefs get changed. Vdirsyncer doesn't synchronize this data, but also shouldn't do things like deleting and re-uploading to the server. .. _ppl: http://ppladdressbook.org/ ''' a = MemoryStorage() b = MemoryStorage() status = {} href, etag = a.upload(Item(u'UID:haha')) sync(a, b, status) b.items['lol'] = b.items.pop('haha') a.delete = a.update = a.upload = blow_up sync(a, b, status) assert len(status) == 1 assert len(list(a.list())) == len(list(b.list())) == 1 assert status['haha'][2] == 'haha'
def test_moved_href(): ''' Concrete application: ppl_ stores contact aliases in filenames, which means item's hrefs get changed. Vdirsyncer doesn't synchronize this data, but also shouldn't do things like deleting and re-uploading to the server. .. _ppl: http://ppladdressbook.org/ ''' a = MemoryStorage() b = MemoryStorage() status = {} href, etag = a.upload(Item(u'UID:haha')) sync(a, b, status) b.items['lol'] = b.items.pop('haha') # The sync algorithm should prefetch `lol`, see that it's the same ident # and not do anything else. a.get_multi = blow_up # Absolutely no prefetch on A # No actual sync actions a.delete = a.update = a.upload = b.delete = b.update = b.upload = blow_up sync(a, b, status) assert len(status) == 1 assert items(a) == items(b) == {'UID:haha'} assert status['haha'][1]['href'] == 'lol' old_status = deepcopy(status) # Further sync should be a noop. Not even prefetching should occur. b.get_multi = blow_up sync(a, b, status) assert old_status == status assert items(a) == items(b) == {'UID:haha'}