def test_bogus_etag_change(): '''Assert that sync algorithm is resilient against etag changes if content didn\'t change. In this particular case we test a scenario where both etags have been updated, but only one side actually changed its item content. ''' a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('ASDASD') href_a, etag_a = a.upload(item) sync(a, b, status) assert len(status) == 1 assert items(a) == items(b) == {item.raw} new_item = format_item('ASDASD') (href_b, etag_b), = b.list() a.update(href_a, item, etag_a) b.update(href_b, new_item, etag_b) b.delete = b.update = b.upload = blow_up sync(a, b, status) assert len(status) == 1 assert items(a) == items(b) == {new_item.raw}
def test_partial_sync_revert(): a = MemoryStorage(instance_name='a') b = MemoryStorage(instance_name='b') status = {} item1 = format_item('1') item2 = format_item('2') a.upload(item1) b.upload(item2) b.read_only = True sync(a, b, status, partial_sync='revert') assert len(status) == 2 assert items(a) == {item1.raw, item2.raw} assert items(b) == {item2.raw} sync(a, b, status, partial_sync='revert') assert len(status) == 1 assert items(a) == {item2.raw} assert items(b) == {item2.raw} # Check that updates get reverted item2_up = format_item('2') a.items[next(iter(a.items))] = ('foo', item2_up) assert items(a) == {item2_up.raw} sync(a, b, status, partial_sync='revert') assert len(status) == 1 assert items(a) == {item2_up.raw} sync(a, b, status, partial_sync='revert') assert items(a) == {item2.raw} # Check that deletions get reverted a.items.clear() sync(a, b, status, partial_sync='revert', force_delete=True) sync(a, b, status, partial_sync='revert', force_delete=True) assert items(a) == {item2.raw}
def test_partial_sync(tmpdir, runner, partial_sync): runner.write_with_general(dedent(''' [pair foobar] a = "foo" b = "bar" collections = null {partial_sync} [storage foo] type = "filesystem" fileext = ".txt" path = "{base}/foo" [storage bar] type = "filesystem" read_only = true fileext = ".txt" path = "{base}/bar" '''.format( partial_sync=('partial_sync = "{}"\n'.format(partial_sync) if partial_sync else ''), base=str(tmpdir) ))) foo = tmpdir.mkdir('foo') bar = tmpdir.mkdir('bar') item = format_item('other') foo.join('other.txt').write(item.raw) bar.join('other.txt').write(item.raw) baritem = bar.join('lol.txt') baritem.write(format_item('lol').raw) r = runner.invoke(['discover']) assert not r.exception r = runner.invoke(['sync']) assert not r.exception fooitem = foo.join('lol.txt') fooitem.remove() r = runner.invoke(['sync']) if partial_sync == 'error': assert r.exception assert 'Attempted change' in r.output elif partial_sync == 'ignore': assert baritem.exists() r = runner.invoke(['sync']) assert not r.exception assert baritem.exists() else: assert baritem.exists() r = runner.invoke(['sync']) assert not r.exception assert baritem.exists() assert fooitem.exists()
def test_conflict_resolution_invalid_mode(): a = MemoryStorage() b = MemoryStorage() item_a = format_item('1') item_b = format_item('1') a.upload(item_a) b.upload(item_b) with pytest.raises(ValueError): sync(a, b, {}, conflict_resolution='yolo')
def test_changed_uids(): a = MemoryStorage() b = MemoryStorage() href_a, etag_a = a.upload(format_item('a1')) href_b, etag_b = b.upload(format_item('b1')) status = {} sync(a, b, status) a.update(href_a, format_item('a2'), etag_a) sync(a, b, status)
def test_no_uids(): a = MemoryStorage() b = MemoryStorage() item_a = format_item('') item_b = format_item('') a.upload(item_a) b.upload(item_b) status = {} sync(a, b, status) assert items(a) == items(b) == {item_a.raw, item_b.raw}
def test_missing_status_and_different_items(): a = MemoryStorage() b = MemoryStorage() status = {} item1 = format_item('1') item2 = format_item('1') a.upload(item1) b.upload(item2) with pytest.raises(SyncConflict): sync(a, b, status) assert not status sync(a, b, status, conflict_resolution='a wins') assert items(a) == items(b) == {item1.raw}
def test_insert_hash(): a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('1') href, etag = a.upload(item) sync(a, b, status) for d in status['1']: del d['hash'] a.update(href, format_item('1'), etag) # new item content sync(a, b, status) assert 'hash' in status['1'][0] and 'hash' in status['1'][1]
def test_updated_and_deleted(): a = MemoryStorage() b = MemoryStorage() item = format_item('1') href_a, etag_a = a.upload(item) status = {} sync(a, b, status, force_delete=True) (href_b, etag_b), = b.list() b.delete(href_b, etag_b) updated = format_item('1') a.update(href_a, updated, etag_a) sync(a, b, status, force_delete=True) assert items(a) == items(b) == {updated.raw}
def test_read_only_and_prefetch(): a = MemoryStorage() b = MemoryStorage() b.read_only = True status = {} item1 = format_item('1') item2 = format_item('2') a.upload(item1) a.upload(item2) sync(a, b, status, force_delete=True) sync(a, b, status, force_delete=True) assert not items(a) and not items(b)
def test_already_synced(): a = MemoryStorage(fileext='.a') b = MemoryStorage(fileext='.b') item = format_item('1') a.upload(item) b.upload(item) status = { '1': ({ 'href': '1.a', 'hash': item.hash, 'etag': a.get('1.a')[1] }, { 'href': '1.b', 'hash': item.hash, 'etag': b.get('1.b')[1] }) } old_status = deepcopy(status) a.update = b.update = a.upload = b.upload = \ lambda *a, **kw: pytest.fail('Method shouldn\'t have been called.') for _ in (1, 2): sync(a, b, status) assert status == old_status assert items(a) == items(b) == {item.raw}
def test_simple_run(tmpdir, runner): runner.write_with_general(dedent(''' [pair my_pair] a = "my_a" b = "my_b" collections = null [storage my_a] type = "filesystem" path = "{0}/path_a/" fileext = ".txt" [storage my_b] type = "filesystem" path = "{0}/path_b/" fileext = ".txt" ''').format(str(tmpdir))) tmpdir.mkdir('path_a') tmpdir.mkdir('path_b') result = runner.invoke(['discover']) assert not result.exception result = runner.invoke(['sync']) assert not result.exception item = format_item('haha') tmpdir.join('path_a/haha.txt').write(item.raw) result = runner.invoke(['sync']) assert 'Copying (uploading) item haha to my_b' in result.output assert tmpdir.join('path_b/haha.txt').read().splitlines() == \ item.raw.splitlines()
def test_unicode_hrefs(): a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('รครครค') href, etag = a.upload(item) sync(a, b, status)
def test_uses_get_multi(monkeypatch): def breakdown(*a, **kw): raise AssertionError('Expected use of get_multi') get_multi_calls = [] old_get = MemoryStorage.get def get_multi(self, hrefs): hrefs = list(hrefs) get_multi_calls.append(hrefs) for href in hrefs: item, etag = old_get(self, href) yield href, item, etag monkeypatch.setattr(MemoryStorage, 'get', breakdown) monkeypatch.setattr(MemoryStorage, 'get_multi', get_multi) a = MemoryStorage() b = MemoryStorage() item = format_item('1') expected_href, etag = a.upload(item) sync(a, b, {}) assert get_multi_calls == [[expected_href]]
def test_moved_href(): ''' Concrete application: ppl_ stores contact aliases in filenames, which means item's hrefs get changed. Vdirsyncer doesn't synchronize this data, but also shouldn't do things like deleting and re-uploading to the server. .. _ppl: http://ppladdressbook.org/ ''' a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('haha') href, etag = a.upload(item) sync(a, b, status) b.items['lol'] = b.items.pop('haha') # The sync algorithm should prefetch `lol`, see that it's the same ident # and not do anything else. a.get_multi = blow_up # Absolutely no prefetch on A # No actual sync actions a.delete = a.update = a.upload = b.delete = b.update = b.upload = blow_up sync(a, b, status) assert len(status) == 1 assert items(a) == items(b) == {item.raw} assert status['haha'][1]['href'] == 'lol' old_status = deepcopy(status) # Further sync should be a noop. Not even prefetching should occur. b.get_multi = blow_up sync(a, b, status) assert old_status == status assert items(a) == items(b) == {item.raw}
def test_ident_conflict(sync_inbetween): a = MemoryStorage() b = MemoryStorage() status = {} item_a = format_item('aaa') item_b = format_item('bbb') href_a, etag_a = a.upload(item_a) href_b, etag_b = a.upload(item_b) if sync_inbetween: sync(a, b, status) item_x = format_item('xxx') a.update(href_a, item_x, etag_a) a.update(href_b, item_x, etag_b) with pytest.raises(IdentConflict): sync(a, b, status)
def test_post_hook_inactive(self, tmpdir, monkeypatch): def check_call_mock(*args, **kwargs): assert False monkeypatch.setattr(subprocess, 'call', check_call_mock) s = self.storage_class(str(tmpdir), '.txt', post_hook=None) s.upload(format_item('a/b/c'))
def test_missing_status(): a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('asdf') a.upload(item) b.upload(item) sync(a, b, status) assert len(status) == 1 assert items(a) == items(b) == {item.raw}
def test_partial_sync_ignore(): a = MemoryStorage() b = MemoryStorage() status = {} item0 = format_item('0') a.upload(item0) b.upload(item0) b.read_only = True item1 = format_item('1') a.upload(item1) sync(a, b, status, partial_sync='ignore') sync(a, b, status, partial_sync='ignore') assert items(a) == {item0.raw, item1.raw} assert items(b) == {item0.raw}
def test_duplicate_hrefs(): a = MemoryStorage() b = MemoryStorage() a.list = lambda: [('a', 'a')] * 3 a.items['a'] = ('a', format_item('a')) status = {} sync(a, b, status) with pytest.raises(AssertionError): sync(a, b, status)
def test_conflict_resolution_both_etags_new(winning_storage): a = MemoryStorage() b = MemoryStorage() item = format_item('1') href_a, etag_a = a.upload(item) href_b, etag_b = b.upload(item) status = {} sync(a, b, status) assert status item_a = format_item('1') item_b = format_item('1') a.update(href_a, item_a, etag_a) b.update(href_b, item_b, etag_b) with pytest.raises(SyncConflict): sync(a, b, status) sync(a, b, status, conflict_resolution='{} wins'.format(winning_storage)) assert items(a) == items(b) == { item_a.raw if winning_storage == 'a' else item_b.raw }
def test_deletion(): a = MemoryStorage(fileext='.a') b = MemoryStorage(fileext='.b') status = {} item = format_item('1') a.upload(item) item2 = format_item('2') a.upload(item2) sync(a, b, status) b.delete('1.b', b.get('1.b')[1]) sync(a, b, status) assert items(a) == items(b) == {item2.raw} a.upload(item) sync(a, b, status) assert items(a) == items(b) == {item.raw, item2.raw} a.delete('1.a', a.get('1.a')[1]) sync(a, b, status) assert items(a) == items(b) == {item2.raw}
def test_partial_sync_error(): a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('0') a.upload(item) b.read_only = True with pytest.raises(PartialSync): sync(a, b, status, partial_sync='error')
def test_empty_storage_dataloss(): a = MemoryStorage() b = MemoryStorage() for i in '12': a.upload(format_item(i)) status = {} sync(a, b, status) with pytest.raises(StorageEmpty): sync(MemoryStorage(), b, status) with pytest.raises(StorageEmpty): sync(a, MemoryStorage(), status)
def test_conflict_resolution(tmpdir, runner): item_a = format_item('lol') item_b = format_item('lol') runner.write_with_general(dedent(''' [pair foobar] a = "foo" b = "bar" collections = null conflict_resolution = ["command", "cp"] [storage foo] type = "filesystem" fileext = ".txt" path = "{base}/foo" [storage bar] type = "filesystem" fileext = ".txt" path = "{base}/bar" '''.format(base=str(tmpdir)))) foo = tmpdir.join('foo') bar = tmpdir.join('bar') fooitem = foo.join('lol.txt').ensure() fooitem.write(item_a.raw) baritem = bar.join('lol.txt').ensure() baritem.write(item_b.raw) r = runner.invoke(['discover']) assert not r.exception r = runner.invoke(['sync']) assert not r.exception assert fooitem.read().splitlines() == item_a.raw.splitlines() assert baritem.read().splitlines() == item_a.raw.splitlines()
def test_rollback(error_callback): a = MemoryStorage() b = MemoryStorage() status = {} a.items['0'] = ('', format_item('0')) b.items['1'] = ('', format_item('1')) b.upload = b.update = b.delete = action_failure if error_callback: errors = [] sync(a, b, status=status, conflict_resolution='a wins', error_callback=errors.append) assert len(errors) == 1 assert isinstance(errors[0], ActionIntentionallyFailed) assert len(status) == 1 assert status['1'] else: with pytest.raises(ActionIntentionallyFailed): sync(a, b, status=status, conflict_resolution='a wins')
def test_partial_sync_ignore2(): a = MemoryStorage() b = MemoryStorage() status = {} item = format_item('0') href, etag = a.upload(item) a.read_only = True sync(a, b, status, partial_sync='ignore', force_delete=True) assert items(b) == items(a) == {item.raw} b.items.clear() sync(a, b, status, partial_sync='ignore', force_delete=True) sync(a, b, status, partial_sync='ignore', force_delete=True) assert items(a) == {item.raw} assert not b.items a.read_only = False new_item = format_item('0') a.update(href, new_item, etag) a.read_only = True sync(a, b, status, partial_sync='ignore', force_delete=True) assert items(b) == items(a) == {new_item.raw}
def test_upload_and_update(): a = MemoryStorage(fileext='.a') b = MemoryStorage(fileext='.b') status = {} item = format_item('1') # new item 1 in a a.upload(item) sync(a, b, status) assert items(b) == items(a) == {item.raw} item = format_item('1') # update of item 1 in b b.update('1.b', item, b.get('1.b')[1]) sync(a, b, status) assert items(b) == items(a) == {item.raw} item2 = format_item('2') # new item 2 in b b.upload(item2) sync(a, b, status) assert items(b) == items(a) == {item.raw, item2.raw} item2 = format_item('2') # update of item 2 in a a.update('2.a', item2, a.get('2.a')[1]) sync(a, b, status) assert items(b) == items(a) == {item.raw, item2.raw}
def test_post_hook_active(self, tmpdir, monkeypatch): calls = [] exe = 'foo' def check_call_mock(l, *args, **kwargs): calls.append(True) assert len(l) == 2 assert l[0] == exe monkeypatch.setattr(subprocess, 'call', check_call_mock) s = self.storage_class(str(tmpdir), '.txt', post_hook=exe) s.upload(format_item('a/b/c')) assert calls
def test_conflict_resolution_new_etags_without_changes(): a = MemoryStorage() b = MemoryStorage() item = format_item('1') href_a, etag_a = a.upload(item) href_b, etag_b = b.upload(item) status = {'1': (href_a, 'BOGUS_a', href_b, 'BOGUS_b')} sync(a, b, status) (ident, (status_a, status_b)), = status.items() assert ident == '1' assert status_a['href'] == href_a assert status_a['etag'] == etag_a assert status_b['href'] == href_b assert status_b['etag'] == etag_b
def test_empty_storage(tmpdir, runner): runner.write_with_general(dedent(''' [pair my_pair] a = "my_a" b = "my_b" collections = null [storage my_a] type = "filesystem" path = "{0}/path_a/" fileext = ".txt" [storage my_b] type = "filesystem" path = "{0}/path_b/" fileext = ".txt" ''').format(str(tmpdir))) tmpdir.mkdir('path_a') tmpdir.mkdir('path_b') result = runner.invoke(['discover']) assert not result.exception result = runner.invoke(['sync']) assert not result.exception item = format_item('haha') tmpdir.join('path_a/haha.txt').write(item.raw) result = runner.invoke(['sync']) assert not result.exception tmpdir.join('path_b/haha.txt').remove() result = runner.invoke(['sync']) lines = result.output.splitlines() assert lines[0] == 'Syncing my_pair' assert lines[1].startswith('error: my_pair: ' 'Storage "my_b" was completely emptied.') assert result.exception
def test_ident_conflict(tmpdir, runner): runner.write_with_general(dedent(''' [pair foobar] a = "foo" b = "bar" collections = null [storage foo] type = "filesystem" path = "{base}/foo/" fileext = ".txt" [storage bar] type = "filesystem" path = "{base}/bar/" fileext = ".txt" '''.format(base=str(tmpdir)))) foo = tmpdir.mkdir('foo') tmpdir.mkdir('bar') item = format_item('1') foo.join('one.txt').write(item.raw) foo.join('two.txt').write(item.raw) foo.join('three.txt').write(item.raw) result = runner.invoke(['discover']) assert not result.exception result = runner.invoke(['sync']) assert result.exception assert ('error: foobar: Storage "foo" contains multiple items with the ' 'same UID or even content') in result.output assert sorted([ 'one.txt' in result.output, 'two.txt' in result.output, 'three.txt' in result.output, ]) == [False, True, True]
def test_post_hook_active(self, tmpdir): s = self.storage_class(str(tmpdir), '.txt', post_hook='rm') s.upload(format_item('a/b/c')) assert not list(s.list())
def test_too_long_uid(self, tmpdir): s = self.storage_class(str(tmpdir), '.txt') item = format_item('hue' * 600) href, etag = s.upload(item) assert item.uid not in href
def test_ident_with_slash(self, tmpdir): s = self.storage_class(str(tmpdir), '.txt') s.upload(format_item('a/b/c')) item_file, = tmpdir.listdir() assert '/' not in item_file.basename and item_file.isfile()
def test_collections_cache_invalidation(tmpdir, runner): foo = tmpdir.mkdir('foo') bar = tmpdir.mkdir('bar') for x in 'abc': foo.mkdir(x) bar.mkdir(x) runner.write_with_general(dedent(''' [storage foo] type = "filesystem" path = "{0}/foo/" fileext = ".txt" [storage bar] type = "filesystem" path = "{0}/bar/" fileext = ".txt" [pair foobar] a = "foo" b = "bar" collections = ["a", "b", "c"] ''').format(str(tmpdir))) foo.join('a/itemone.txt').write(format_item('itemone').raw) result = runner.invoke(['discover']) assert not result.exception result = runner.invoke(['sync']) assert not result.exception assert 'detected change in config file' not in result.output.lower() rv = bar.join('a').listdir() assert len(rv) == 1 assert rv[0].basename == 'itemone.txt' runner.write_with_general(dedent(''' [storage foo] type = "filesystem" path = "{0}/foo/" fileext = ".txt" [storage bar] type = "filesystem" path = "{0}/bar2/" fileext = ".txt" [pair foobar] a = "foo" b = "bar" collections = ["a", "b", "c"] ''').format(str(tmpdir))) for entry in tmpdir.join('status').listdir(): if not str(entry).endswith('.collections'): entry.remove() bar2 = tmpdir.mkdir('bar2') for x in 'abc': bar2.mkdir(x) result = runner.invoke(['sync']) assert 'detected change in config file' in result.output.lower() assert result.exception result = runner.invoke(['discover']) assert not result.exception result = runner.invoke(['sync']) assert not result.exception rv = bar.join('a').listdir() rv2 = bar2.join('a').listdir() assert len(rv) == len(rv2) == 1 assert rv[0].basename == rv2[0].basename == 'itemone.txt'