def test_bogus_etag_change():
'''Assert that sync algorithm is resilient against etag changes if content
didn\'t change.
In this particular case we test a scenario where both etags have been
updated, but only one side actually changed its item content.
'''
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('ASDASD')
href_a, etag_a = a.upload(item)
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {item.raw}
new_item = format_item('ASDASD')
(href_b, etag_b), = b.list()
a.update(href_a, item, etag_a)
b.update(href_b, new_item, etag_b)
b.delete = b.update = b.upload = blow_up
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {new_item.raw}
def test_partial_sync_revert():
a = MemoryStorage(instance_name='a')
b = MemoryStorage(instance_name='b')
status = {}
item1 = format_item('1')
item2 = format_item('2')
a.upload(item1)
b.upload(item2)
b.read_only = True
sync(a, b, status, partial_sync='revert')
assert len(status) == 2
assert items(a) == {item1.raw, item2.raw}
assert items(b) == {item2.raw}
sync(a, b, status, partial_sync='revert')
assert len(status) == 1
assert items(a) == {item2.raw}
assert items(b) == {item2.raw}
# Check that updates get reverted
item2_up = format_item('2')
a.items[next(iter(a.items))] = ('foo', item2_up)
assert items(a) == {item2_up.raw}
sync(a, b, status, partial_sync='revert')
assert len(status) == 1
assert items(a) == {item2_up.raw}
sync(a, b, status, partial_sync='revert')
assert items(a) == {item2.raw}
# Check that deletions get reverted
a.items.clear()
sync(a, b, status, partial_sync='revert', force_delete=True)
sync(a, b, status, partial_sync='revert', force_delete=True)
assert items(a) == {item2.raw}
def test_partial_sync(tmpdir, runner, partial_sync):
runner.write_with_general(dedent('''
[pair foobar]
a = "foo"
b = "bar"
collections = null
{partial_sync}
[storage foo]
type = "filesystem"
fileext = ".txt"
path = "{base}/foo"
[storage bar]
type = "filesystem"
read_only = true
fileext = ".txt"
path = "{base}/bar"
'''.format(
partial_sync=('partial_sync = "{}"\n'.format(partial_sync)
if partial_sync else ''),
base=str(tmpdir)
)))
foo = tmpdir.mkdir('foo')
bar = tmpdir.mkdir('bar')
item = format_item('other')
foo.join('other.txt').write(item.raw)
bar.join('other.txt').write(item.raw)
baritem = bar.join('lol.txt')
baritem.write(format_item('lol').raw)
r = runner.invoke(['discover'])
assert not r.exception
r = runner.invoke(['sync'])
assert not r.exception
fooitem = foo.join('lol.txt')
fooitem.remove()
r = runner.invoke(['sync'])
if partial_sync == 'error':
assert r.exception
assert 'Attempted change' in r.output
elif partial_sync == 'ignore':
assert baritem.exists()
r = runner.invoke(['sync'])
assert not r.exception
assert baritem.exists()
else:
assert baritem.exists()
r = runner.invoke(['sync'])
assert not r.exception
assert baritem.exists()
assert fooitem.exists()
def test_conflict_resolution_invalid_mode():
a = MemoryStorage()
b = MemoryStorage()
item_a = format_item('1')
item_b = format_item('1')
a.upload(item_a)
b.upload(item_b)
with pytest.raises(ValueError):
sync(a, b, {}, conflict_resolution='yolo')
def test_changed_uids():
a = MemoryStorage()
b = MemoryStorage()
href_a, etag_a = a.upload(format_item('a1'))
href_b, etag_b = b.upload(format_item('b1'))
status = {}
sync(a, b, status)
a.update(href_a, format_item('a2'), etag_a)
sync(a, b, status)
def test_no_uids():
a = MemoryStorage()
b = MemoryStorage()
item_a = format_item('')
item_b = format_item('')
a.upload(item_a)
b.upload(item_b)
status = {}
sync(a, b, status)
assert items(a) == items(b) == {item_a.raw, item_b.raw}
def test_missing_status_and_different_items():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item1 = format_item('1')
item2 = format_item('1')
a.upload(item1)
b.upload(item2)
with pytest.raises(SyncConflict):
sync(a, b, status)
assert not status
sync(a, b, status, conflict_resolution='a wins')
assert items(a) == items(b) == {item1.raw}
def test_insert_hash():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('1')
href, etag = a.upload(item)
sync(a, b, status)
for d in status['1']:
del d['hash']
a.update(href, format_item('1'), etag) # new item content
sync(a, b, status)
assert 'hash' in status['1'][0] and 'hash' in status['1'][1]
def test_updated_and_deleted():
a = MemoryStorage()
b = MemoryStorage()
item = format_item('1')
href_a, etag_a = a.upload(item)
status = {}
sync(a, b, status, force_delete=True)
(href_b, etag_b), = b.list()
b.delete(href_b, etag_b)
updated = format_item('1')
a.update(href_a, updated, etag_a)
sync(a, b, status, force_delete=True)
assert items(a) == items(b) == {updated.raw}
def test_read_only_and_prefetch():
a = MemoryStorage()
b = MemoryStorage()
b.read_only = True
status = {}
item1 = format_item('1')
item2 = format_item('2')
a.upload(item1)
a.upload(item2)
sync(a, b, status, force_delete=True)
sync(a, b, status, force_delete=True)
assert not items(a) and not items(b)
def test_already_synced():
a = MemoryStorage(fileext='.a')
b = MemoryStorage(fileext='.b')
item = format_item('1')
a.upload(item)
b.upload(item)
status = {
'1': ({
'href': '1.a',
'hash': item.hash,
'etag': a.get('1.a')[1]
}, {
'href': '1.b',
'hash': item.hash,
'etag': b.get('1.b')[1]
})
}
old_status = deepcopy(status)
a.update = b.update = a.upload = b.upload = \
lambda *a, **kw: pytest.fail('Method shouldn\'t have been called.')
for _ in (1, 2):
sync(a, b, status)
assert status == old_status
assert items(a) == items(b) == {item.raw}
def test_simple_run(tmpdir, runner):
runner.write_with_general(dedent('''
[pair my_pair]
a = "my_a"
b = "my_b"
collections = null
[storage my_a]
type = "filesystem"
path = "{0}/path_a/"
fileext = ".txt"
[storage my_b]
type = "filesystem"
path = "{0}/path_b/"
fileext = ".txt"
''').format(str(tmpdir)))
tmpdir.mkdir('path_a')
tmpdir.mkdir('path_b')
result = runner.invoke(['discover'])
assert not result.exception
result = runner.invoke(['sync'])
assert not result.exception
item = format_item('haha')
tmpdir.join('path_a/haha.txt').write(item.raw)
result = runner.invoke(['sync'])
assert 'Copying (uploading) item haha to my_b' in result.output
assert tmpdir.join('path_b/haha.txt').read().splitlines() == \
item.raw.splitlines()
def test_unicode_hrefs():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('äää')
href, etag = a.upload(item)
sync(a, b, status)
def test_uses_get_multi(monkeypatch):
def breakdown(*a, **kw):
raise AssertionError('Expected use of get_multi')
get_multi_calls = []
old_get = MemoryStorage.get
def get_multi(self, hrefs):
hrefs = list(hrefs)
get_multi_calls.append(hrefs)
for href in hrefs:
item, etag = old_get(self, href)
yield href, item, etag
monkeypatch.setattr(MemoryStorage, 'get', breakdown)
monkeypatch.setattr(MemoryStorage, 'get_multi', get_multi)
a = MemoryStorage()
b = MemoryStorage()
item = format_item('1')
expected_href, etag = a.upload(item)
sync(a, b, {})
assert get_multi_calls == [[expected_href]]
def test_moved_href():
'''
Concrete application: ppl_ stores contact aliases in filenames, which means
item's hrefs get changed. Vdirsyncer doesn't synchronize this data, but
also shouldn't do things like deleting and re-uploading to the server.
.. _ppl: http://ppladdressbook.org/
'''
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('haha')
href, etag = a.upload(item)
sync(a, b, status)
b.items['lol'] = b.items.pop('haha')
# The sync algorithm should prefetch `lol`, see that it's the same ident
# and not do anything else.
a.get_multi = blow_up # Absolutely no prefetch on A
# No actual sync actions
a.delete = a.update = a.upload = b.delete = b.update = b.upload = blow_up
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {item.raw}
assert status['haha'][1]['href'] == 'lol'
old_status = deepcopy(status)
# Further sync should be a noop. Not even prefetching should occur.
b.get_multi = blow_up
sync(a, b, status)
assert old_status == status
assert items(a) == items(b) == {item.raw}
def test_ident_conflict(sync_inbetween):
a = MemoryStorage()
b = MemoryStorage()
status = {}
item_a = format_item('aaa')
item_b = format_item('bbb')
href_a, etag_a = a.upload(item_a)
href_b, etag_b = a.upload(item_b)
if sync_inbetween:
sync(a, b, status)
item_x = format_item('xxx')
a.update(href_a, item_x, etag_a)
a.update(href_b, item_x, etag_b)
with pytest.raises(IdentConflict):
sync(a, b, status)
def test_post_hook_inactive(self, tmpdir, monkeypatch):
def check_call_mock(*args, **kwargs):
assert False
monkeypatch.setattr(subprocess, 'call', check_call_mock)
s = self.storage_class(str(tmpdir), '.txt', post_hook=None)
s.upload(format_item('a/b/c'))
def test_missing_status():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('asdf')
a.upload(item)
b.upload(item)
sync(a, b, status)
assert len(status) == 1
assert items(a) == items(b) == {item.raw}
def test_partial_sync_ignore():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item0 = format_item('0')
a.upload(item0)
b.upload(item0)
b.read_only = True
item1 = format_item('1')
a.upload(item1)
sync(a, b, status, partial_sync='ignore')
sync(a, b, status, partial_sync='ignore')
assert items(a) == {item0.raw, item1.raw}
assert items(b) == {item0.raw}
def test_duplicate_hrefs():
a = MemoryStorage()
b = MemoryStorage()
a.list = lambda: [('a', 'a')] * 3
a.items['a'] = ('a', format_item('a'))
status = {}
sync(a, b, status)
with pytest.raises(AssertionError):
sync(a, b, status)
def test_conflict_resolution_both_etags_new(winning_storage):
a = MemoryStorage()
b = MemoryStorage()
item = format_item('1')
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {}
sync(a, b, status)
assert status
item_a = format_item('1')
item_b = format_item('1')
a.update(href_a, item_a, etag_a)
b.update(href_b, item_b, etag_b)
with pytest.raises(SyncConflict):
sync(a, b, status)
sync(a, b, status, conflict_resolution='{} wins'.format(winning_storage))
assert items(a) == items(b) == {
item_a.raw if winning_storage == 'a' else item_b.raw
}
def test_deletion():
a = MemoryStorage(fileext='.a')
b = MemoryStorage(fileext='.b')
status = {}
item = format_item('1')
a.upload(item)
item2 = format_item('2')
a.upload(item2)
sync(a, b, status)
b.delete('1.b', b.get('1.b')[1])
sync(a, b, status)
assert items(a) == items(b) == {item2.raw}
a.upload(item)
sync(a, b, status)
assert items(a) == items(b) == {item.raw, item2.raw}
a.delete('1.a', a.get('1.a')[1])
sync(a, b, status)
assert items(a) == items(b) == {item2.raw}
def test_partial_sync_error():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('0')
a.upload(item)
b.read_only = True
with pytest.raises(PartialSync):
sync(a, b, status, partial_sync='error')
def test_empty_storage_dataloss():
a = MemoryStorage()
b = MemoryStorage()
for i in '12':
a.upload(format_item(i))
status = {}
sync(a, b, status)
with pytest.raises(StorageEmpty):
sync(MemoryStorage(), b, status)
with pytest.raises(StorageEmpty):
sync(a, MemoryStorage(), status)
def test_conflict_resolution(tmpdir, runner):
item_a = format_item('lol')
item_b = format_item('lol')
runner.write_with_general(dedent('''
[pair foobar]
a = "foo"
b = "bar"
collections = null
conflict_resolution = ["command", "cp"]
[storage foo]
type = "filesystem"
fileext = ".txt"
path = "{base}/foo"
[storage bar]
type = "filesystem"
fileext = ".txt"
path = "{base}/bar"
'''.format(base=str(tmpdir))))
foo = tmpdir.join('foo')
bar = tmpdir.join('bar')
fooitem = foo.join('lol.txt').ensure()
fooitem.write(item_a.raw)
baritem = bar.join('lol.txt').ensure()
baritem.write(item_b.raw)
r = runner.invoke(['discover'])
assert not r.exception
r = runner.invoke(['sync'])
assert not r.exception
assert fooitem.read().splitlines() == item_a.raw.splitlines()
assert baritem.read().splitlines() == item_a.raw.splitlines()
def test_rollback(error_callback):
a = MemoryStorage()
b = MemoryStorage()
status = {}
a.items['0'] = ('', format_item('0'))
b.items['1'] = ('', format_item('1'))
b.upload = b.update = b.delete = action_failure
if error_callback:
errors = []
sync(a, b, status=status, conflict_resolution='a wins',
error_callback=errors.append)
assert len(errors) == 1
assert isinstance(errors[0], ActionIntentionallyFailed)
assert len(status) == 1
assert status['1']
else:
with pytest.raises(ActionIntentionallyFailed):
sync(a, b, status=status, conflict_resolution='a wins')
def test_partial_sync_ignore2():
a = MemoryStorage()
b = MemoryStorage()
status = {}
item = format_item('0')
href, etag = a.upload(item)
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {item.raw}
b.items.clear()
sync(a, b, status, partial_sync='ignore', force_delete=True)
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(a) == {item.raw}
assert not b.items
a.read_only = False
new_item = format_item('0')
a.update(href, new_item, etag)
a.read_only = True
sync(a, b, status, partial_sync='ignore', force_delete=True)
assert items(b) == items(a) == {new_item.raw}
def test_upload_and_update():
a = MemoryStorage(fileext='.a')
b = MemoryStorage(fileext='.b')
status = {}
item = format_item('1') # new item 1 in a
a.upload(item)
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item = format_item('1') # update of item 1 in b
b.update('1.b', item, b.get('1.b')[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw}
item2 = format_item('2') # new item 2 in b
b.upload(item2)
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
item2 = format_item('2') # update of item 2 in a
a.update('2.a', item2, a.get('2.a')[1])
sync(a, b, status)
assert items(b) == items(a) == {item.raw, item2.raw}
def test_post_hook_active(self, tmpdir, monkeypatch):
calls = []
exe = 'foo'
def check_call_mock(l, *args, **kwargs):
calls.append(True)
assert len(l) == 2
assert l[0] == exe
monkeypatch.setattr(subprocess, 'call', check_call_mock)
s = self.storage_class(str(tmpdir), '.txt', post_hook=exe)
s.upload(format_item('a/b/c'))
assert calls
def test_conflict_resolution_new_etags_without_changes():
a = MemoryStorage()
b = MemoryStorage()
item = format_item('1')
href_a, etag_a = a.upload(item)
href_b, etag_b = b.upload(item)
status = {'1': (href_a, 'BOGUS_a', href_b, 'BOGUS_b')}
sync(a, b, status)
(ident, (status_a, status_b)), = status.items()
assert ident == '1'
assert status_a['href'] == href_a
assert status_a['etag'] == etag_a
assert status_b['href'] == href_b
assert status_b['etag'] == etag_b
def test_empty_storage(tmpdir, runner):
runner.write_with_general(dedent('''
[pair my_pair]
a = "my_a"
b = "my_b"
collections = null
[storage my_a]
type = "filesystem"
path = "{0}/path_a/"
fileext = ".txt"
[storage my_b]
type = "filesystem"
path = "{0}/path_b/"
fileext = ".txt"
''').format(str(tmpdir)))
tmpdir.mkdir('path_a')
tmpdir.mkdir('path_b')
result = runner.invoke(['discover'])
assert not result.exception
result = runner.invoke(['sync'])
assert not result.exception
item = format_item('haha')
tmpdir.join('path_a/haha.txt').write(item.raw)
result = runner.invoke(['sync'])
assert not result.exception
tmpdir.join('path_b/haha.txt').remove()
result = runner.invoke(['sync'])
lines = result.output.splitlines()
assert lines[0] == 'Syncing my_pair'
assert lines[1].startswith('error: my_pair: '
'Storage "my_b" was completely emptied.')
assert result.exception
def test_ident_conflict(tmpdir, runner):
runner.write_with_general(dedent('''
[pair foobar]
a = "foo"
b = "bar"
collections = null
[storage foo]
type = "filesystem"
path = "{base}/foo/"
fileext = ".txt"
[storage bar]
type = "filesystem"
path = "{base}/bar/"
fileext = ".txt"
'''.format(base=str(tmpdir))))
foo = tmpdir.mkdir('foo')
tmpdir.mkdir('bar')
item = format_item('1')
foo.join('one.txt').write(item.raw)
foo.join('two.txt').write(item.raw)
foo.join('three.txt').write(item.raw)
result = runner.invoke(['discover'])
assert not result.exception
result = runner.invoke(['sync'])
assert result.exception
assert ('error: foobar: Storage "foo" contains multiple items with the '
'same UID or even content') in result.output
assert sorted([
'one.txt' in result.output,
'two.txt' in result.output,
'three.txt' in result.output,
]) == [False, True, True]
def test_post_hook_active(self, tmpdir):
s = self.storage_class(str(tmpdir), '.txt', post_hook='rm')
s.upload(format_item('a/b/c'))
assert not list(s.list())
def test_too_long_uid(self, tmpdir):
s = self.storage_class(str(tmpdir), '.txt')
item = format_item('hue' * 600)
href, etag = s.upload(item)
assert item.uid not in href
def test_ident_with_slash(self, tmpdir):
s = self.storage_class(str(tmpdir), '.txt')
s.upload(format_item('a/b/c'))
item_file, = tmpdir.listdir()
assert '/' not in item_file.basename and item_file.isfile()
def test_collections_cache_invalidation(tmpdir, runner):
foo = tmpdir.mkdir('foo')
bar = tmpdir.mkdir('bar')
for x in 'abc':
foo.mkdir(x)
bar.mkdir(x)
runner.write_with_general(dedent('''
[storage foo]
type = "filesystem"
path = "{0}/foo/"
fileext = ".txt"
[storage bar]
type = "filesystem"
path = "{0}/bar/"
fileext = ".txt"
[pair foobar]
a = "foo"
b = "bar"
collections = ["a", "b", "c"]
''').format(str(tmpdir)))
foo.join('a/itemone.txt').write(format_item('itemone').raw)
result = runner.invoke(['discover'])
assert not result.exception
result = runner.invoke(['sync'])
assert not result.exception
assert 'detected change in config file' not in result.output.lower()
rv = bar.join('a').listdir()
assert len(rv) == 1
assert rv[0].basename == 'itemone.txt'
runner.write_with_general(dedent('''
[storage foo]
type = "filesystem"
path = "{0}/foo/"
fileext = ".txt"
[storage bar]
type = "filesystem"
path = "{0}/bar2/"
fileext = ".txt"
[pair foobar]
a = "foo"
b = "bar"
collections = ["a", "b", "c"]
''').format(str(tmpdir)))
for entry in tmpdir.join('status').listdir():
if not str(entry).endswith('.collections'):
entry.remove()
bar2 = tmpdir.mkdir('bar2')
for x in 'abc':
bar2.mkdir(x)
result = runner.invoke(['sync'])
assert 'detected change in config file' in result.output.lower()
assert result.exception
result = runner.invoke(['discover'])
assert not result.exception
result = runner.invoke(['sync'])
assert not result.exception
rv = bar.join('a').listdir()
rv2 = bar2.join('a').listdir()
assert len(rv) == len(rv2) == 1
assert rv[0].basename == rv2[0].basename == 'itemone.txt'
