def test_nochanges_but_different_remote_timestamps(self, local_client,
s3_client):
utils.set_local_index(local_client, {
'german.txt': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
}
})
utils.set_s3_index(s3_client, {
'german.txt': {
'local_timestamp': 6000,
'remote_timestamp': 6000,
}
})
utils.set_local_contents(local_client, 'german.txt', timestamp=4000)
utils.set_s3_contents(s3_client, 'german.txt', timestamp=6000)
worker = sync.SyncWorker(local_client, s3_client)
resolutions, unhandled_events = worker.get_sync_states()
expected_resolutions = {
'german.txt':
Resolution("UPDATE", local_client, s3_client, 'german.txt', 6000)
}
assert resolutions == expected_resolutions
assert unhandled_events == {}
def test_get_all_real_local_timestamps(self, local_client):
utils.set_local_contents(local_client, "red", 2323230)
utils.set_local_contents(local_client, "blue", 80808008)
expected_output = {"red": 2323230, "blue": 80808008}
actual_output = local_client.get_all_real_local_timestamps()
assert actual_output == expected_output
def test_previously_deleted_now_created(self, local_client, s3_client):
utils.set_s3_index(s3_client, {
'foo': {
'local_timestamp': None,
'remote_timestamp': 4000,
}
})
utils.set_local_index(local_client, {
'foo': {
'local_timstamp': None,
'remote_timestamp': 4000,
}
})
utils.set_local_contents(local_client, 'foo', timestamp=7000)
# Will create previously deleted file
worker = sync.SyncWorker(local_client, s3_client)
worker.sync()
clients = [local_client, s3_client]
assert_local_keys(clients, ['foo'])
assert_remote_timestamp(clients, 'foo', 7000)
# delete the file again and check that it is successful
utils.delete_local(local_client, 'foo')
worker.sync()
assert_local_keys(clients, [])
assert_remote_timestamp(clients, 'foo', 7000)
def test_ignore_files(self, local_client):
utils.set_local_contents(local_client,
".syncignore",
timestamp=3200,
data=("*.zip\n"
"foo*\n"))
local_client.reload_ignore_files()
utils.set_local_index(
local_client,
{"pony.tar": {
"local_timestamp": 4000,
"remote_timestamp": 3000
}},
)
utils.set_local_contents(local_client, "test.zip")
utils.set_local_contents(local_client, "foobar")
utils.set_local_contents(local_client, "foo")
utils.set_local_contents(local_client, "pony.tar", timestamp=8000)
local_client.update_index()
expected_index = {
".syncignore": {
"local_timestamp": 3200,
"remote_timestamp": None
},
"pony.tar": {
"local_timestamp": 8000,
"remote_timestamp": 3000
},
}
assert local_client.index == expected_index
def test_ignore_files(self, local_client):
utils.set_local_contents(local_client,
'.syncignore',
timestamp=3200,
data=('*.zip\n'
'foo*\n'))
local_client.reload_ignore_files()
utils.set_local_index(local_client, {
'pony.tar': {
'local_timestamp': 4000,
'remote_timestamp': 3000,
}
})
utils.set_local_contents(local_client, 'test.zip')
utils.set_local_contents(local_client, 'foobar')
utils.set_local_contents(local_client, 'foo')
utils.set_local_contents(local_client, 'pony.tar', timestamp=8000)
local_client.update_index()
expected_index = {
'.syncignore': {
'local_timestamp': 3200,
'remote_timestamp': None,
},
'pony.tar': {
'local_timestamp': 8000,
'remote_timestamp': 3000,
}
}
assert local_client.index == expected_index
def test_delete_existing(self, local_client):
target_file = os.path.join(local_client.path, "foo")
utils.set_local_contents(local_client, "foo", 222222)
assert os.path.exists(target_file) is True
assert local_client.delete("foo") is True
assert os.path.exists(target_file) is False
def test_updated_but_different_remote_timestamp(self, local_client, s3_client):
utils.set_local_index(local_client, {
'biology.txt': {
'local_timestamp': 4000,
'remote_timestamp': 3000,
}
})
utils.set_s3_index(s3_client, {
'biology.txt': {
'local_timestamp': 6000,
'remote_timestamp': 6000,
}
})
utils.set_local_contents(local_client, 'biology.txt', timestamp=4500)
utils.set_s3_contents(s3_client, 'biology.txt', timestamp=6000)
worker = sync.SyncWorker(local_client, s3_client)
resolutions, unhandled_events = worker.get_sync_states()
expected_unhandled_events = {
'biology.txt': (
SyncState(SyncState.UPDATED, 4500, 3000), SyncState(SyncState.NOCHANGES, 6000, 6000)
)
}
assert resolutions == {}
assert unhandled_events == expected_unhandled_events
def test_get_existing(self, local_client):
data = b"blue green yellow"
utils.set_local_contents(local_client,
"whatup.md",
data=data.decode("utf8"))
sync_object = local_client.get("whatup.md")
assert sync_object.fp.read() == data
assert sync_object.total_size == len(data)
def test_get_existing(self, local_client):
data = b'blue green yellow'
utils.set_local_contents(local_client,
'whatup.md',
data=data.decode('utf8'))
sync_object = local_client.get('whatup.md')
assert sync_object.fp.read() == data
assert sync_object.total_size == len(data)
def test_diff(self, call, local_client, s3_client):
utils.set_local_contents(local_client, "something", 4000, "wow")
utils.set_s3_contents(s3_client, "something", 3000, "nice")
cli.show_diff(local_client, s3_client, "something")
assert call.call_count == 2
assert call.call_args_list[0][0][0][0] == "diff"
assert call.call_args_list[1][0][0][0] == "less"
def test_get_all_real_local_timestamps(self, local_client):
utils.set_local_contents(local_client, 'red', 2323230)
utils.set_local_contents(local_client, 'blue', 80808008)
expected_output = {
'red': 2323230,
'blue': 80808008,
}
actual_output = local_client.get_all_real_local_timestamps()
assert actual_output == expected_output
def test_conflict_choose_second_client(self, local_client, s3_client):
utils.set_local_contents(local_client, 'foo', timestamp=2000)
utils.set_s3_contents(s3_client, 'foo', timestamp=3000, data='123')
worker = sync.SyncWorker(local_client, s3_client)
worker.sync(conflict_choice='2')
clients = [local_client, s3_client]
assert_local_keys(clients, ['foo'])
# Chooses second client
assert_remote_timestamp(clients, 'foo', 3000)
assert_contents(clients, 'foo', b'123')
def test_conflict_choose_second_client(self, local_client, s3_client):
utils.set_local_contents(local_client, "foo", timestamp=2000)
utils.set_s3_contents(s3_client, "foo", timestamp=3000, data="123")
worker = sync.SyncWorker(local_client, s3_client)
worker.sync(conflict_choice="2")
clients = [local_client, s3_client]
assert_local_keys(clients, ["foo"])
# Chooses second client
assert_remote_timestamp(clients, "foo", 3000)
assert_contents(clients, "foo", b"123")
def test_interrupted_put(self, local_client):
utils.set_local_contents(local_client,
"keychain",
data="iamsomedatathatexists")
sync_object = SyncObject(utils.InterruptedBytesIO(), 900000, 3000)
with pytest.raises(ValueError):
local_client.put("keychain", sync_object)
result = local_client.get("keychain")
assert result.fp.read() == b"iamsomedatathatexists"
def test_correct_output(self, local_client, s3_client):
clients = [local_client, s3_client]
worker = sync.SyncWorker(local_client, s3_client)
utils.set_local_contents(local_client, "foo")
utils.set_s3_contents(s3_client, "baz", timestamp=2000, data="testing")
success = worker.run_resolutions({
"foo":
Resolution(Resolution.DELETE, local_client, None, "foo", 1000),
"baz":
Resolution(Resolution.CREATE, local_client, s3_client, "baz", 20),
})
assert sorted(success) == sorted(["foo", "baz"])
assert_local_keys(clients, ["baz"])
def test_progressbar_smoketest(s3_client, local_client):
# Just test that nothing blows up
set_local_contents(local_client,
"history.txt",
data="a long long time ago",
timestamp=5000)
worker = SyncWorker(
s3_client,
local_client,
start_callback=display_progress_bar,
update_callback=update_progress_bar,
complete_callback=hide_progress_bar,
)
worker.sync()
def test_correct_output(self, local_client, s3_client):
def keyboard_interrupt():
raise KeyboardInterrupt()
clients = [local_client, s3_client]
worker = sync.SyncWorker(local_client, s3_client)
utils.set_local_contents(local_client, 'foo')
utils.set_s3_contents(s3_client, 'baz', timestamp=2000, data='testing')
success = worker.run_resolutions({
'foo': Resolution(Resolution.DELETE, local_client, None, 'foo', 1000),
'baz': Resolution(Resolution.CREATE, local_client, s3_client, 'baz', 20),
})
assert sorted(success) == sorted(['foo', 'baz'])
assert_local_keys(clients, ['baz'])
def test_action_callback(self, local_client, s3_client):
utils.set_s3_contents(s3_client, 'colors/cream', 9999)
utils.set_local_contents(local_client, 'colors/green', 3000)
utils.set_local_contents(local_client, 'colors/blue', 2000)
callback_mock = mock.MagicMock()
worker = sync.SyncWorker(local_client,
s3_client,
action_callback=callback_mock)
worker.sync()
assert callback_mock.call_count == 3
assert callback_mock.call_args_list[0][0][0].key == 'colors/blue'
assert callback_mock.call_args_list[1][0][0].key == 'colors/cream'
assert callback_mock.call_args_list[2][0][0].key == 'colors/green'
def test_three_way_sync(self, local_client, s3_client, local_client_2):
utils.set_local_contents(local_client, "foo", timestamp=1000)
utils.set_s3_contents(s3_client, "bar", timestamp=2000, data="red")
utils.set_local_contents(local_client_2, "baz", timestamp=3000)
worker_1 = sync.SyncWorker(local_client, s3_client)
worker_2 = sync.SyncWorker(local_client_2, s3_client)
worker_3 = sync.SyncWorker(local_client, local_client_2)
worker_1.sync()
worker_2.sync()
worker_3.sync()
clients = [local_client, s3_client, local_client_2]
assert_local_keys(clients, ["foo", "bar", "baz"])
assert_contents(clients, "bar", b"red")
assert_remote_timestamp(clients, "foo", 1000)
assert_remote_timestamp(clients, "bar", 2000)
assert_remote_timestamp(clients, "baz", 3000)
utils.set_s3_contents(s3_client, "bar", timestamp=8000, data="green")
worker_1.sync()
worker_2.sync()
worker_3.sync()
assert_local_keys(clients, ["foo", "bar", "baz"])
assert_contents(clients, "bar", b"green")
assert_remote_timestamp(clients, "foo", 1000)
assert_remote_timestamp(clients, "bar", 8000)
assert_remote_timestamp(clients, "baz", 3000)
utils.delete_local(local_client_2, "foo")
worker_1.sync()
worker_2.sync()
worker_3.sync()
assert_existence(clients, ["foo"], False)
assert_local_keys(clients, ["bar", "baz"])
assert_remote_timestamp(clients, "foo", 1000)
worker_1.sync()
worker_2.sync()
worker_3.sync()
def test_update_index_empty(self, local_client):
utils.set_local_contents(local_client, 'foo', 13371337)
utils.set_local_contents(local_client, 'bar', 50032003)
local_client.update_index()
# remote timestamp should not be included since it does not exist
expected_output = {
'foo': {
'local_timestamp': 13371337,
'remote_timestamp': None,
},
'bar': {
'local_timestamp': 50032003,
'remote_timestamp': None,
},
}
assert local_client.index == expected_output
def test_three_way_sync(self, local_client, s3_client, local_client_2):
utils.set_local_contents(local_client, 'foo', timestamp=1000)
utils.set_s3_contents(s3_client, 'bar', timestamp=2000, data='red')
utils.set_local_contents(local_client_2, 'baz', timestamp=3000)
worker_1 = sync.SyncWorker(local_client, s3_client)
worker_2 = sync.SyncWorker(local_client_2, s3_client)
worker_3 = sync.SyncWorker(local_client, local_client_2)
worker_1.sync()
worker_2.sync()
worker_3.sync()
clients = [local_client, s3_client, local_client_2]
assert_local_keys(clients, ['foo', 'bar', 'baz'])
assert_contents(clients, 'bar', b'red')
assert_remote_timestamp(clients, 'foo', 1000)
assert_remote_timestamp(clients, 'bar', 2000)
assert_remote_timestamp(clients, 'baz', 3000)
utils.set_s3_contents(s3_client, 'bar', timestamp=8000, data='green')
worker_1.sync()
worker_2.sync()
worker_3.sync()
assert_local_keys(clients, ['foo', 'bar', 'baz'])
assert_contents(clients, 'bar', b'green')
assert_remote_timestamp(clients, 'foo', 1000)
assert_remote_timestamp(clients, 'bar', 8000)
assert_remote_timestamp(clients, 'baz', 3000)
utils.delete_local(local_client_2, 'foo')
worker_1.sync()
worker_2.sync()
worker_3.sync()
assert_existence(clients, ['foo'], False)
assert_local_keys(clients, ['bar', 'baz'])
assert_remote_timestamp(clients, 'foo', 1000)
worker_1.sync()
worker_2.sync()
worker_3.sync()
def test_progressbar_smoketest(s3_client, local_client):
# Just test that nothing blows up
utils.set_local_contents(
local_client,
'history.txt',
data='a long long time ago',
timestamp=5000,
)
worker = sync.SyncWorker(
s3_client,
local_client,
start_callback=cli.display_progress_bar,
update_callback=cli.update_progress_bar,
complete_callback=cli.hide_progress_bar,
)
worker.sync()
def test_update_index_empty(self, local_client):
utils.set_local_contents(local_client, "foo", 13371337)
utils.set_local_contents(local_client, "bar", 50032003)
local_client.update_index()
# remote timestamp should not be included since it does not exist
expected_output = {
"foo": {
"local_timestamp": 13371337,
"remote_timestamp": None
},
"bar": {
"local_timestamp": 50032003,
"remote_timestamp": None
},
}
assert local_client.index == expected_output
def test_ignore_conflicts(self, local_client, s3_client):
utils.set_local_contents(local_client, "foo", timestamp=2000)
utils.set_s3_contents(s3_client, "foo", timestamp=3000)
utils.set_s3_contents(s3_client, "bar", timestamp=5600, data="usador")
worker = sync.SyncWorker(local_client, s3_client)
worker.sync(conflict_choice="ignore")
clients = [local_client, s3_client]
assert_local_keys(clients, ["foo", "bar"])
assert_remote_timestamp(clients, "foo", None)
assert_remote_timestamp(clients, "bar", 5600)
assert_existence(clients, ["foo", "bar"], True)
assert_contents(clients, "bar", b"usador")
# Check that none of the files have been modified due to ignore flag
assert local_client.get_real_local_timestamp("foo") == 2000
assert s3_client.get_real_local_timestamp("foo") == 3000
def test_ignore_conflicts(self, local_client, s3_client):
utils.set_local_contents(local_client, 'foo', timestamp=2000)
utils.set_s3_contents(s3_client, 'foo', timestamp=3000)
utils.set_s3_contents(s3_client, 'bar', timestamp=5600, data='usador')
worker = sync.SyncWorker(local_client, s3_client)
worker.sync(conflict_choice='ignore')
clients = [local_client, s3_client]
assert_local_keys(clients, ['foo', 'bar'])
assert_remote_timestamp(clients, 'foo', None)
assert_remote_timestamp(clients, 'bar', 5600)
assert_existence(clients, ['foo', 'bar'], True)
assert_contents(clients, 'bar', b'usador')
# Check that none of the files have been modified due to ignore flag
assert local_client.get_real_local_timestamp('foo') == 2000
assert s3_client.get_real_local_timestamp('foo') == 3000
def test_local_with_s3(self, local_client, s3_client):
utils.set_s3_contents(s3_client, 'colors/cream', 9999, '#ddeeff')
utils.set_local_contents(local_client, 'colors/red', 5000, '#ff0000')
utils.set_local_contents(local_client, 'colors/green', 3000, '#00ff00')
utils.set_local_contents(local_client, 'colors/blue', 2000, '#0000ff')
worker = sync.SyncWorker(local_client, s3_client)
worker.sync()
clients = [local_client, s3_client]
expected_keys = [
'colors/red', 'colors/green', 'colors/blue', 'colors/cream'
]
assert_local_keys(clients, expected_keys)
assert_contents(clients, 'colors/red', b'#ff0000')
assert_contents(clients, 'colors/green', b'#00ff00')
assert_contents(clients, 'colors/blue', b'#0000ff')
assert_contents(clients, 'colors/cream', b'#ddeeff')
assert_remote_timestamp(clients, 'colors/red', 5000)
assert_remote_timestamp(clients, 'colors/green', 3000)
assert_remote_timestamp(clients, 'colors/blue', 2000)
assert_remote_timestamp(clients, 'colors/cream', 9999)
utils.delete_local(local_client, 'colors/red')
worker.sync()
expected_keys = ['colors/green', 'colors/blue', 'colors/cream']
assert_local_keys(clients, expected_keys)
def test_fresh_sync(self, local_client, s3_client):
utils.set_local_contents(local_client, 'foo', timestamp=1000)
utils.set_local_contents(local_client, 'bar', timestamp=2000)
utils.set_s3_contents(s3_client, 'baz', timestamp=3000, data='what is up?')
worker = sync.SyncWorker(local_client, s3_client)
worker.sync()
clients = [local_client, s3_client]
assert_local_keys(clients, ['foo', 'bar', 'baz'])
assert_remote_timestamp(clients, 'foo', 1000)
assert_remote_timestamp(clients, 'bar', 2000)
assert_remote_timestamp(clients, 'baz', 3000)
assert_existence(clients, ['foo', 'bar', 'baz'], True)
assert_contents(clients, 'baz', b'what is up?')
utils.delete_local(local_client, 'foo')
utils.set_local_contents(local_client, 'test', timestamp=5000)
utils.set_s3_contents(s3_client, 'hello', timestamp=6000)
utils.set_s3_contents(s3_client, 'baz', timestamp=8000, data='just syncing some stuff')
worker.sync()
assert_existence(clients, ['foo'], False)
assert_local_keys(clients, ['test', 'bar', 'baz', 'hello'])
assert_remote_timestamp(clients, 'bar', 2000)
assert_remote_timestamp(clients, 'foo', 1000)
assert_remote_timestamp(clients, 'test', 5000)
assert_remote_timestamp(clients, 'hello', 6000)
assert_remote_timestamp(clients, 'baz', 8000)
assert_contents(clients, 'baz', b'just syncing some stuff')
worker.sync()
def test_local_with_s3(self, local_client, s3_client):
utils.set_s3_contents(s3_client, "colors/cream", 9999, "#ddeeff")
utils.set_local_contents(local_client, "colors/red", 5000, "#ff0000")
utils.set_local_contents(local_client, "colors/green", 3000, "#00ff00")
utils.set_local_contents(local_client, "colors/blue", 2000, "#0000ff")
worker = sync.SyncWorker(local_client, s3_client)
worker.sync()
clients = [local_client, s3_client]
expected_keys = [
"colors/red", "colors/green", "colors/blue", "colors/cream"
]
assert_local_keys(clients, expected_keys)
assert_contents(clients, "colors/red", b"#ff0000")
assert_contents(clients, "colors/green", b"#00ff00")
assert_contents(clients, "colors/blue", b"#0000ff")
assert_contents(clients, "colors/cream", b"#ddeeff")
assert_remote_timestamp(clients, "colors/red", 5000)
assert_remote_timestamp(clients, "colors/green", 3000)
assert_remote_timestamp(clients, "colors/blue", 2000)
assert_remote_timestamp(clients, "colors/cream", 9999)
utils.delete_local(local_client, "colors/red")
worker.sync()
expected_keys = ["colors/green", "colors/blue", "colors/cream"]
assert_local_keys(clients, expected_keys)
def test_update_index_non_empty(self, local_client):
utils.set_local_contents(local_client, "foo", 13371337)
utils.set_local_contents(local_client, "bar", 50032003)
utils.set_local_index(
local_client,
{
"foo": {
"local_timestamp": 4000,
"remote_timestamp": 4000
},
"bar": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
"baz": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
},
)
local_client.update_index()
expected_output = {
"foo": {
"local_timestamp": 13371337,
"remote_timestamp": 4000
},
"bar": {
"local_timestamp": 50032003,
"remote_timestamp": 5000
},
"baz": {
"local_timestamp": None,
"remote_timestamp": 5000
},
}
assert local_client.index == expected_output
def test_update_index_non_empty(self, local_client):
utils.set_local_contents(local_client, 'foo', 13371337)
utils.set_local_contents(local_client, 'bar', 50032003)
utils.set_local_index(
local_client, {
'foo': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
},
'bar': {
'local_timestamp': 5000,
'remote_timestamp': 5000,
},
'baz': {
'local_timestamp': 5000,
'remote_timestamp': 5000,
}
})
local_client.update_index()
expected_output = {
'foo': {
'local_timestamp': 13371337,
'remote_timestamp': 4000,
},
'bar': {
'local_timestamp': 50032003,
'remote_timestamp': 5000,
},
'baz': {
'local_timestamp': None,
'remote_timestamp': 5000,
},
}
assert local_client.index == expected_output