def test_correct_output_nonempty(self, s3_client, local_client, capsys):
config = {
"targets": {
"foo": {
"local_folder": local_client.get_uri(),
"s3_uri": s3_client.get_uri(),
"aws_access_key_id": "",
"aws_secret_access_key": "",
"region_name": "eu-west-2",
}
}
}
set_s3_index(
s3_client,
{
"milk": {
"local_timestamp": get_timestamp(1989, 10, 23, 11, 30)
},
"honey": {
"local_timestamp": get_timestamp(2016, 12, 12, 8, 30)
},
"ginger": {
"local_timestamp": None
},
},
)
set_local_index(
local_client,
{
"milk": {
"local_timestamp": get_timestamp(2016, 12, 12, 8, 30)
},
"honey": {
"local_timestamp": get_timestamp(2016, 11, 10, 18, 40)
},
"lemon": {
"local_timestamp": get_timestamp(2017, 2, 2, 8, 30)
},
"ginger": {
"local_timestamp": None
},
},
)
args = argparse.Namespace(target="foo",
sort_by="key",
show_all=False,
descending=False)
command = LsCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert err == ""
assert out == ("key local s3\n"
"----- ------------------- -------------------\n"
"honey 2016-11-10 18:40:00 2016-12-12 08:30:00\n"
"lemon 2017-02-02 08:30:00\n"
"milk 2016-12-12 08:30:00 1989-10-23 11:30:00\n"
"Total Size: 0.00Mb\n")
def test_deleted_but_different_remote_timestamp(self, local_client,
s3_client):
utils.set_local_index(
local_client,
{
"chemistry.txt": {
"local_timestamp": 4000,
"remote_timestamp": 3000
}
},
)
utils.set_s3_index(
s3_client,
{
"chemistry.txt": {
"local_timestamp": 6000,
"remote_timestamp": 6000
}
},
)
utils.set_s3_contents(s3_client, "chemistry.txt", timestamp=6000)
worker = sync.SyncWorker(local_client, s3_client)
resolutions, unhandled_events = worker.get_sync_states()
expected_unhandled_events = {
"chemistry.txt": (
SyncState(SyncState.DELETED, None, 3000),
SyncState(SyncState.NOCHANGES, 6000, 6000),
)
}
assert resolutions == {}
assert unhandled_events == expected_unhandled_events
def test_nochanges_but_different_remote_timestamps(self, local_client,
s3_client):
utils.set_local_index(local_client, {
'german.txt': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
}
})
utils.set_s3_index(s3_client, {
'german.txt': {
'local_timestamp': 6000,
'remote_timestamp': 6000,
}
})
utils.set_local_contents(local_client, 'german.txt', timestamp=4000)
utils.set_s3_contents(s3_client, 'german.txt', timestamp=6000)
worker = sync.SyncWorker(local_client, s3_client)
resolutions, unhandled_events = worker.get_sync_states()
expected_resolutions = {
'german.txt':
Resolution("UPDATE", local_client, s3_client, 'german.txt', 6000)
}
assert resolutions == expected_resolutions
assert unhandled_events == {}
def test_updated_but_different_remote_timestamp(self, local_client, s3_client):
utils.set_local_index(local_client, {
'biology.txt': {
'local_timestamp': 4000,
'remote_timestamp': 3000,
}
})
utils.set_s3_index(s3_client, {
'biology.txt': {
'local_timestamp': 6000,
'remote_timestamp': 6000,
}
})
utils.set_local_contents(local_client, 'biology.txt', timestamp=4500)
utils.set_s3_contents(s3_client, 'biology.txt', timestamp=6000)
worker = sync.SyncWorker(local_client, s3_client)
resolutions, unhandled_events = worker.get_sync_states()
expected_unhandled_events = {
'biology.txt': (
SyncState(SyncState.UPDATED, 4500, 3000), SyncState(SyncState.NOCHANGES, 6000, 6000)
)
}
assert resolutions == {}
assert unhandled_events == expected_unhandled_events
def test_ignore_files(self, local_client):
utils.set_local_contents(local_client,
".syncignore",
timestamp=3200,
data=("*.zip\n"
"foo*\n"))
local_client.reload_ignore_files()
utils.set_local_index(
local_client,
{"pony.tar": {
"local_timestamp": 4000,
"remote_timestamp": 3000
}},
)
utils.set_local_contents(local_client, "test.zip")
utils.set_local_contents(local_client, "foobar")
utils.set_local_contents(local_client, "foo")
utils.set_local_contents(local_client, "pony.tar", timestamp=8000)
local_client.update_index()
expected_index = {
".syncignore": {
"local_timestamp": 3200,
"remote_timestamp": None
},
"pony.tar": {
"local_timestamp": 8000,
"remote_timestamp": 3000
},
}
assert local_client.index == expected_index
def test_ignore_files(self, local_client):
utils.set_local_contents(local_client,
'.syncignore',
timestamp=3200,
data=('*.zip\n'
'foo*\n'))
local_client.reload_ignore_files()
utils.set_local_index(local_client, {
'pony.tar': {
'local_timestamp': 4000,
'remote_timestamp': 3000,
}
})
utils.set_local_contents(local_client, 'test.zip')
utils.set_local_contents(local_client, 'foobar')
utils.set_local_contents(local_client, 'foo')
utils.set_local_contents(local_client, 'pony.tar', timestamp=8000)
local_client.update_index()
expected_index = {
'.syncignore': {
'local_timestamp': 3200,
'remote_timestamp': None,
},
'pony.tar': {
'local_timestamp': 8000,
'remote_timestamp': 3000,
}
}
assert local_client.index == expected_index
def test_previously_deleted_now_created(self, local_client, s3_client):
utils.set_s3_index(s3_client, {
'foo': {
'local_timestamp': None,
'remote_timestamp': 4000,
}
})
utils.set_local_index(local_client, {
'foo': {
'local_timstamp': None,
'remote_timestamp': 4000,
}
})
utils.set_local_contents(local_client, 'foo', timestamp=7000)
# Will create previously deleted file
worker = sync.SyncWorker(local_client, s3_client)
worker.sync()
clients = [local_client, s3_client]
assert_local_keys(clients, ['foo'])
assert_remote_timestamp(clients, 'foo', 7000)
# delete the file again and check that it is successful
utils.delete_local(local_client, 'foo')
worker.sync()
assert_local_keys(clients, [])
assert_remote_timestamp(clients, 'foo', 7000)
def test_set_index_local_timestamp(self, local_client):
utils.set_local_index(local_client, {
'foo': {
'local_timestamp': 100000,
'remote_timestamp': 2222222222,
}
})
local_client.set_index_local_timestamp('foo', 123456)
assert local_client.get_index_local_timestamp('foo') == 123456
def test_correct_output_nonempty(self, s3_client, local_client, capsys):
config = {
'targets': {
'foo': {
'local_folder': local_client.get_uri(),
's3_uri': s3_client.get_uri(),
'aws_access_key_id': '',
'aws_secret_access_key': '',
'region_name': 'eu-west-2',
}
}
}
set_s3_index(
s3_client, {
'milk': {
'local_timestamp': get_timestamp(1989, 10, 23, 11, 30)
},
'honey': {
'local_timestamp': get_timestamp(2016, 12, 12, 8, 30)
},
'ginger': {
'local_timestamp': None,
}
})
set_local_index(
local_client, {
'milk': {
'local_timestamp': get_timestamp(2016, 12, 12, 8, 30)
},
'honey': {
'local_timestamp': get_timestamp(2016, 11, 10, 18, 40)
},
'lemon': {
'local_timestamp': get_timestamp(2017, 2, 2, 8, 30)
},
'ginger': {
'local_timestamp': None,
}
})
args = argparse.Namespace(
target='foo',
sort_by='key',
show_all=False,
descending=False,
)
command = LsCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert err == ""
assert out == ('key local s3\n'
'----- ------------------- -------------------\n'
'honey 2016-11-10 18:40:00 2016-12-12 08:30:00\n'
'lemon 2017-02-02 08:30:00\n'
'milk 2016-12-12 08:30:00 1989-10-23 11:30:00\n')
def test_get_local_keys(self, local_client):
utils.set_local_index(local_client,
{}) # .index file should not come up in results
utils.set_local_contents(local_client, '.bashrc')
utils.set_local_contents(local_client, 'foo')
utils.set_local_contents(local_client, 'bar')
actual_output = local_client.get_local_keys()
expected_output = ['foo', 'bar', '.bashrc']
assert sorted(expected_output) == sorted(actual_output)
def test_get_local_keys(self, local_client):
utils.set_local_index(local_client,
{}) # .index file should not come up in results
utils.set_local_contents(local_client, ".bashrc")
utils.set_local_contents(local_client, "foo")
utils.set_local_contents(local_client, "bar")
actual_output = local_client.get_local_keys()
expected_output = ["foo", "bar", ".bashrc"]
assert sorted(expected_output) == sorted(actual_output)
def test_deleted_doesnotexist(self, local_client, s3_client):
utils.set_local_index(local_client, {
'physics.txt': {
'local_timestamp': 5000,
'remote_timestamp': 4550,
}
})
worker = sync.SyncWorker(local_client, s3_client)
resolutions, unhandled_events = worker.get_sync_states()
assert resolutions == {}
assert unhandled_events == {}
def test_set_index_local_timestamp(self, local_client):
utils.set_local_index(
local_client,
{
"foo": {
"local_timestamp": 100000,
"remote_timestamp": 2222222222
}
},
)
local_client.set_index_local_timestamp("foo", 123456)
assert local_client.get_index_local_timestamp("foo") == 123456
def test_load_index(self, local_client):
data = {
"foo": {
"local_timestamp": 4000,
"remote_timestamp": 4000
},
"bar/baz.txt": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
}
utils.set_local_index(local_client, data)
assert local_client.index == data
def test_load_index(self, local_client):
data = {
'foo': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
},
'bar/baz.txt': {
'local_timestamp': 5000,
'remote_timestamp': 5000,
},
}
utils.set_local_index(local_client, data)
assert local_client.index == data
def test_show_all(self, s3_client, local_client, capsys):
config = {
"targets": {
"foo": {
"local_folder": local_client.get_uri(),
"s3_uri": s3_client.get_uri(),
"aws_access_key_id": "",
"aws_secret_access_key": "",
"region_name": "eu-west-2",
}
}
}
set_s3_index(
s3_client,
{
"cheese": {
"local_timestamp": get_timestamp(2017, 12, 12, 8, 30)
},
"crackers": {
"local_timestamp": None
},
},
)
set_local_index(
local_client,
{
"cheese": {
"local_timestamp": get_timestamp(2017, 2, 2, 8, 30)
},
"crackers": {
"local_timestamp": None
},
},
)
args = argparse.Namespace(target="foo",
sort_by="key",
show_all=True,
descending=False)
command = LsCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert err == ""
assert out == ("key local s3\n"
"-------- ------------------- -------------------\n"
"cheese 2017-02-02 08:30:00 2017-12-12 08:30:00\n"
"crackers <deleted>\n"
"Total Size: 0.00Mb\n")
def test_put_existing(self, local_client):
utils.set_local_index(local_client,
{"doge.txt": {
"local_timestamp": 1111111
}})
data = b"canis lupus familiaris"
local_client.put(key="doge.txt",
sync_object=SyncObject(io.BytesIO(data), len(data),
20000))
assert local_client.index["doge.txt"]["remote_timestamp"] == 20000
assert local_client.index["doge.txt"]["local_timestamp"] == 1111111
assert utils.get_local_contents(local_client, "doge.txt") == data
def test_put_existing(self, local_client):
utils.set_local_index(local_client,
{'doge.txt': {
'local_timestamp': 1111111
}})
data = b'canis lupus familiaris'
local_client.put(key='doge.txt',
sync_object=SyncObject(io.BytesIO(data), len(data),
20000))
assert local_client.index['doge.txt']['remote_timestamp'] == 20000
assert local_client.index['doge.txt']['local_timestamp'] == 1111111
assert utils.get_local_contents(local_client, 'doge.txt') == data
def test_get_index_keys(self, local_client):
data = {
'foo': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
},
'bar/baz.txt': {
'local_timestamp': 5000,
'remote_timestamp': 5000,
},
}
utils.set_local_index(local_client, data)
actual_output = local_client.get_index_keys()
expected_output = ['foo', 'bar/baz.txt']
assert sorted(list(actual_output)) == sorted(expected_output)
def test_get_all_remote_timestamps(self, local_client):
utils.set_local_index(
local_client,
{
"frap": {
"remote_timestamp": 4000
},
"brap": {
"remote_timestamp": 9999999
}
},
)
expected_output = {"frap": 4000, "brap": 9999999}
actual_output = local_client.get_all_remote_timestamps()
assert actual_output == expected_output
def test_get_index_keys(self, local_client):
data = {
"foo": {
"local_timestamp": 4000,
"remote_timestamp": 4000
},
"bar/baz.txt": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
}
utils.set_local_index(local_client, data)
actual_output = local_client.get_index_keys()
expected_output = ["foo", "bar/baz.txt"]
assert sorted(list(actual_output)) == sorted(expected_output)
def test_show_all(self, s3_client, local_client, capsys):
config = {
'targets': {
'foo': {
'local_folder': local_client.get_uri(),
's3_uri': s3_client.get_uri(),
'aws_access_key_id': '',
'aws_secret_access_key': '',
'region_name': 'eu-west-2',
}
}
}
set_s3_index(
s3_client, {
'cheese': {
'local_timestamp': get_timestamp(2017, 12, 12, 8, 30)
},
'crackers': {
'local_timestamp': None,
}
})
set_local_index(
local_client, {
'cheese': {
'local_timestamp': get_timestamp(2017, 2, 2, 8, 30)
},
'crackers': {
'local_timestamp': None,
}
})
args = argparse.Namespace(
target='foo',
sort_by='key',
show_all=True,
descending=False,
)
command = LsCommand(args, config, create_logger())
command.run()
out, err = capsys.readouterr()
assert err == ""
assert out == ('key local s3\n'
'-------- ------------------- -------------------\n'
'cheese 2017-02-02 08:30:00 2017-12-12 08:30:00\n'
'crackers <deleted>\n')
def test_get_all_remote_timestamps(self, local_client):
utils.set_local_index(
local_client, {
'frap': {
'remote_timestamp': 4000,
},
'brap': {
'remote_timestamp': 9999999,
}
})
expected_output = {
'frap': 4000,
'brap': 9999999,
}
actual_output = local_client.get_all_remote_timestamps()
assert actual_output == expected_output
def test_get_index_timestamps(self, local_client):
utils.set_local_index(
local_client, {
'foo': {
'local_timestamp': 4000,
'remote_timestamp': 32000,
},
'bar': {
'local_timestamp': 9999999,
}
})
assert local_client.get_index_local_timestamp('foo') == 4000
assert local_client.get_index_local_timestamp('dontexist') is None
assert local_client.get_index_local_timestamp('bar') == 9999999
assert local_client.get_remote_timestamp('foo') == 32000
assert local_client.get_remote_timestamp('dontexist') is None
assert local_client.get_remote_timestamp('bar') is None
def test_all_keys(self, local_client):
utils.set_local_contents(local_client, '.index')
utils.set_local_contents(local_client, 'foo')
utils.set_local_contents(local_client, 'bar/boo.md')
data = {
'foo': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
},
'bar/baz.txt': {
'local_timestamp': 5000,
'remote_timestamp': 5000,
},
}
utils.set_local_index(local_client, data)
actual_output = local_client.get_all_keys()
expected_output = ['foo', 'bar/boo.md', 'bar/baz.txt']
assert sorted(actual_output) == sorted(expected_output)
def test_all_keys(self, local_client):
utils.set_local_contents(local_client, ".index")
utils.set_local_contents(local_client, "foo")
utils.set_local_contents(local_client, "bar/boo.md")
data = {
"foo": {
"local_timestamp": 4000,
"remote_timestamp": 4000
},
"bar/baz.txt": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
}
utils.set_local_index(local_client, data)
actual_output = local_client.get_all_keys()
expected_output = ["foo", "bar/boo.md", "bar/baz.txt"]
assert sorted(actual_output) == sorted(expected_output)
def test_get_index_timestamps(self, local_client):
utils.set_local_index(
local_client,
{
"foo": {
"local_timestamp": 4000,
"remote_timestamp": 32000
},
"bar": {
"local_timestamp": 9999999
},
},
)
assert local_client.get_index_local_timestamp("foo") == 4000
assert local_client.get_index_local_timestamp("dontexist") is None
assert local_client.get_index_local_timestamp("bar") == 9999999
assert local_client.get_remote_timestamp("foo") == 32000
assert local_client.get_remote_timestamp("dontexist") is None
assert local_client.get_remote_timestamp("bar") is None
def test_interrupted_flush_index(self, local_client):
target_index = {
"red": {
"local_timestamp": 4000,
"remote_timestamp": 4000
}
}
utils.set_local_index(local_client, target_index)
local_client.index = {"invalid_data": []}
with mock.patch("json.dump") as json_dump:
json_dump.side_effect = ValueError("something went wrong")
with pytest.raises(ValueError):
local_client.flush_index(compressed=False)
with open(local_client.index_path(), "rt") as fp:
index = json.load(fp)
assert index == target_index
def test_interrupted_flush_index(self, local_client):
target_index = {
'red': {
'local_timestamp': 4000,
'remote_timestamp': 4000,
}
}
utils.set_local_index(local_client, target_index)
local_client.index = {'invalid_data': []}
with mock.patch('json.dump') as json_dump:
json_dump.side_effect = ValueError('something went wrong')
with pytest.raises(ValueError):
local_client.flush_index(compressed=False)
with open(local_client.index_path(), 'rt') as fp:
index = json.load(fp)
assert index == target_index
def test_update_index_non_empty(self, local_client):
utils.set_local_contents(local_client, "foo", 13371337)
utils.set_local_contents(local_client, "bar", 50032003)
utils.set_local_index(
local_client,
{
"foo": {
"local_timestamp": 4000,
"remote_timestamp": 4000
},
"bar": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
"baz": {
"local_timestamp": 5000,
"remote_timestamp": 5000
},
},
)
local_client.update_index()
expected_output = {
"foo": {
"local_timestamp": 13371337,
"remote_timestamp": 4000
},
"bar": {
"local_timestamp": 50032003,
"remote_timestamp": 5000
},
"baz": {
"local_timestamp": None,
"remote_timestamp": 5000
},
}
assert local_client.index == expected_output
