def test_correct_output_nonempty(self, s3_client, local_client, capsys): config = { "targets": { "foo": { "local_folder": local_client.get_uri(), "s3_uri": s3_client.get_uri(), "aws_access_key_id": "", "aws_secret_access_key": "", "region_name": "eu-west-2", } } } set_s3_index( s3_client, { "milk": { "local_timestamp": get_timestamp(1989, 10, 23, 11, 30) }, "honey": { "local_timestamp": get_timestamp(2016, 12, 12, 8, 30) }, "ginger": { "local_timestamp": None }, }, ) set_local_index( local_client, { "milk": { "local_timestamp": get_timestamp(2016, 12, 12, 8, 30) }, "honey": { "local_timestamp": get_timestamp(2016, 11, 10, 18, 40) }, "lemon": { "local_timestamp": get_timestamp(2017, 2, 2, 8, 30) }, "ginger": { "local_timestamp": None }, }, ) args = argparse.Namespace(target="foo", sort_by="key", show_all=False, descending=False) command = LsCommand(args, config, create_logger()) command.run() out, err = capsys.readouterr() assert err == "" assert out == ("key local s3\n" "----- ------------------- -------------------\n" "honey 2016-11-10 18:40:00 2016-12-12 08:30:00\n" "lemon 2017-02-02 08:30:00\n" "milk 2016-12-12 08:30:00 1989-10-23 11:30:00\n" "Total Size: 0.00Mb\n")
def test_deleted_but_different_remote_timestamp(self, local_client, s3_client): utils.set_local_index( local_client, { "chemistry.txt": { "local_timestamp": 4000, "remote_timestamp": 3000 } }, ) utils.set_s3_index( s3_client, { "chemistry.txt": { "local_timestamp": 6000, "remote_timestamp": 6000 } }, ) utils.set_s3_contents(s3_client, "chemistry.txt", timestamp=6000) worker = sync.SyncWorker(local_client, s3_client) resolutions, unhandled_events = worker.get_sync_states() expected_unhandled_events = { "chemistry.txt": ( SyncState(SyncState.DELETED, None, 3000), SyncState(SyncState.NOCHANGES, 6000, 6000), ) } assert resolutions == {} assert unhandled_events == expected_unhandled_events
def test_nochanges_but_different_remote_timestamps(self, local_client, s3_client): utils.set_local_index(local_client, { 'german.txt': { 'local_timestamp': 4000, 'remote_timestamp': 4000, } }) utils.set_s3_index(s3_client, { 'german.txt': { 'local_timestamp': 6000, 'remote_timestamp': 6000, } }) utils.set_local_contents(local_client, 'german.txt', timestamp=4000) utils.set_s3_contents(s3_client, 'german.txt', timestamp=6000) worker = sync.SyncWorker(local_client, s3_client) resolutions, unhandled_events = worker.get_sync_states() expected_resolutions = { 'german.txt': Resolution("UPDATE", local_client, s3_client, 'german.txt', 6000) } assert resolutions == expected_resolutions assert unhandled_events == {}
def test_updated_but_different_remote_timestamp(self, local_client, s3_client): utils.set_local_index(local_client, { 'biology.txt': { 'local_timestamp': 4000, 'remote_timestamp': 3000, } }) utils.set_s3_index(s3_client, { 'biology.txt': { 'local_timestamp': 6000, 'remote_timestamp': 6000, } }) utils.set_local_contents(local_client, 'biology.txt', timestamp=4500) utils.set_s3_contents(s3_client, 'biology.txt', timestamp=6000) worker = sync.SyncWorker(local_client, s3_client) resolutions, unhandled_events = worker.get_sync_states() expected_unhandled_events = { 'biology.txt': ( SyncState(SyncState.UPDATED, 4500, 3000), SyncState(SyncState.NOCHANGES, 6000, 6000) ) } assert resolutions == {} assert unhandled_events == expected_unhandled_events
def test_ignore_files(self, local_client): utils.set_local_contents(local_client, ".syncignore", timestamp=3200, data=("*.zip\n" "foo*\n")) local_client.reload_ignore_files() utils.set_local_index( local_client, {"pony.tar": { "local_timestamp": 4000, "remote_timestamp": 3000 }}, ) utils.set_local_contents(local_client, "test.zip") utils.set_local_contents(local_client, "foobar") utils.set_local_contents(local_client, "foo") utils.set_local_contents(local_client, "pony.tar", timestamp=8000) local_client.update_index() expected_index = { ".syncignore": { "local_timestamp": 3200, "remote_timestamp": None }, "pony.tar": { "local_timestamp": 8000, "remote_timestamp": 3000 }, } assert local_client.index == expected_index
def test_ignore_files(self, local_client): utils.set_local_contents(local_client, '.syncignore', timestamp=3200, data=('*.zip\n' 'foo*\n')) local_client.reload_ignore_files() utils.set_local_index(local_client, { 'pony.tar': { 'local_timestamp': 4000, 'remote_timestamp': 3000, } }) utils.set_local_contents(local_client, 'test.zip') utils.set_local_contents(local_client, 'foobar') utils.set_local_contents(local_client, 'foo') utils.set_local_contents(local_client, 'pony.tar', timestamp=8000) local_client.update_index() expected_index = { '.syncignore': { 'local_timestamp': 3200, 'remote_timestamp': None, }, 'pony.tar': { 'local_timestamp': 8000, 'remote_timestamp': 3000, } } assert local_client.index == expected_index
def test_previously_deleted_now_created(self, local_client, s3_client): utils.set_s3_index(s3_client, { 'foo': { 'local_timestamp': None, 'remote_timestamp': 4000, } }) utils.set_local_index(local_client, { 'foo': { 'local_timstamp': None, 'remote_timestamp': 4000, } }) utils.set_local_contents(local_client, 'foo', timestamp=7000) # Will create previously deleted file worker = sync.SyncWorker(local_client, s3_client) worker.sync() clients = [local_client, s3_client] assert_local_keys(clients, ['foo']) assert_remote_timestamp(clients, 'foo', 7000) # delete the file again and check that it is successful utils.delete_local(local_client, 'foo') worker.sync() assert_local_keys(clients, []) assert_remote_timestamp(clients, 'foo', 7000)
def test_set_index_local_timestamp(self, local_client): utils.set_local_index(local_client, { 'foo': { 'local_timestamp': 100000, 'remote_timestamp': 2222222222, } }) local_client.set_index_local_timestamp('foo', 123456) assert local_client.get_index_local_timestamp('foo') == 123456
def test_correct_output_nonempty(self, s3_client, local_client, capsys): config = { 'targets': { 'foo': { 'local_folder': local_client.get_uri(), 's3_uri': s3_client.get_uri(), 'aws_access_key_id': '', 'aws_secret_access_key': '', 'region_name': 'eu-west-2', } } } set_s3_index( s3_client, { 'milk': { 'local_timestamp': get_timestamp(1989, 10, 23, 11, 30) }, 'honey': { 'local_timestamp': get_timestamp(2016, 12, 12, 8, 30) }, 'ginger': { 'local_timestamp': None, } }) set_local_index( local_client, { 'milk': { 'local_timestamp': get_timestamp(2016, 12, 12, 8, 30) }, 'honey': { 'local_timestamp': get_timestamp(2016, 11, 10, 18, 40) }, 'lemon': { 'local_timestamp': get_timestamp(2017, 2, 2, 8, 30) }, 'ginger': { 'local_timestamp': None, } }) args = argparse.Namespace( target='foo', sort_by='key', show_all=False, descending=False, ) command = LsCommand(args, config, create_logger()) command.run() out, err = capsys.readouterr() assert err == "" assert out == ('key local s3\n' '----- ------------------- -------------------\n' 'honey 2016-11-10 18:40:00 2016-12-12 08:30:00\n' 'lemon 2017-02-02 08:30:00\n' 'milk 2016-12-12 08:30:00 1989-10-23 11:30:00\n')
def test_get_local_keys(self, local_client): utils.set_local_index(local_client, {}) # .index file should not come up in results utils.set_local_contents(local_client, '.bashrc') utils.set_local_contents(local_client, 'foo') utils.set_local_contents(local_client, 'bar') actual_output = local_client.get_local_keys() expected_output = ['foo', 'bar', '.bashrc'] assert sorted(expected_output) == sorted(actual_output)
def test_get_local_keys(self, local_client): utils.set_local_index(local_client, {}) # .index file should not come up in results utils.set_local_contents(local_client, ".bashrc") utils.set_local_contents(local_client, "foo") utils.set_local_contents(local_client, "bar") actual_output = local_client.get_local_keys() expected_output = ["foo", "bar", ".bashrc"] assert sorted(expected_output) == sorted(actual_output)
def test_deleted_doesnotexist(self, local_client, s3_client): utils.set_local_index(local_client, { 'physics.txt': { 'local_timestamp': 5000, 'remote_timestamp': 4550, } }) worker = sync.SyncWorker(local_client, s3_client) resolutions, unhandled_events = worker.get_sync_states() assert resolutions == {} assert unhandled_events == {}
def test_set_index_local_timestamp(self, local_client): utils.set_local_index( local_client, { "foo": { "local_timestamp": 100000, "remote_timestamp": 2222222222 } }, ) local_client.set_index_local_timestamp("foo", 123456) assert local_client.get_index_local_timestamp("foo") == 123456
def test_load_index(self, local_client): data = { "foo": { "local_timestamp": 4000, "remote_timestamp": 4000 }, "bar/baz.txt": { "local_timestamp": 5000, "remote_timestamp": 5000 }, } utils.set_local_index(local_client, data) assert local_client.index == data
def test_load_index(self, local_client): data = { 'foo': { 'local_timestamp': 4000, 'remote_timestamp': 4000, }, 'bar/baz.txt': { 'local_timestamp': 5000, 'remote_timestamp': 5000, }, } utils.set_local_index(local_client, data) assert local_client.index == data
def test_show_all(self, s3_client, local_client, capsys): config = { "targets": { "foo": { "local_folder": local_client.get_uri(), "s3_uri": s3_client.get_uri(), "aws_access_key_id": "", "aws_secret_access_key": "", "region_name": "eu-west-2", } } } set_s3_index( s3_client, { "cheese": { "local_timestamp": get_timestamp(2017, 12, 12, 8, 30) }, "crackers": { "local_timestamp": None }, }, ) set_local_index( local_client, { "cheese": { "local_timestamp": get_timestamp(2017, 2, 2, 8, 30) }, "crackers": { "local_timestamp": None }, }, ) args = argparse.Namespace(target="foo", sort_by="key", show_all=True, descending=False) command = LsCommand(args, config, create_logger()) command.run() out, err = capsys.readouterr() assert err == "" assert out == ("key local s3\n" "-------- ------------------- -------------------\n" "cheese 2017-02-02 08:30:00 2017-12-12 08:30:00\n" "crackers <deleted>\n" "Total Size: 0.00Mb\n")
def test_put_existing(self, local_client): utils.set_local_index(local_client, {"doge.txt": { "local_timestamp": 1111111 }}) data = b"canis lupus familiaris" local_client.put(key="doge.txt", sync_object=SyncObject(io.BytesIO(data), len(data), 20000)) assert local_client.index["doge.txt"]["remote_timestamp"] == 20000 assert local_client.index["doge.txt"]["local_timestamp"] == 1111111 assert utils.get_local_contents(local_client, "doge.txt") == data
def test_put_existing(self, local_client): utils.set_local_index(local_client, {'doge.txt': { 'local_timestamp': 1111111 }}) data = b'canis lupus familiaris' local_client.put(key='doge.txt', sync_object=SyncObject(io.BytesIO(data), len(data), 20000)) assert local_client.index['doge.txt']['remote_timestamp'] == 20000 assert local_client.index['doge.txt']['local_timestamp'] == 1111111 assert utils.get_local_contents(local_client, 'doge.txt') == data
def test_get_index_keys(self, local_client): data = { 'foo': { 'local_timestamp': 4000, 'remote_timestamp': 4000, }, 'bar/baz.txt': { 'local_timestamp': 5000, 'remote_timestamp': 5000, }, } utils.set_local_index(local_client, data) actual_output = local_client.get_index_keys() expected_output = ['foo', 'bar/baz.txt'] assert sorted(list(actual_output)) == sorted(expected_output)
def test_get_all_remote_timestamps(self, local_client): utils.set_local_index( local_client, { "frap": { "remote_timestamp": 4000 }, "brap": { "remote_timestamp": 9999999 } }, ) expected_output = {"frap": 4000, "brap": 9999999} actual_output = local_client.get_all_remote_timestamps() assert actual_output == expected_output
def test_get_index_keys(self, local_client): data = { "foo": { "local_timestamp": 4000, "remote_timestamp": 4000 }, "bar/baz.txt": { "local_timestamp": 5000, "remote_timestamp": 5000 }, } utils.set_local_index(local_client, data) actual_output = local_client.get_index_keys() expected_output = ["foo", "bar/baz.txt"] assert sorted(list(actual_output)) == sorted(expected_output)
def test_show_all(self, s3_client, local_client, capsys): config = { 'targets': { 'foo': { 'local_folder': local_client.get_uri(), 's3_uri': s3_client.get_uri(), 'aws_access_key_id': '', 'aws_secret_access_key': '', 'region_name': 'eu-west-2', } } } set_s3_index( s3_client, { 'cheese': { 'local_timestamp': get_timestamp(2017, 12, 12, 8, 30) }, 'crackers': { 'local_timestamp': None, } }) set_local_index( local_client, { 'cheese': { 'local_timestamp': get_timestamp(2017, 2, 2, 8, 30) }, 'crackers': { 'local_timestamp': None, } }) args = argparse.Namespace( target='foo', sort_by='key', show_all=True, descending=False, ) command = LsCommand(args, config, create_logger()) command.run() out, err = capsys.readouterr() assert err == "" assert out == ('key local s3\n' '-------- ------------------- -------------------\n' 'cheese 2017-02-02 08:30:00 2017-12-12 08:30:00\n' 'crackers <deleted>\n')
def test_get_all_remote_timestamps(self, local_client): utils.set_local_index( local_client, { 'frap': { 'remote_timestamp': 4000, }, 'brap': { 'remote_timestamp': 9999999, } }) expected_output = { 'frap': 4000, 'brap': 9999999, } actual_output = local_client.get_all_remote_timestamps() assert actual_output == expected_output
def test_get_index_timestamps(self, local_client): utils.set_local_index( local_client, { 'foo': { 'local_timestamp': 4000, 'remote_timestamp': 32000, }, 'bar': { 'local_timestamp': 9999999, } }) assert local_client.get_index_local_timestamp('foo') == 4000 assert local_client.get_index_local_timestamp('dontexist') is None assert local_client.get_index_local_timestamp('bar') == 9999999 assert local_client.get_remote_timestamp('foo') == 32000 assert local_client.get_remote_timestamp('dontexist') is None assert local_client.get_remote_timestamp('bar') is None
def test_all_keys(self, local_client): utils.set_local_contents(local_client, '.index') utils.set_local_contents(local_client, 'foo') utils.set_local_contents(local_client, 'bar/boo.md') data = { 'foo': { 'local_timestamp': 4000, 'remote_timestamp': 4000, }, 'bar/baz.txt': { 'local_timestamp': 5000, 'remote_timestamp': 5000, }, } utils.set_local_index(local_client, data) actual_output = local_client.get_all_keys() expected_output = ['foo', 'bar/boo.md', 'bar/baz.txt'] assert sorted(actual_output) == sorted(expected_output)
def test_all_keys(self, local_client): utils.set_local_contents(local_client, ".index") utils.set_local_contents(local_client, "foo") utils.set_local_contents(local_client, "bar/boo.md") data = { "foo": { "local_timestamp": 4000, "remote_timestamp": 4000 }, "bar/baz.txt": { "local_timestamp": 5000, "remote_timestamp": 5000 }, } utils.set_local_index(local_client, data) actual_output = local_client.get_all_keys() expected_output = ["foo", "bar/boo.md", "bar/baz.txt"] assert sorted(actual_output) == sorted(expected_output)
def test_get_index_timestamps(self, local_client): utils.set_local_index( local_client, { "foo": { "local_timestamp": 4000, "remote_timestamp": 32000 }, "bar": { "local_timestamp": 9999999 }, }, ) assert local_client.get_index_local_timestamp("foo") == 4000 assert local_client.get_index_local_timestamp("dontexist") is None assert local_client.get_index_local_timestamp("bar") == 9999999 assert local_client.get_remote_timestamp("foo") == 32000 assert local_client.get_remote_timestamp("dontexist") is None assert local_client.get_remote_timestamp("bar") is None
def test_interrupted_flush_index(self, local_client): target_index = { "red": { "local_timestamp": 4000, "remote_timestamp": 4000 } } utils.set_local_index(local_client, target_index) local_client.index = {"invalid_data": []} with mock.patch("json.dump") as json_dump: json_dump.side_effect = ValueError("something went wrong") with pytest.raises(ValueError): local_client.flush_index(compressed=False) with open(local_client.index_path(), "rt") as fp: index = json.load(fp) assert index == target_index
def test_interrupted_flush_index(self, local_client): target_index = { 'red': { 'local_timestamp': 4000, 'remote_timestamp': 4000, } } utils.set_local_index(local_client, target_index) local_client.index = {'invalid_data': []} with mock.patch('json.dump') as json_dump: json_dump.side_effect = ValueError('something went wrong') with pytest.raises(ValueError): local_client.flush_index(compressed=False) with open(local_client.index_path(), 'rt') as fp: index = json.load(fp) assert index == target_index
def test_update_index_non_empty(self, local_client): utils.set_local_contents(local_client, "foo", 13371337) utils.set_local_contents(local_client, "bar", 50032003) utils.set_local_index( local_client, { "foo": { "local_timestamp": 4000, "remote_timestamp": 4000 }, "bar": { "local_timestamp": 5000, "remote_timestamp": 5000 }, "baz": { "local_timestamp": 5000, "remote_timestamp": 5000 }, }, ) local_client.update_index() expected_output = { "foo": { "local_timestamp": 13371337, "remote_timestamp": 4000 }, "bar": { "local_timestamp": 50032003, "remote_timestamp": 5000 }, "baz": { "local_timestamp": None, "remote_timestamp": 5000 }, } assert local_client.index == expected_output