def test_get_changes_since(self):
make_data_dir()
file_put_contents(cpjoin(DATA_DIR, 'test 1'), 'test')
file_put_contents(cpjoin(DATA_DIR, 'test 2'), 'test 1')
file_put_contents(cpjoin(DATA_DIR, 'test 3'), 'test 2')
#==================
data_store = versioned_storage(DATA_DIR)
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 1'), {'path' : '/test/path'})
id1 = data_store.commit('test msg', 'test user')
changes = data_store.get_changes_since('root', data_store.get_head())
self.assertEqual(changes, {u'/test/path': {u'hash': u'9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08',
u'path': u'/test/path',
u'status': u'new'}})
#==================
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 2'), {'path' : '/another/path'})
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 3'), {'path' : '/yet/another/path'})
data_store.commit('test msg', 'test user')
changes = data_store.get_changes_since(id1, data_store.get_head())
self.assertEqual(changes, {u'/another/path': {u'hash': u'f67213b122a5d442d2b93bda8cc45c564a70ec5d2a4e0e95bb585cf199869c98',
u'path': u'/another/path',
u'status': u'new'},
u'/yet/another/path': {u'hash': u'dec2e4bc4992314a9c9a51bbd859e1b081b74178818c53c19d18d6f761f5d804',
u'path': u'/yet/another/path',
u'status': u'new'}})
delete_data_dir()
def test_hash_file(self):
""" Test that file hash returns the correct result. """
make_data_dir()
file_path = cpjoin(DATA_DIR, 'test')
file_put_contents(file_path, 'some file contents')
p1 = subprocess.Popen (['sha256sum', file_path], stdout=subprocess.PIPE)
result1= p1.communicate()[0].split(' ')[0]
result2 = hash_file(file_path)
self.assertEqual(result1, result2, msg = 'Hashes are not the same')
delete_data_dir()
def test_storage_delete_rollback(self):
""" Test file delete rolls back correctly """
make_data_dir()
s = storage(DATA_DIR, CONF_DIR)
s.begin()
s.file_put_contents('hello', 'test content')
s.commit(True)
s.delete_file('hello')
s.rollback()
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, 'hello')),
msg = 'error, file "hello" does not exist, delete rollback failed')
delete_data_dir()
def test_storage_move_rollback(self):
""" Test file move rolls back correctly """
make_data_dir()
s = storage(DATA_DIR, CONF_DIR)
s.begin()
s.file_put_contents('hello', 'test content')
s.commit(True)
s.move_file('hello', 'hello2')
s.rollback()
self.assertFalse( os.path.isfile(cpjoin(DATA_DIR, 'hello2')),
msg = 'File "hello2" still exists, move rollback failed')
delete_data_dir()
def test_hash_file(self):
""" Test that file hash returns the correct result. """
make_data_dir()
file_path = cpjoin(DATA_DIR, 'test')
file_put_contents(file_path, b'some file contents')
expected_result = 'cf57fcf9d6d7fb8fd7d8c30527c8f51026aa1d99ad77cc769dd0c757d4fe8667'
result = hash_file(file_path)
self.assertEqual(expected_result,
result,
msg='Hashes are not the same')
delete_data_dir()
def test_storage_put_rollback(self):
""" Test that file put rolls back correctly """
make_data_dir()
s = storage(DATA_DIR, CONF_DIR)
s.begin()
s.file_put_contents('hello', 'test content')
s.rollback()
self.assertFalse( os.path.isfile(cpjoin(DATA_DIR, 'hello')),
msg = 'File "hello" still exists, put rollback failed')
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, CONF_DIR, BACKUP_DIR, '1_hello')),
msg = 'Backup file "1_hello" does not exist, put rollback failed')
delete_data_dir()
def test_storage_move_overwrite_rollback(self):
""" Test file move rolls back correctly when move overwrites another file """
make_data_dir()
s = storage(DATA_DIR, CONF_DIR)
s.begin()
s.file_put_contents('hello', 'test content')
s.file_put_contents('hello2', 'test content 2')
s.commit(True)
s.move_file('hello', 'hello2')
s.rollback()
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, 'hello')),
msg = 'File "hello" does not exist, move overwrite rollback failed')
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, 'hello2')),
msg = 'File "hello2" does not exist, move overwrite rollback failed')
delete_data_dir()
def test_rollback(self):
make_data_dir()
file_put_contents(cpjoin(DATA_DIR, 'test 1'), 'test')
file_put_contents(cpjoin(DATA_DIR, 'test 2'), 'test')
file_put_contents(cpjoin(DATA_DIR, 'test 3'), 'test 2')
#==================
data_store = versioned_storage(DATA_DIR)
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 1'), {'path' : '/test/path'})
data_store.commit('test msg', 'test user')
data_store.begin()
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 2'), {'path' : '/another/path'})
data_store.fs_put_from_file(cpjoin(DATA_DIR, 'test 3'), {'path' : '/yet/another/path'})
data_store.rollback()
self.assertEqual(os.listdir(cpjoin(DATA_DIR, 'files')), ['9f'])
self.assertEqual(os.listdir(cpjoin(DATA_DIR, 'files', '9f')), ['86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08'])
delete_data_dir()
def test_storage_multiple_rollback(self):
""" Test rollback of multiple things at once """
make_data_dir()
s = storage(DATA_DIR, CONF_DIR)
s.begin()
s.file_put_contents('hello', 'test content')
s.commit(True)
s.file_put_contents('hello2', 'test content 2')
s.file_put_contents('hello3', 'test content 3')
s.move_file('hello', 'goodbye')
s.move_file('hello2', 'hello3')
s.delete_file('hello3')
s.file_put_contents('hello3', 'something else')
s.rollback()
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, 'hello')),
msg = 'File "hello" does not exist, multiple rollback failed')
self.assertFalse( os.path.isfile(cpjoin(DATA_DIR, 'hello3')),
msg = 'File "hello3" still exists, multiple rollback failed')
self.assertFalse( os.path.isfile(cpjoin(DATA_DIR, 'goodbye')),
msg = 'File "goodbye" still exists, multiple rollback failed')
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, CONF_DIR, BACKUP_DIR, '1_hello3')),
msg = 'Backup file "1_hello3" does not exist, multiple rollback failed')
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, CONF_DIR, BACKUP_DIR, '2_hello3')),
msg = 'Backup file "2_hello3" does not exist, multiple rollback failed')
self.assertTrue( os.path.isfile(cpjoin(DATA_DIR, CONF_DIR, BACKUP_DIR, '3_hello2')),
msg = 'Backup file "3_hello2" does not exist, multiple rollback failed')
delete_data_dir()
def tearDown(self):
delete_data_dir()
def setUp(self):
delete_data_dir() # Ensure clean start
make_data_dir()
def test_system(self):
test_content_1 = b'test file jhgrtelkj'
test_content_2 = b''.join([struct.pack('B', i) for i in range(256)
]) # binary string with all byte values
test_content_2_2 = test_content_2[::-1]
test_content_3 = b'test content 3 sdavcxreiltlj'
test_content_4 = b'test content 4 fsdwqtruytuyt'
test_content_5 = b'test content 5 .,myuitouys'
#=========
setup()
setup_client('client1')
#==================================================
# test_initial commit
#==================================================
file_put_contents(DATA_DIR + 'client1/test1', test_content_1)
file_put_contents(DATA_DIR + 'client1/test2',
test_content_2) # test with a binary blob
#file_put_contents(DATA_DIR + u'client1/GȞƇØzÇ â˜¸k😒â™', test_content_2) # test unicode file name
# commit the files
session_token = client.authenticate()
print(session_token)
version_id = client.commit(session_token, 'test commit')
self.assertNotEqual(version_id, None)
# commit message should be in log
req_result = client.get_versions(session_token)[0]
self.assertEqual(
'test commit',
json.loads(req_result)['versions'][0]['commit_message'])
# file should show up in list_changes
req_result = client.get_files_in_version(session_token, version_id)[0]
self.assertTrue('/test1' in json.loads(req_result)['files'])
self.assertTrue('/test2' in json.loads(req_result)['files'])
# file should exist in server fs
self.assertEqual(
test_content_1,
file_get_contents(DATA_DIR + 'server/files/' +
get_server_file_name(test_content_1)))
self.assertEqual(
test_content_2,
file_get_contents(DATA_DIR + 'server/files/' +
get_server_file_name(test_content_2)))
# NOTE As change detection is done using access timestamps, need a
# delay between tests to make sure changes are detected correctly
time.sleep(0.5)
#==================================================
# test update
#==================================================
setup_client('client2')
session_token = client.authenticate()
client.update(session_token)
self.assertEqual(test_content_1,
file_get_contents(DATA_DIR + 'client2/test1'))
self.assertEqual(test_content_2,
file_get_contents(DATA_DIR + 'client2/test2'))
time.sleep(0.5) # See above
#==================================================
# test delete and add
#==================================================
os.unlink(DATA_DIR + 'client2/test1')
file_put_contents(DATA_DIR + 'client2/test2',
test_content_2_2) # test changing an existing file
file_put_contents(DATA_DIR + 'client2/test3', test_content_3)
file_put_contents(DATA_DIR + 'client2/test4', test_content_4)
setup_client('client2')
session_token = client.authenticate()
version_id = client.commit(session_token,
'create and delete some files')
# check change is reflected correctly in the commit log
req_result = client.get_changes_in_version(session_token,
version_id)[0]
res_index = {v['path']: v for v in json.loads(req_result)['changes']}
self.assertEqual('deleted', res_index['/test1']['status'])
self.assertEqual('new', res_index['/test2']['status'])
self.assertEqual('new', res_index['/test3']['status'])
self.assertEqual('new', res_index['/test4']['status'])
# update first repo, file should be deleted and new file added
setup_client('client1')
session_token = client.authenticate()
client.update(session_token)
# Verify changes are reflected in FS
self.assertFalse(os.path.isfile(DATA_DIR + 'client1/test1'))
self.assertEqual(test_content_2_2,
file_get_contents(DATA_DIR + 'client1/test2'))
self.assertEqual(test_content_3,
file_get_contents(DATA_DIR + 'client1/test3'))
self.assertEqual(test_content_4,
file_get_contents(DATA_DIR + 'client1/test4'))
time.sleep(0.5) # See above
#==================================================
# setup for next test
#==================================================
file_put_contents(DATA_DIR + 'client1/test1', test_content_1)
file_put_contents(DATA_DIR + 'client1/test5', test_content_1)
file_put_contents(DATA_DIR + 'client1/test6', test_content_1)
setup_client('client1')
client.commit(client.authenticate(), 'test setup')
setup_client('client2')
client.update(client.authenticate())
time.sleep(0.5) # See above
#==================================================
# test conflict resolution, both to the server
# and client version
#==================================================
# Delete on client, change on server resolution
file_put_contents(DATA_DIR + 'client1/test1', test_content_5 + b'11')
os.unlink(DATA_DIR + 'client2/test1')
file_put_contents(DATA_DIR + 'client1/test2', test_content_5 + b'00')
os.unlink(DATA_DIR + 'client2/test2')
# Delete on server, change on client resolution
os.unlink(DATA_DIR + 'client1/test5')
file_put_contents(DATA_DIR + 'client2/test5', test_content_5 + b'ff')
os.unlink(DATA_DIR + 'client1/test6')
file_put_contents(DATA_DIR + 'client2/test6', test_content_5 + b'gg')
# Double change resolution
file_put_contents(DATA_DIR + 'client1/test3', test_content_5 + b'aa')
file_put_contents(DATA_DIR + 'client2/test3', test_content_5 + b'bb')
file_put_contents(DATA_DIR + 'client1/test4', test_content_5 + b'cc')
file_put_contents(DATA_DIR + 'client2/test4', test_content_5 + b'dd')
# commit both clients second to commit should error
setup_client('client1')
session_token = client.authenticate()
version_id = client.commit(session_token,
'initial commit for conflict test')
setup_client('client2')
session_token = client.authenticate()
try:
version_id = client.commit(session_token, 'this should conflict')
self.fail()
except SystemExit:
pass
# Update should begin conflict resolution process
try:
client.update(session_token, testing=True)
self.fail()
except SystemExit:
pass
# test server versions of conflict files downloaded correctly
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test1'),
test_content_5 + b'11')
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test2'),
test_content_5 + b'00')
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test3'),
test_content_5 + b'aa')
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test4'),
test_content_5 + b'cc')
# NOTE nothing to download in delete on server case
#test resolving it
path = DATA_DIR + 'client2/.shttpfs/conflict_resolution.json'
resolve = json.loads(file_get_contents(path))
resolve_index = {v['1_path']: v for v in resolve}
resolve_index['/test1']['4_resolution'] = ['client']
resolve_index['/test2']['4_resolution'] = ['server']
resolve_index['/test3']['4_resolution'] = ['client']
resolve_index['/test4']['4_resolution'] = ['server']
resolve_index['/test5']['4_resolution'] = ['client']
resolve_index['/test6']['4_resolution'] = ['server']
file_put_contents(
path,
json.dumps([v
for v in list(resolve_index.values())]).encode('utf8'))
# perform update and test resolve as expected
client.update(session_token)
self.assertFalse(os.path.isfile(DATA_DIR + 'client2/test1'))
self.assertEqual(test_content_5 + b'00',
file_get_contents(DATA_DIR + 'client2/test2'))
self.assertEqual(test_content_5 + b'bb',
file_get_contents(DATA_DIR + 'client2/test3'))
self.assertEqual(test_content_5 + b'cc',
file_get_contents(DATA_DIR + 'client2/test4'))
self.assertEqual(test_content_5 + b'ff',
file_get_contents(DATA_DIR + 'client2/test5'))
self.assertFalse(os.path.isfile(DATA_DIR + 'client2/test6'))
# This should now commit
version_id = client.commit(session_token, 'this should be ok')
self.assertNotEqual(None, version_id)
req_result = client.get_changes_in_version(session_token,
version_id)[0]
res_index = {v['path']: v for v in json.loads(req_result)['changes']}
self.assertEqual('deleted', res_index['/test1']['status'])
self.assertTrue('/test2' not in res_index)
self.assertEqual('new', res_index['/test3']['status'])
self.assertTrue('/test4' not in res_index)
self.assertEqual('new', res_index['/test5']['status'])
self.assertTrue('/test6' not in res_index)
#==================================================
delete_data_dir()
def test_system(self):
test_content_1 = 'test file jhgrtelkj'
test_content_2 = b''.join([struct.pack('B', i) for i in range(256)]) # binary string with all byte values
test_content_2_2 = test_content_2[::-1]
test_content_3 = 'test content 3 sdavcxreiltlj'
test_content_4 = 'test content 4 fsdwqtruytuyt'
test_content_5 = 'test content 5 .,myuitouys'
#=========
setup()
setup_client('client1')
#==================================================
# test_initial commit
#==================================================
file_put_contents(DATA_DIR + 'client1/test1', test_content_1)
file_put_contents(DATA_DIR + 'client1/test2', test_content_2) # test with a binary blob
#file_put_contents(DATA_DIR + u'client1/GȞƇØzÇ â˜¸k😒â™', test_content_2) # test unicode file name
# commit the files
session_token = client.authenticate()
version_id = client.commit(session_token, 'test commit')
self.assertNotEqual(version_id, None)
# commit message should be in log
req_result = client.get_versions(session_token)[0]
self.assertEqual('test commit', json.loads(req_result)['versions'][0]['commit_message'])
# file should show up in list_changes
req_result = client.get_files_in_version(session_token, version_id)[0]
self.assertTrue('/test1' in json.loads(req_result)['files'])
self.assertTrue('/test2' in json.loads(req_result)['files'])
# file should exist in server fs
self.assertEqual(test_content_1, file_get_contents(DATA_DIR + 'server/files/' + get_server_file_name(test_content_1)))
self.assertEqual(test_content_2, file_get_contents(DATA_DIR + 'server/files/' + get_server_file_name(test_content_2)))
# NOTE As change detection is done using access timestamps, need a
# delay between tests to make sure changes are detected correctly
time.sleep(0.5)
#==================================================
# test update
#==================================================
setup_client('client2')
session_token = client.authenticate()
client.update(session_token)
self.assertEqual(test_content_1, file_get_contents(DATA_DIR + 'client2/test1'))
self.assertEqual(test_content_2, file_get_contents(DATA_DIR + 'client2/test2'))
time.sleep(0.5) # See above
#==================================================
# test delete and add
#==================================================
os.unlink(DATA_DIR + 'client2/test1')
file_put_contents(DATA_DIR + 'client2/test2', test_content_2_2) # test changing an existing file
file_put_contents(DATA_DIR + 'client2/test3', test_content_3)
file_put_contents(DATA_DIR + 'client2/test4', test_content_4)
setup_client('client2')
session_token = client.authenticate()
version_id = client.commit(session_token, 'create and delete some files')
# check change is reflected correctly in the commit log
req_result = client.get_changes_in_version(session_token, version_id)[0]
res_index = { v['path'] : v for v in json.loads(req_result)['changes']}
self.assertEqual('deleted', res_index['/test1']['status'])
self.assertEqual('new' , res_index['/test2']['status'])
self.assertEqual('new' , res_index['/test3']['status'])
self.assertEqual('new' , res_index['/test4']['status'])
# update first repo, file should be deleted and new file added
setup_client('client1')
session_token = client.authenticate()
client.update(session_token)
# Verify changes are reflected in FS
self.assertFalse(os.path.isfile(DATA_DIR + 'client1/test1'))
self.assertEqual(test_content_2_2, file_get_contents(DATA_DIR + 'client1/test2'))
self.assertEqual(test_content_3, file_get_contents(DATA_DIR + 'client1/test3'))
self.assertEqual(test_content_4, file_get_contents(DATA_DIR + 'client1/test4'))
time.sleep(0.5) # See above
#==================================================
# setup for next test
#==================================================
file_put_contents(DATA_DIR + 'client1/test1', test_content_1)
file_put_contents(DATA_DIR + 'client1/test5', test_content_1)
file_put_contents(DATA_DIR + 'client1/test6', test_content_1)
setup_client('client1')
client.commit(client.authenticate(), 'test setup')
setup_client('client2')
client.update(client.authenticate())
time.sleep(0.5) # See above
#==================================================
# test conflict resolution, both to the server
# and client version
#==================================================
# Delete on client, change on server resolution
file_put_contents(DATA_DIR + 'client1/test1', test_content_5 + '11')
os.unlink( DATA_DIR + 'client2/test1')
file_put_contents(DATA_DIR + 'client1/test2', test_content_5 + '00')
os.unlink( DATA_DIR + 'client2/test2')
# Delete on server, change on client resolution
os.unlink( DATA_DIR + 'client1/test5')
file_put_contents(DATA_DIR + 'client2/test5', test_content_5 + 'ff')
os.unlink( DATA_DIR + 'client1/test6')
file_put_contents(DATA_DIR + 'client2/test6', test_content_5 + 'gg')
# Double change resolution
file_put_contents(DATA_DIR + 'client1/test3', test_content_5 + 'aa')
file_put_contents(DATA_DIR + 'client2/test3', test_content_5 + 'bb')
file_put_contents(DATA_DIR + 'client1/test4', test_content_5 + 'cc')
file_put_contents(DATA_DIR + 'client2/test4', test_content_5 + 'dd')
# commit both clients second to commit should error
setup_client('client1')
session_token = client.authenticate()
version_id = client.commit(session_token, 'initial commit for conflict test')
setup_client('client2')
session_token = client.authenticate()
try:
version_id = client.commit(session_token, 'this should conflict')
self.fail()
except SystemExit: pass
# Update should begin conflict resolution process
try:
client.update(session_token, testing=True)
self.fail()
except SystemExit: pass
# test server versions of conflict files downloaded correctly
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test1'), test_content_5 + '11')
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test2'), test_content_5 + '00')
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test3'), test_content_5 + 'aa')
self.assertEqual(file_get_contents(DATA_DIR + 'client1/test4'), test_content_5 + 'cc')
# NOTE nothing to download in delete on server case
#test resolving it
path = DATA_DIR + 'client2/.shttpfs/conflict_resolution.json'
resolve = json.loads(file_get_contents(path))
resolve_index = {v['1_path'] : v for v in resolve}
resolve_index['/test1']['4_resolution'] = ['client']
resolve_index['/test2']['4_resolution'] = ['server']
resolve_index['/test3']['4_resolution'] = ['client']
resolve_index['/test4']['4_resolution'] = ['server']
resolve_index['/test5']['4_resolution'] = ['client']
resolve_index['/test6']['4_resolution'] = ['server']
file_put_contents(path, json.dumps([v for v in resolve_index.values()]))
# perform update and test resolve as expected
client.update(session_token)
self.assertFalse( os.path.isfile(DATA_DIR + 'client2/test1'))
self.assertEqual(test_content_5 + '00', file_get_contents(DATA_DIR + 'client2/test2'))
self.assertEqual(test_content_5 + 'bb', file_get_contents(DATA_DIR + 'client2/test3'))
self.assertEqual(test_content_5 + 'cc', file_get_contents(DATA_DIR + 'client2/test4'))
self.assertEqual(test_content_5 + 'ff', file_get_contents(DATA_DIR + 'client2/test5'))
self.assertFalse( os.path.isfile(DATA_DIR + 'client2/test6'))
# This should now commit
version_id = client.commit(session_token, 'this should be ok')
self.assertNotEqual(None, version_id)
req_result = client.get_changes_in_version(session_token, version_id)[0]
res_index = { v['path'] : v for v in json.loads(req_result)['changes']}
self.assertEqual('deleted', res_index['/test1']['status'])
self.assertTrue('/test2' not in res_index)
self.assertEqual('new', res_index['/test3']['status'])
self.assertTrue('/test4' not in res_index)
self.assertEqual('new', res_index['/test5']['status'])
self.assertTrue('/test6' not in res_index)
#==================================================
delete_data_dir()