def do_sequential(start_first):
"""
test that we can bring up nodes sequentially and they do the right thing (e.g. when a 2nd node comes up
it doesn't signal a file delete just because it doesn't have the file to start with).
"""
nodes = ['a', 'b']
# so we can run this multiple times
clean(get_sequential_root())
log_folder = os.path.join(get_sequential_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_sequential_root(), key) for node in nodes]
latus_folders = []
latus_file = get_file_name(nodes[0]) # only one file for this test
latus_paths = []
for node in nodes:
folder = get_latus_folder(get_sequential_root(), node)
latus_folders.append(folder)
path = os.path.join(folder, latus_file)
latus_paths.append(path)
if start_first:
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(3)
[sync.request_exit() for sync in syncs]
time.sleep(3)
write_to_file(latus_folders[0], latus_file, nodes[0])
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
syncs[0].start() # just start 'a'
# check we have the proper files
assert(os.path.exists(latus_paths[0]))
assert(not os.path.exists(latus_paths[1]))
syncs[1].start() # start 'b'
wait_for_file(latus_paths[1])
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(os.path.exists(latus_paths[0]))
assert(os.path.exists(latus_paths[1]))
latus.logger.log.info('test_simple exiting')
return
def test_delete(setup):
"""
test a simple delete across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_delete_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.DEBUG)
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_delete_root(), key) for node in nodes]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_delete_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
file_name = nodes[0] + '.txt'
path_node_0 = write_to_file(get_latus_folder(get_delete_root(), nodes[0]), file_name, nodes[0])
path_node_1 = os.path.join(get_latus_folder(get_delete_root(), nodes[1]), file_name)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(2)
# wait for the file to get sync'd
wait_for_file(path_node_1)
wait_for_file(path_node_0)
time.sleep(2)
assert(os.path.exists(path_node_0))
assert(os.path.exists(path_node_1))
# now remove the file on the node that it was sync'd to
os.remove(path_node_1)
time.sleep(5) # todo: determine why these delays are needed on Windows
# wait for the file to be removed from both nodes
wait_for_file(path_node_0, False)
wait_for_file(path_node_1, False)
# ok .. should be done ... exit
[sync.request_exit() for sync in syncs]
# make sure it worked OK
assert(not os.path.exists(path_node_0))
assert(not os.path.exists(path_node_1))
latus.logger.log.info('test_delete exiting')
return
def test_move(setup):
"""
test a simple delete across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_move_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_move_root(), key) for node in nodes]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_move_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
file_name = 'src.txt'
path_node_0 = write_to_file(get_latus_folder(get_move_root(), nodes[0]), file_name, nodes[0])
path_node_1 = os.path.join(get_latus_folder(get_move_root(), nodes[1]), file_name)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# wait for the file to get sync'd
wait_for_file(path_node_1)
wait_for_file(path_node_0)
# move to a new file name
new_file_name = 'dest.txt'
new_path_node_0 = os.path.join(get_latus_folder(get_move_root(), nodes[0]), new_file_name)
new_path_node_1 = os.path.join(get_latus_folder(get_move_root(), nodes[1]), new_file_name)
latus.logger.log.info('moving %s to %s' % (path_node_0, new_path_node_0))
shutil.move(path_node_0, new_path_node_0)
# wait for the file to be moved on both notes
wait_for_file(path_node_0, False)
wait_for_file(path_node_1, False)
wait_for_file(new_path_node_0)
wait_for_file(new_path_node_1)
# ok .. should be done ... exit
[sync.request_exit() for sync in syncs]
# make sure it worked OK
assert(not os.path.exists(path_node_0))
assert(not os.path.exists(path_node_1))
assert(os.path.exists(new_path_node_0))
assert(os.path.exists(new_path_node_1))
latus.logger.log.info('test_move exiting')
return
def main():
"""
main test function
"""
number_of_nodes = 3
parser = argparse.ArgumentParser(description='latus system test')
parser.add_argument('-n',
'--no_sync',
action='store_true',
default=False,
help="don't do sync (for testing)")
args = parser.parse_args()
# set up test folder and logging
test_root = os.path.join('temp', 'systst')
try:
shutil.rmtree(test_root)
except FileNotFoundError:
pass
# backup_count=0 so we don't have log file rollover (the code that looks into the logs can't handle log file rollover)
latus.logger.init(os.path.join(
appdirs.user_log_dir(latus.const.NAME, latus.const.COMPANY), 'systst'),
delete_existing_log_files=True,
backup_count=0)
latus.logger.set_console_log_level(logging.WARN)
latus.logger.set_file_log_level(logging.DEBUG)
latus.logger.log.info('starting %s' % __name__)
time.sleep(1)
# set up the preferences for the nodes we'll run
app_data_folders = setup_preferences(test_root, number_of_nodes)
latus_folders = []
for app_data_folder in app_data_folders:
pref = latus.preferences.Preferences(app_data_folder)
latus_folders.append(pref.get_latus_folder())
# create the nodes and start them
syncs = []
if not args.no_sync:
# start the sync
syncs = [
SyncProc(app_data_folder) for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
# do the tests (and provide the user with a "Stop" button)
files_test = FilesTest(app_data_folders, latus_folders, syncs)
files_test.start()
user_interface()
files_test.request_exit()
files_test.join()
if not args.no_sync:
[sync.request_exit() for sync in syncs]
def test_move_out_of_latus_root(setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_move_out_of_latus_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_move_out_of_latus_root(), key) for node in nodes]
# get list of folders and files, and write to them
local_folders = []
for node in nodes:
latus_folder = get_latus_folder(get_move_out_of_latus_root(), node)
local_folders.append(latus_folder)
file_name = 'a.txt'
write_to_file(local_folders[0], file_name, 'move_out_of_latus', '')
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name))
temp_folder = os.path.join('temp', 'move_out_of_latus')
os.makedirs(temp_folder, mode=latus.const.MAKE_DIRS_MODE, exist_ok=True)
temp_path = os.path.join(temp_folder, file_name)
# make sure there's not already something in the dest (e.g. from another test)
try:
os.remove(temp_path)
except FileNotFoundError:
pass
shutil.move(os.path.join(local_folders[0], file_name), temp_path)
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name), False)
# stop the syncs
[sync.request_exit() for sync in syncs]
time.sleep(2)
# check the results
for local_folder in local_folders:
assert(not os.path.exists(os.path.join(local_folder, file_name)))
latus.logger.log.info('test_move_out_of_latus_root exiting')
return
def test_simple(setup):
"""
test a simple sync of 2 files across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_simple_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_simple_root(), key) for node in nodes]
# get list of folders and files, and write to them
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_simple_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node, '')
time.sleep(1)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# wait for files to sync
b_to_a = os.path.join(local_folders[0], file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], file_names[0])
wait_for_file(a_to_b)
time.sleep(1)
latus.logger.log.info('wait for any pending (but will be filtered) watchdog events')
time.sleep(3)
latus.logger.log.info('done wait for any pending (but will be filtered) watchdog events')
time.sleep(1)
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(os.path.exists(b_to_a))
assert(os.path.exists(a_to_b))
latus.logger.log.info('test_simple exiting')
return
def main():
"""
main test function
"""
number_of_nodes = 3
parser = argparse.ArgumentParser(description='latus system test')
parser.add_argument('-n', '--no_sync', action='store_true', default=False, help="don't do sync (for testing)")
args = parser.parse_args()
# set up test folder and logging
test_root = os.path.join('temp', 'systst')
try:
shutil.rmtree(test_root)
except FileNotFoundError:
pass
# backup_count=0 so we don't have log file rollover (the code that looks into the logs can't handle log file rollover)
latus.logger.init(os.path.join(appdirs.user_log_dir(latus.const.NAME, latus.const.COMPANY), 'systst'),
delete_existing_log_files=True, backup_count=0)
latus.logger.set_console_log_level(logging.WARN)
latus.logger.set_file_log_level(logging.DEBUG)
latus.logger.log.info('starting %s' % __name__)
time.sleep(1)
# set up the preferences for the nodes we'll run
app_data_folders = setup_preferences(test_root, number_of_nodes)
latus_folders = []
for app_data_folder in app_data_folders:
pref = latus.preferences.Preferences(app_data_folder)
latus_folders.append(pref.get_latus_folder())
# create the nodes and start them
syncs = []
if not args.no_sync:
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# do the tests (and provide the user with a "Stop" button)
files_test = FilesTest(app_data_folders, latus_folders, syncs)
files_test.start()
user_interface()
files_test.request_exit()
files_test.join()
if not args.no_sync:
[sync.request_exit() for sync in syncs]
def test_start_first(setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_start_first_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_start_first_root(), key) for node in nodes]
# start the sync BEFORE we have any files
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(2)
# get list of folders and files, and write to them
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_start_first_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node)
# wait for files to sync
b_to_a = os.path.join(local_folders[0], file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], file_names[0])
wait_for_file(a_to_b)
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(os.path.exists(b_to_a))
assert(os.path.exists(a_to_b))
latus.logger.log.info('test_start_first exiting')
return
def test_move_into_latus_root(setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_move_into_latus_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_move_into_latus_root(), key) for node in nodes]
# get list of folders and files, and write to them
local_folders = []
for node in nodes:
latus_folder = get_latus_folder(get_move_into_latus_root(), node)
local_folders.append(latus_folder)
temp_folder = os.path.join('temp', 'move_to_latus')
file_name = 'a.txt'
write_to_file(temp_folder, file_name, 'move_to_latus', '')
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(3) # wait for syncs to come up
shutil.move(os.path.join(temp_folder, file_name), os.path.join(local_folders[0], file_name))
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name))
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
for local_folder in local_folders:
assert(os.path.exists(os.path.join(local_folder, file_name)))
latus.logger.log.info('test_move_into_latus_root exiting')
return
def test_subdir(setup):
"""
test a subdir sync of 2 files across 2 nodes
"""
nodes = ['a', 'b']
subdir_name = 'mysubdir'
log_folder = os.path.join(get_subdir_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_subdir_root(), key) for node in nodes]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_subdir_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node, subdir_name)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
b_to_a = os.path.join(local_folders[0], subdir_name, file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], subdir_name, file_names[0])
wait_for_file(a_to_b)
[sync.request_exit() for sync in syncs]
assert(os.path.exists(b_to_a))
assert(os.path.exists(a_to_b))
latus.logger.log.info('test_subdir exiting')
return
def test_create_modify_delete(setup):
"""
test that we can bring up nodes sequentially and they do the right thing (e.g. when a 2nd node comes up
it doesn't signal a file delete just because it doesn't have the file to start with).
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_create_modify_delete_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_create_modify_delete_root(), key) for node in nodes]
latus_folders = []
latus_file = get_file_name(nodes[0]) # only one file for this test
latus_paths = []
for node in nodes:
folder = get_latus_folder(get_create_modify_delete_root(), node)
latus_folders.append(folder)
latus_paths.append(os.path.join(folder, latus_file))
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(1)
write_to_file(latus_folders[0], latus_file, 'abc')
[wait_for_file(p) for p in latus_paths]
# check we have the proper files
assert(os.path.exists(latus_paths[0]))
assert(os.path.exists(latus_paths[1]))
# append so it's a modify
write_to_file(latus_folders[0], latus_file, 'def', mode='a')
# wait for append to propagate
while os.path.getsize(os.path.join(latus_folders[1], latus_file)) != 6:
time.sleep(1)
assert(os.path.getsize(os.path.join(latus_folders[1], latus_file)) == 6)
os.remove(latus_paths[0])
# wait for delete to propagate
[wait_for_file(p, False) for p in latus_paths]
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(not os.path.exists(latus_paths[0]))
assert(not os.path.exists(latus_paths[1]))
latus.logger.log.info('test_create_modify_delete exiting')
return