def test_delete(setup):
"""
test a simple delete across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_delete_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.DEBUG)
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_delete_root(), key) for node in nodes]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_delete_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
file_name = nodes[0] + '.txt'
path_node_0 = write_to_file(get_latus_folder(get_delete_root(), nodes[0]), file_name, nodes[0])
path_node_1 = os.path.join(get_latus_folder(get_delete_root(), nodes[1]), file_name)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(2)
# wait for the file to get sync'd
wait_for_file(path_node_1)
wait_for_file(path_node_0)
time.sleep(2)
assert(os.path.exists(path_node_0))
assert(os.path.exists(path_node_1))
# now remove the file on the node that it was sync'd to
os.remove(path_node_1)
time.sleep(5) # todo: determine why these delays are needed on Windows
# wait for the file to be removed from both nodes
wait_for_file(path_node_0, False)
wait_for_file(path_node_1, False)
# ok .. should be done ... exit
[sync.request_exit() for sync in syncs]
# make sure it worked OK
assert(not os.path.exists(path_node_0))
assert(not os.path.exists(path_node_1))
latus.logger.log.info('test_delete exiting')
return
def writer(node_id, label):
log_folder = os.path.join(get_node_db_retries_root(), 'log')
logger_init(log_folder)
latus.logger.log.info('entering writer: %s' % node_id)
db_writer = nodedb.NodeDB(get_node_db_retries_root(), node_id, True)
while True:
db_writer.set_heartbeat()
def test_log_upload(session_setup, module_setup):
logger_init(os.path.join(get_data_root(), 'log'))
app_data_folder = write_preferences('testnode', get_test_upload_root(),
'test_key')
latus.logger.set_appdata_folder(app_data_folder)
latus.logger.set_console_log_level(logging.INFO)
latus.logger.log.error('test_upload_error_message')
def writer(node_id, label):
log_folder = os.path.join(get_node_db_retries_root(), 'log')
logger_init(log_folder)
latus.logger.log.info('entering writer: %s' % node_id)
db_writer = latus.nodedb.NodeDB(get_node_db_retries_root(), node_id, True)
while True:
db_writer.set_heartbeat()
def test_move(setup):
"""
test a simple delete across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_move_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_move_root(), key) for node in nodes]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_move_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
file_name = 'src.txt'
path_node_0 = write_to_file(get_latus_folder(get_move_root(), nodes[0]), file_name, nodes[0])
path_node_1 = os.path.join(get_latus_folder(get_move_root(), nodes[1]), file_name)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# wait for the file to get sync'd
wait_for_file(path_node_1)
wait_for_file(path_node_0)
# move to a new file name
new_file_name = 'dest.txt'
new_path_node_0 = os.path.join(get_latus_folder(get_move_root(), nodes[0]), new_file_name)
new_path_node_1 = os.path.join(get_latus_folder(get_move_root(), nodes[1]), new_file_name)
latus.logger.log.info('moving %s to %s' % (path_node_0, new_path_node_0))
shutil.move(path_node_0, new_path_node_0)
# wait for the file to be moved on both notes
wait_for_file(path_node_0, False)
wait_for_file(path_node_1, False)
wait_for_file(new_path_node_0)
wait_for_file(new_path_node_1)
# ok .. should be done ... exit
[sync.request_exit() for sync in syncs]
# make sure it worked OK
assert(not os.path.exists(path_node_0))
assert(not os.path.exists(path_node_1))
assert(os.path.exists(new_path_node_0))
assert(os.path.exists(new_path_node_1))
latus.logger.log.info('test_move exiting')
return
def test_gui_preferences(session_setup, module_setup):
global g_coord, g_window_closed
log_folder = os.path.join(get_gui_preferences_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
if latus.util.is_mac():
# todo: get this to work on Windows (I use virtualization - that might be a problem)
# Timing of this is tricky. We have to launch the automation thread that looks for the image of the OK button
# *then* launch the preferences GUI, since the preferences GUI will block once it starts running. But since
# the preferences GUI won't be on the screen when the automation thread starts, the automation thread may
# have to search multiple times (and pause momentarily between searches to give the GUI time to come up
# if it hasn't already).
automation_thread = threading.Thread(target=automation)
automation_thread.start()
g_window_closed = False
latus.gui_preferences.main() # must be in main thread
g_window_closed = True
automation_thread.join()
#if __name__ == '__main__':
# test_gui_preferences(None, None)
def test_simple(session_setup, module_setup):
"""
test a simple sync of 2 files across 2 nodes
"""
nodes = ['a', 'b']
sleep_time = latus.const.FILTER_TIME_OUT * 2
log_folder = os.path.join(get_simple_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_simple_root(), key) for node in nodes
]
# get list of folders and files, and write to them
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_simple_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node, '')
time.sleep(sleep_time)
# start the sync
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
# wait for files to sync
b_to_a = os.path.join(local_folders[0], file_names[1])
assert (wait_for_file(b_to_a))
a_to_b = os.path.join(local_folders[1], file_names[0])
assert (wait_for_file(a_to_b))
time.sleep(sleep_time)
# stop the syncs
for sync in syncs:
assert (not sync.request_exit(1)) # make sure we exited cleanly
time.sleep(sleep_time)
# final check of the results
paths = [b_to_a, a_to_b]
for p in paths:
assert (os.path.exists(p))
latus.logger.log.info('test_simple exiting')
return
def test_move_into_latus(session_setup, module_setup):
nodes = ['a', 'b']
sleep_time = latus.const.FILTER_TIME_OUT * 2
log_folder = os.path.join(get_move_into_latus_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_move_into_latus_root(), key)
for node in nodes
]
# get list of folders and files, and write to them
local_folders = []
for node in nodes:
latus_folder = get_latus_folder(get_move_into_latus_root(), node)
os.makedirs(latus_folder, exist_ok=True)
local_folders.append(latus_folder)
file_name = 'a.txt'
src = write_to_file(os.path.join('temp', test_name), file_name, test_name)
time.sleep(sleep_time)
# start the sync
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
time.sleep(sleep_time)
shutil.move(src, os.path.join(local_folders[0], file_name))
time.sleep(sleep_time)
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name))
time.sleep(sleep_time)
# stop the syncs
[sync.request_exit() for sync in syncs]
time.sleep(sleep_time)
# check the results
for local_folder in local_folders:
assert (os.path.exists(os.path.join(local_folder, file_name)))
latus.logger.log.info('test_move_into_latus exiting')
return
def do_sequential(start_first):
"""
test that we can bring up nodes sequentially and they do the right thing (e.g. when a 2nd node comes up
it doesn't signal a file delete just because it doesn't have the file to start with).
"""
nodes = ['a', 'b']
# so we can run this multiple times
clean(get_sequential_root())
log_folder = os.path.join(get_sequential_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_sequential_root(), key) for node in nodes]
latus_folders = []
latus_file = get_file_name(nodes[0]) # only one file for this test
latus_paths = []
for node in nodes:
folder = get_latus_folder(get_sequential_root(), node)
latus_folders.append(folder)
path = os.path.join(folder, latus_file)
latus_paths.append(path)
if start_first:
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(3)
[sync.request_exit() for sync in syncs]
time.sleep(3)
write_to_file(latus_folders[0], latus_file, nodes[0])
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
syncs[0].start() # just start 'a'
# check we have the proper files
assert(os.path.exists(latus_paths[0]))
assert(not os.path.exists(latus_paths[1]))
syncs[1].start() # start 'b'
wait_for_file(latus_paths[1])
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(os.path.exists(latus_paths[0]))
assert(os.path.exists(latus_paths[1]))
latus.logger.log.info('test_simple exiting')
return
def test_start_first(session_setup, module_setup):
nodes = ['a', 'b']
sleep_time = latus.const.FILTER_TIME_OUT * 2.0
log_folder = os.path.join(get_start_first_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_start_first_root(), key) for node in nodes
]
# start the sync BEFORE we have any files
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
time.sleep(sleep_time)
# get list of folders and files, and write to them
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_start_first_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node)
time.sleep(sleep_time)
# wait for files to sync
b_to_a = os.path.join(local_folders[0], file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], file_names[0])
wait_for_file(a_to_b)
time.sleep(sleep_time)
# stop the syncs
[sync.request_exit() for sync in syncs]
time.sleep(sleep_time)
# check the results
assert (os.path.exists(b_to_a))
assert (os.path.exists(a_to_b))
latus.logger.log.info('test_start_first exiting')
return
def test_move_out_of_latus_root(setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_move_out_of_latus_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_move_out_of_latus_root(), key) for node in nodes]
# get list of folders and files, and write to them
local_folders = []
for node in nodes:
latus_folder = get_latus_folder(get_move_out_of_latus_root(), node)
local_folders.append(latus_folder)
file_name = 'a.txt'
write_to_file(local_folders[0], file_name, 'move_out_of_latus', '')
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name))
temp_folder = os.path.join('temp', 'move_out_of_latus')
os.makedirs(temp_folder, mode=latus.const.MAKE_DIRS_MODE, exist_ok=True)
temp_path = os.path.join(temp_folder, file_name)
# make sure there's not already something in the dest (e.g. from another test)
try:
os.remove(temp_path)
except FileNotFoundError:
pass
shutil.move(os.path.join(local_folders[0], file_name), temp_path)
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name), False)
# stop the syncs
[sync.request_exit() for sync in syncs]
time.sleep(2)
# check the results
for local_folder in local_folders:
assert(not os.path.exists(os.path.join(local_folder, file_name)))
latus.logger.log.info('test_move_out_of_latus_root exiting')
return
def test_simple(setup):
"""
test a simple sync of 2 files across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_simple_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_simple_root(), key) for node in nodes]
# get list of folders and files, and write to them
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_simple_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node, '')
time.sleep(1)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
# wait for files to sync
b_to_a = os.path.join(local_folders[0], file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], file_names[0])
wait_for_file(a_to_b)
time.sleep(1)
latus.logger.log.info('wait for any pending (but will be filtered) watchdog events')
time.sleep(3)
latus.logger.log.info('done wait for any pending (but will be filtered) watchdog events')
time.sleep(1)
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(os.path.exists(b_to_a))
assert(os.path.exists(a_to_b))
latus.logger.log.info('test_simple exiting')
return
def test_subdir(session_setup, module_setup):
"""
test a subdir sync of 2 files across 2 nodes
"""
nodes = ['a', 'b']
subdir_name = 'mysubdir'
log_folder = os.path.join(get_subdir_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_subdir_root(), key) for node in nodes
]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_subdir_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node, subdir_name)
time.sleep(2)
# start the sync
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
b_to_a = os.path.join(local_folders[0], subdir_name, file_names[1])
wait_for_file(b_to_a)
time.sleep(2)
a_to_b = os.path.join(local_folders[1], subdir_name, file_names[0])
wait_for_file(a_to_b)
time.sleep(2)
[sync.request_exit() for sync in syncs]
assert (os.path.exists(b_to_a))
assert (os.path.exists(a_to_b))
latus.logger.log.info('test_subdir exiting')
return
def reader(node_id, label):
log_folder = os.path.join(get_node_db_retries_root(), 'log')
logger_init(log_folder)
latus.logger.log.info('entering reader: %s' % node_id)
db_reader = nodedb.NodeDB(get_node_db_retries_root(), node_id)
# I'd rather not have a count down, but we're not guaranteed we'll ever get a retry
count_down = 1000
while count_down > 0:
# print(label, db_reader.get_retry_count(), db_reader.get_heartbeat())
db_reader.get_heartbeat()
if db_reader.get_retry_count() > 0:
count_down = 0
count_down -= 1
def reader(node_id, label):
log_folder = os.path.join(get_node_db_retries_root(), 'log')
logger_init(log_folder)
latus.logger.log.info('entering reader: %s' % node_id)
db_reader = latus.nodedb.NodeDB(get_node_db_retries_root(), node_id)
# I'd rather not have a count down, but we're not guaranteed we'll ever get a retry
count_down = 1000
while count_down > 0:
# print(label, db_reader.get_retry_count(), db_reader.get_heartbeat())
db_reader.get_heartbeat()
if db_reader.get_retry_count() > 0:
count_down = 0
count_down -= 1
def test_node_db_retries():
log_folder = os.path.join(get_node_db_retries_root(), 'log')
logger_init(log_folder)
node_id = 'abc'
w = multiprocessing.Process(target=writer, args=(node_id, 'w', ))
w.start()
time.sleep(1) # todo: make this some sort of poll
read_processes = []
for r in range(0, 20):
read_processes.append(multiprocessing.Process(target=reader, args=(node_id, str(r), )))
[r.start() for r in read_processes]
latus.logger.log.info('all processes started')
while all(r.is_alive() for r in read_processes):
time.sleep(1)
w.terminate()
[r.terminate() for r in read_processes]
def test_move_into_latus_root(setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_move_into_latus_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_move_into_latus_root(), key) for node in nodes]
# get list of folders and files, and write to them
local_folders = []
for node in nodes:
latus_folder = get_latus_folder(get_move_into_latus_root(), node)
local_folders.append(latus_folder)
temp_folder = os.path.join('temp', 'move_to_latus')
file_name = 'a.txt'
write_to_file(temp_folder, file_name, 'move_to_latus', '')
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(3) # wait for syncs to come up
shutil.move(os.path.join(temp_folder, file_name), os.path.join(local_folders[0], file_name))
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name))
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
for local_folder in local_folders:
assert(os.path.exists(os.path.join(local_folder, file_name)))
latus.logger.log.info('test_move_into_latus_root exiting')
return
def test_start_first(setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_start_first_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_start_first_root(), key) for node in nodes]
# start the sync BEFORE we have any files
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(2)
# get list of folders and files, and write to them
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_start_first_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node)
# wait for files to sync
b_to_a = os.path.join(local_folders[0], file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], file_names[0])
wait_for_file(a_to_b)
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(os.path.exists(b_to_a))
assert(os.path.exists(a_to_b))
latus.logger.log.info('test_start_first exiting')
return
def test_subdir(setup):
"""
test a subdir sync of 2 files across 2 nodes
"""
nodes = ['a', 'b']
subdir_name = 'mysubdir'
log_folder = os.path.join(get_subdir_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_subdir_root(), key) for node in nodes]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_subdir_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
write_to_file(latus_folder, file_name, node, subdir_name)
# start the sync
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
b_to_a = os.path.join(local_folders[0], subdir_name, file_names[1])
wait_for_file(b_to_a)
a_to_b = os.path.join(local_folders[1], subdir_name, file_names[0])
wait_for_file(a_to_b)
[sync.request_exit() for sync in syncs]
assert(os.path.exists(b_to_a))
assert(os.path.exists(a_to_b))
latus.logger.log.info('test_subdir exiting')
return
def test_node_db_retries(session_setup, module_setup):
log_folder = os.path.join(get_node_db_retries_root(), 'log')
logger_init(log_folder)
node_id = 'abc'
w = multiprocessing.Process(target=writer, args=(
node_id,
'w',
))
w.start()
time.sleep(1) # todo: make this some sort of poll
read_processes = []
for r in range(0, 20):
read_processes.append(
multiprocessing.Process(target=reader, args=(
node_id,
str(r),
)))
[r.start() for r in read_processes]
latus.logger.log.info('all processes started')
while all(r.is_alive() for r in read_processes):
time.sleep(1)
w.terminate()
[r.terminate() for r in read_processes]
def test_delete(session_setup, module_setup):
"""
test a simple delete across 2 nodes
"""
nodes = ['a', 'b']
sleep_time = latus.const.FILTER_TIME_OUT * 2
log_folder = os.path.join(get_delete_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.DEBUG)
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_delete_root(), key) for node in nodes
]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_delete_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
file_name = nodes[0] + '.txt'
path_node_0 = write_to_file(get_latus_folder(get_delete_root(), nodes[0]),
file_name, nodes[0])
path_node_1 = os.path.join(get_latus_folder(get_delete_root(), nodes[1]),
file_name)
# start the sync
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
time.sleep(sleep_time)
# wait for the file to get sync'd
wait_for_file(path_node_1)
wait_for_file(path_node_0)
time.sleep(sleep_time)
assert (os.path.exists(path_node_0))
assert (os.path.exists(path_node_1))
# now remove the file on the node that it was sync'd to
os.remove(path_node_1)
time.sleep(sleep_time)
# wait for the file to be removed from both nodes
wait_for_file(path_node_0, False)
wait_for_file(path_node_1, False)
# ok .. should be done ... exit
[sync.request_exit() for sync in syncs]
time.sleep(sleep_time)
# make sure it worked OK
assert (not os.path.exists(path_node_0))
assert (not os.path.exists(path_node_1))
latus.logger.log.info('test_delete exiting')
return
def do_sequential(start_first):
"""
test that we can bring up nodes sequentially and they do the right thing (e.g. when a 2nd node comes up
it doesn't signal a file delete just because it doesn't have the file to start with).
"""
nodes = ['a', 'b']
# so we can run this multiple times
clean(get_sequential_root(), False)
log_folder = os.path.join(get_sequential_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_sequential_root(), key) for node in nodes
]
latus_folders = []
latus_file = get_file_name(nodes[0]) # only one file for this test
latus_paths = []
for node in nodes:
folder = get_latus_folder(get_sequential_root(), node)
latus_folders.append(folder)
path = os.path.join(folder, latus_file)
latus_paths.append(path)
time.sleep(latus.const.FILTER_TIME_OUT * 2)
if start_first:
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
time.sleep(3)
[sync.request_exit() for sync in syncs]
time.sleep(3)
write_to_file(latus_folders[0], latus_file, nodes[0])
time.sleep(latus.const.FILTER_TIME_OUT * 2)
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
syncs[0].start() # just start 'a'
# check we have the proper files
assert (os.path.exists(latus_paths[0]))
assert (not os.path.exists(latus_paths[1]))
time.sleep(latus.const.FILTER_TIME_OUT * 2)
syncs[1].start() # start 'b'
wait_for_file(latus_paths[1])
time.sleep(latus.const.FILTER_TIME_OUT * 2)
# stop the syncs
[sync.request_exit() for sync in syncs]
time.sleep(latus.const.FILTER_TIME_OUT * 2)
# check the results
assert (os.path.exists(latus_paths[0]))
assert (os.path.exists(latus_paths[1]))
latus.logger.log.info('test_simple exiting')
return
def test_create_modify_delete(setup):
"""
test that we can bring up nodes sequentially and they do the right thing (e.g. when a 2nd node comes up
it doesn't signal a file delete just because it doesn't have the file to start with).
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_create_modify_delete_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_create_modify_delete_root(), key) for node in nodes]
latus_folders = []
latus_file = get_file_name(nodes[0]) # only one file for this test
latus_paths = []
for node in nodes:
folder = get_latus_folder(get_create_modify_delete_root(), node)
latus_folders.append(folder)
latus_paths.append(os.path.join(folder, latus_file))
syncs = [SyncProc(app_data_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(1)
write_to_file(latus_folders[0], latus_file, 'abc')
[wait_for_file(p) for p in latus_paths]
# check we have the proper files
assert(os.path.exists(latus_paths[0]))
assert(os.path.exists(latus_paths[1]))
# append so it's a modify
write_to_file(latus_folders[0], latus_file, 'def', mode='a')
# wait for append to propagate
while os.path.getsize(os.path.join(latus_folders[1], latus_file)) != 6:
time.sleep(1)
assert(os.path.getsize(os.path.join(latus_folders[1], latus_file)) == 6)
os.remove(latus_paths[0])
# wait for delete to propagate
[wait_for_file(p, False) for p in latus_paths]
# stop the syncs
[sync.request_exit() for sync in syncs]
# check the results
assert(not os.path.exists(latus_paths[0]))
assert(not os.path.exists(latus_paths[1]))
latus.logger.log.info('test_create_modify_delete exiting')
return
def test_log_upload():
logger_init(os.path.join(get_data_root(), 'log'))
app_data_folder = write_preferences('testnode', get_test_upload_root(), 'test_key')
latus.logger.set_appdata_folder(app_data_folder)
latus.logger.set_console_log_level(logging.INFO)
latus.logger.log.error('test_upload_error_message')
def test_move(session_setup, module_setup):
"""
test a move across 2 nodes
"""
nodes = ['a', 'b']
log_folder = os.path.join(get_move_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_move_root(), key) for node in nodes
]
local_folders = []
file_names = []
for node in nodes:
latus_folder = get_latus_folder(get_move_root(), node)
file_name = get_file_name(node)
local_folders.append(latus_folder)
file_names.append(file_name)
file_name = 'src.txt'
path_node_0 = write_to_file(get_latus_folder(get_move_root(), nodes[0]),
file_name, nodes[0])
path_node_1 = os.path.join(get_latus_folder(get_move_root(), nodes[1]),
file_name)
time.sleep(latus.const.FILTER_TIME_OUT * 2)
# start the sync
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
time.sleep(latus.const.FILTER_TIME_OUT * 2)
# wait for the file to get sync'd
wait_for_file(path_node_1, message_prefix='stage 1')
wait_for_file(path_node_0, message_prefix='stage 2')
time.sleep(latus.const.FILTER_TIME_OUT * 2) # wait for filters to timeout
# move to a new file name
new_file_name = 'dest.txt'
new_path_node_0 = os.path.join(get_latus_folder(get_move_root(), nodes[0]),
new_file_name)
new_path_node_1 = os.path.join(get_latus_folder(get_move_root(), nodes[1]),
new_file_name)
latus.logger.log.info('moving %s to %s' % (path_node_0, new_path_node_0))
os.makedirs(os.path.dirname(new_path_node_0), exist_ok=True)
time.sleep(1) # sometimes the makedirs doesn't actually get done!
shutil.move(path_node_0, new_path_node_0)
time.sleep(latus.const.FILTER_TIME_OUT * 2)
# wait for the file to be moved on both notes
wait_for_file(path_node_0, False, message_prefix='stage 3')
wait_for_file(path_node_1, False, message_prefix='stage 4')
wait_for_file(new_path_node_0, message_prefix='stage 5')
wait_for_file(new_path_node_1, message_prefix='stage 6')
# ok .. should be done ... exit
[sync.request_exit() for sync in syncs]
time.sleep(latus.const.FILTER_TIME_OUT * 2)
# make sure it worked OK
assert (not os.path.exists(path_node_0))
assert (not os.path.exists(path_node_1))
assert (os.path.exists(new_path_node_0))
assert (os.path.exists(new_path_node_1))
latus.logger.log.info('test_move exiting')
return
def test_move_out_of_latus(session_setup, module_setup):
nodes = ['a', 'b']
log_folder = os.path.join(get_move_out_of_latus_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [
write_preferences(node, get_move_out_of_latus_root(), key)
for node in nodes
]
# get list of folders and files, and write to them
local_folders = []
for node in nodes:
latus_folder = get_latus_folder(get_move_out_of_latus_root(), node)
local_folders.append(latus_folder)
file_name = 'a.txt'
write_to_file(local_folders[0], file_name, 'move_out_of_latus', '')
# start the sync
syncs = [
SyncProc(app_data_folder, log_folder=log_folder)
for app_data_folder in app_data_folders
]
[sync.start() for sync in syncs]
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name))
temp_folder = os.path.join('temp', 'move_out_of_latus')
os.makedirs(temp_folder, mode=latus.const.MAKE_DIRS_MODE, exist_ok=True)
temp_path = os.path.join(temp_folder, file_name)
# make sure there's not already something in the dest (e.g. from another test)
try:
os.remove(temp_path)
except FileNotFoundError:
pass
shutil.move(os.path.join(local_folders[0], file_name), temp_path)
time.sleep(1)
# wait for files to sync
for local_folder in local_folders:
wait_for_file(os.path.join(local_folder, file_name), False)
# stop the syncs
[sync.request_exit() for sync in syncs]
# time.sleep(5)
# check the results
for local_folder in local_folders:
assert (not os.path.exists(os.path.join(local_folder, file_name)))
latus.logger.log.info('test_move_out_of_latus_root exiting')
return
def test_create_modify_delete(session_setup, module_setup):
"""
test that we can bring up nodes sequentially and they do the right thing (e.g. when a 2nd node comes up
it doesn't signal a file delete just because it doesn't have the file to start with).
"""
nodes = ['a', 'b']
sleep_time = latus.const.FILTER_TIME_OUT * 2.0
log_folder = os.path.join(get_create_modify_delete_root(), 'log')
logger_init(log_folder)
latus.logger.set_console_log_level(logging.INFO)
# write preferences
key = latus.crypto.new_key()
app_data_folders = [write_preferences(node, get_create_modify_delete_root(), key) for node in nodes]
latus_folders = []
latus_file = get_file_name(nodes[0]) # only one file for this test
latus_paths = []
for node in nodes:
folder = get_latus_folder(get_create_modify_delete_root(), node)
latus_folders.append(folder)
latus_paths.append(os.path.join(folder, latus_file))
syncs = [SyncProc(app_data_folder, log_folder=log_folder) for app_data_folder in app_data_folders]
[sync.start() for sync in syncs]
time.sleep(sleep_time)
write_to_file(latus_folders[0], latus_file, 'abc')
[wait_for_file(p) for p in latus_paths]
# check we have the proper files
time.sleep(sleep_time)
assert(os.path.exists(latus_paths[0]))
assert(os.path.exists(latus_paths[1]))
# append so it's a modify
write_to_file(latus_folders[0], latus_file, 'def', mode='a')
# wait for append to propagate
while os.path.getsize(os.path.join(latus_folders[1], latus_file)) != 6:
time.sleep(sleep_time)
assert(os.path.getsize(os.path.join(latus_folders[1], latus_file)) == 6)
wait_for_file(latus_paths[0])
os.remove(latus_paths[0])
# wait for delete to propagate
[wait_for_file(p, False) for p in latus_paths]
time.sleep(sleep_time)
# stop the syncs
[sync.request_exit() for sync in syncs]
time.sleep(sleep_time)
# check the results
assert(not os.path.exists(latus_paths[0]))
assert(not os.path.exists(latus_paths[1]))
latus.logger.log.info('test_create_modify_delete exiting')
return
