def run_client_service(client_sent_snapshots_counter,
test_user_id=None,
client_sent_user_id=None):
client_service = ClientService(SERVER_TEST_HOST, DEFAULT_PORT)
test_user_id = test_user_id if test_user_id else get_test_user(1).ID
test_file_path = get_user_test_file_path(test_user_id)
# Upload to server
client_service.upload_sample(test_file_path, DEFAULT_FILE_VERSION)
client_sent_snapshots_counter.value = client_service._total_snapshots_uploaded
if client_sent_user_id:
client_sent_user_id.value = client_service._user_id
def create_test_mind_file(file_path='',
version='',
test_user_number=1,
snapshot_manipulator_flags=None):
file_path = file_path if file_path else DEFAULT_FILE_PATH
version = version if version else DEFAULT_FILE_VERSION
if version in DEPRECTED_FILE_VERSION:
print(
f'ERROR! {version} is deprecated - won\'t export to example file')
return
example_file_path = get_user_test_file_path(test_user_number)
with MindFileReader(file_path, version) as sample_reader:
with MindFileWriter(example_file_path, version) as sample_writer:
user_information = sample_reader.user_information
if test_user_number:
user_information.user_id = int(
get_test_user(test_user_number).ID)
user_information.username = get_test_user(
test_user_number).NAME
sample_writer.write_user_information(user_information)
snapshots_counter = 0
for snapshot in sample_reader:
snapshots_counter += 1
if snapshot_manipulator_flags:
snapshot_manipulator(snapshot, snapshot_manipulator_flags)
sample_writer.write_snapshot(snapshot)
if EXAMPLE_SNAPSHOTS_NUMBER == snapshots_counter:
break
with MindFileReader(example_file_path, version) as sample_reader:
user_information = sample_reader.user_information
snapshots_counter = 0
for snapshot in sample_reader:
snapshots_counter += 1
assert EXAMPLE_SNAPSHOTS_NUMBER == snapshots_counter
print(f'Exported {file_path} ---> {example_file_path} successfully!')
def delete_server_user_folder_before_and_after(user_id=get_test_user(1).ID):
def decorator(function):
@functools.wraps(function)
def wrapper(*args, **kwargs):
user_snapshots_path = ServerHandler.get_user_snapshots_path(user_id)
delete_under_folder(user_snapshots_path)
result = function(*args, **kwargs)
delete_under_folder(user_snapshots_path)
return result
return wrapper
return decorator
def test_server_snapshots_dedup():
# Creating shared-memory value to count sent snapshots
client_sent_snapshots_counter = Value('i', 0)
# Creating shared-memory value to count published snapshots
test_server_snapshot_published_counter = Value('i', 0)
def run_server_service(test_server_snapshot_published_counter):
def snapshot_publish(message):
# Update shared-memory published snapshots counter
test_server_snapshot_published_counter.value += 1
run_server(SERVER_TEST_HOST, DEFAULT_PORT, publish=snapshot_publish)
# Spawn process for server thread
server_proccess = multiprocessing.Process(
target=run_server_service,
args=[test_server_snapshot_published_counter])
server_proccess.start()
# Uploading twice same file
for _ in range(2):
# Spawn process for client thread and kill it upon finishing
client_proccess = multiprocessing.Process(
target=run_client_service, args=[client_sent_snapshots_counter])
client_proccess.start()
client_proccess.join()
client_proccess.kill()
# Wait for server to handle messages and kill it
server_proccess.join(SERVER_SNAPSHOT_MAX_DURATION_HANDLING *
client_sent_snapshots_counter.value)
server_proccess.kill()
# Ensure snapshots have been published
assert 0 < test_server_snapshot_published_counter.value
# Ensure number of snapshots published equals the number of snapshots in current last execution, thus, duplicate snapshot uploading won't be published
assert client_sent_snapshots_counter.value == test_server_snapshot_published_counter.value
# Ensure total snapshots folders is as much as snapshots in file
total_snapshots_folders = count_folders_subfolders(
ServerHandler.get_user_snapshots_path(get_test_user(1).ID))
assert total_snapshots_folders == client_sent_snapshots_counter.value
def test_server_service_multiple_clients():
# Creating shared-memory value to count published snapshots
test_server_snapshot_published_counter = Value('i', 0)
test_server_snapshot_published_ids = Manager().list()
def run_server_service(test_server_snapshot_published_counter,
test_server_snapshot_published_ids):
def snapshot_publish(message):
snapshot = MessageQueueMessages().get_message(
MessageQueueMessagesTyeps.RAW_SNAPSHOT_MESSAGE).deserialize(
message)
user_id = snapshot.user_info.user_id
test_server_snapshot_published_ids.append(user_id)
# Update shared-memory published snapshots counter
test_server_snapshot_published_counter.value += 1
run_server(SERVER_TEST_HOST, DEFAULT_PORT, publish=snapshot_publish)
# Spawn process for server thread
server_proccess = multiprocessing.Process(
target=run_server_service,
args=[
test_server_snapshot_published_counter,
test_server_snapshot_published_ids
])
server_proccess.start()
# Clients processes handling
test_users_ids = [get_test_user(1).ID, get_test_user(2).ID]
test_users_snapshots_counters = []
client_proccesses_list = []
# Create process for client threads
for user_proccess_index in range(len(test_users_ids)):
user_id = int(test_users_ids[user_proccess_index])
client_sent_snapshots_counter = Value('i', 0)
test_users_snapshots_counters.append(client_sent_snapshots_counter)
client_proccess = multiprocessing.Process(
target=run_client_service,
args=[client_sent_snapshots_counter, user_id])
client_proccesses_list.append(client_proccess)
# Spawn client processes
for client_proccess in client_proccesses_list:
client_proccess.start()
# Tiny sleep to avoid connection reset (which is a matter of security)
# "It seems like the server side limits the amount of requests per timeunit (hour, day, second) as a security issue." @FelixMartinez
# https://stackoverflow.com/questions/20568216/python-handling-socket-error-errno-104-connection-reset-by-peer
time.sleep(0.5)
# Kill client processes upon finishing
for client_proccess in client_proccesses_list:
client_proccess.join()
client_proccess.kill()
# Wait for server to handle messages and kill it
total_snapshots_clients_sent = sum([
client_sent_snapshots_counter.value
for client_sent_snapshots_counter in test_users_snapshots_counters
])
server_proccess.join(SERVER_SNAPSHOT_MAX_DURATION_HANDLING *
total_snapshots_clients_sent)
server_proccess.kill()
# Ensuring requests order is interleaved
assert is_interleaved_list(test_server_snapshot_published_ids)
# Ensure snapshots have been published
assert 0 < test_server_snapshot_published_counter.value
assert total_snapshots_clients_sent == test_server_snapshot_published_counter.value
# Ensure total snapshots folders is as much as snapshots in file
for user_proccess_index in range(len(test_users_ids)):
user_id = test_users_ids[user_proccess_index]
total_snapshots_folders = count_folders_subfolders(
ServerHandler.get_user_snapshots_path(user_id))
assert total_snapshots_folders == test_users_snapshots_counters[
user_proccess_index].value
# Spawn process for client thread and kill it upon finishing
client_proccess = multiprocessing.Process(
target=run_client_service, args=[client_sent_snapshots_counter])
client_proccess.start()
client_proccess.join()
client_proccess.kill()
# Wait for server to handle messages and kill it
server_proccess.join(SERVER_SNAPSHOT_MAX_DURATION_HANDLING *
client_sent_snapshots_counter.value)
server_proccess.kill()
# Ensure snapshots have been published
assert 0 < test_server_snapshot_published_counter.value
assert client_sent_snapshots_counter.value == test_server_snapshot_published_counter.value
@delete_server_user_folder_before_and_after(get_test_user(1).ID)
@delete_server_user_folder_before_and_after(get_test_user(2).ID)
def test_server_service_multiple_clients():
# Creating shared-memory value to count published snapshots
test_server_snapshot_published_counter = Value('i', 0)
test_server_snapshot_published_ids = Manager().list()
def run_server_service(test_server_snapshot_published_counter,
test_server_snapshot_published_ids):
def snapshot_publish(message):
snapshot = MessageQueueMessages().get_message(
MessageQueueMessagesTyeps.RAW_SNAPSHOT_MESSAGE).deserialize(
message)
user_id = snapshot.user_info.user_id
test_server_snapshot_published_ids.append(user_id)
# Update shared-memory published snapshots counter