def send_image_file(middlebox_module, testing_part_1):
""" Verifies that sending an image (as opposed to text) file works correctly. """
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Send a file from client 1 to client 2.
filename = "sample.jpg"
client1.send_file(filename, client2_address)
# Make sure that the files have the same contents.
with open(filename, "r") as input_file:
input_data = input_file.read()
output_file_name = "{}-{}".format("client2", filename)
with open(output_file_name, "r") as output_file:
result_data = output_file.read()
# Removing the output file just created
os.remove(output_file_name)
if input_data != result_data:
raise Exception("send_image_file failed, because the file received" +
"did not match the file sent.")
def send_one_byte_at_a_time(middlebox_module, testing_part_1):
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
sent_data = ""
for _ in range(
80000
): # 8KB * 10 bytes so that we get a block end somewhere for part 2
random_byte = os.urandom(1)
client1.send_data(random_byte, client2_address)
sent_data += random_byte
client1.send_fin(client2_address)
# Verify that the correct data was received.
if not client2.received_fin:
raise Exception("Client 2 never received a fin")
test_utils.verify_data_sent_equals_data_received(sent_data,
client2_output_filename)
os.remove(client1_output_filename) # I have no idea why we need this.
os.remove(client2_output_filename)
def send_multiple_files(middlebox_module, testing_part_1):
total_count = 0
sent_files = 4
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize clients 1 and 2, which are connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
client2_address = "1.2.3.5"
client2 = client.EndHost("client2", client2_address, middlebox1)
# Initialize clients 3 and 4, which are connected to middlebox 2.
client3_address = "5.6.7.8"
client3 = client.EndHost("client3", client3_address, middlebox2)
client4_address = "5.6.7.9"
client4 = client.EndHost("client4", client4_address, middlebox2)
filename = "sample.txt"
with open(filename, "rb") as input_file:
input_data = input_file.read()
# Send the sample file from client 1 to both clients 3 and 4.
client1.send_file(filename, client3_address)
client1.send_file(filename, client4_address)
# Make sure that the files have the same contents.
for receiver in ["client3", "client4"]:
output_file_name = "{}-{}".format(receiver, filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
# Remove the output file just created.
os.remove(output_file_name)
if input_data == result_data:
total_count += 1
# Send a file from client 2 to clients 3 and 4.
client2.send_file(filename, client3_address)
client2.send_file(filename, client4_address)
# Make sure that the files have the same contents.
for receiver in ["client3", "client4"]:
output_file_name = "{}-{}".format(receiver, filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
# Removing the output file just created
os.remove(output_file_name)
if input_data == result_data:
total_count += 1
if total_count != sent_files:
raise Exception(
"send_mutiple_files failed, because the all files" +
"received did not match the file sent. Files received correctly:" +
" {} and files sent are: {}\n".format(total_count, sent_files))
def data_reduction_same_files_small(middlebox_module, testing_part_1):
""" Tests that the WAN optimizer reduces data sent over the WAN.
This test has the same functionality as data_reduction_same_files, but
operates on smaller files.
This test sends the same file twice, and then checks that the reduction
ratio:
(bytes sent from client - bytes sent over wan) /
bytes sent from client
is as expected. The reduction ratios in the test are hardcoded based on
a reference solution.
"""
if testing_part_1:
expected_value = 0.49
else:
expected_value = 0.49
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Send a file twice from client 1 to client 2.
filename = "sample_short.txt"
output_file_name = "{}-{}".format("client2", filename)
client1.send_file(filename, client2_address)
# Removing the file just created
os.remove(output_file_name)
client1.send_file(filename, client2_address)
# Removing the file just created
os.remove(output_file_name)
# Compute the data reduction ratio
with open(filename, "rb") as input_file:
input_data = input_file.read()
extra_data_length = len(filename) + len(client.FILENAME_DELIMITER)
bytes_in_sent_file = len(input_data) + extra_data_length
bytes_sent = wide_area_network.get_total_bytes_sent()
# print bytes_sent, bytes_in_sent_file, bytes_in_received_file
reduction = (float(bytes_in_sent_file * 2 - bytes_sent) /
float(bytes_in_sent_file * 2))
if (reduction < expected_value):
raise Exception("data_reduction_same_files failed," +
" because reduction ratio should be greater than " +
" {}, was {}.".format(expected_value, reduction))
def verify_middlebox_handles_interleaved_data(middlebox_module,
is_testing_part1):
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize 2 clients connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
client3_address = "5.6.7.9"
client3_output_filename = "{}_output".format(client3_address)
client3 = simple_client.SimpleClient(client3_address, middlebox2,
client3_output_filename)
# Send part of a block from client 1 to client 2.
first_client2_block = "a" * 3000
client1.send_data(first_client2_block, client2_address)
# Now send part of a block from client 1 to client 3.
first_client3_block = "b" * 7000
client1.send_data(first_client3_block, client3_address)
# Now send some more data to client 2.
second_client2_block = "a" * 14000
client1.send_data(second_client2_block, client2_address)
# Send more data to client 3 and close that stream.
second_client3_block = "b" * 9000
client1.send_data(second_client3_block, client3_address)
client1.send_fin(client3_address)
if not client3.received_fin:
raise Exception("Client 3 didn't receive the fin")
# Close the client 2 stream.
client1.send_fin(client2_address)
if not client2.received_fin:
raise Exception("Client 2 didnt't receive the fin")
# Make sure the data is correct.
test_utils.verify_data_sent_equals_data_received(
first_client2_block + second_client2_block, client2_output_filename)
test_utils.verify_data_sent_equals_data_received(
first_client3_block + second_client3_block, client3_output_filename)
def one_client_with_multiple_receivers(middlebox_module, testing_part_1):
""" Tests a scenario where one client is sending data to 3 other clients.
Verifies that the data received equals the data sent, and throw an Exception
if this is not the case.
"""
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1, client1_output_filename)
# Initialize 3 clients connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(
client2_address, middlebox2, client2_output_filename)
client3_address = "5.6.7.9"
client3_output_filename = "{}_output".format(client3_address)
client3 = simple_client.SimpleClient(
client3_address, middlebox2, client3_output_filename)
client4_address = "5.6.7.10"
client4_output_filename = "{}_output".format(client4_address)
client4 = simple_client.SimpleClient(
client4_address, middlebox2, client4_output_filename)
# Send data from client 1 to client 2.
data_to_client2 = "2" * 8000
client1.send_data(data_to_client2, client2_address)
client1.send_fin(client2_address)
test_utils.verify_data_sent_equals_data_received(
data_to_client2,
client2_output_filename)
# Send different data from client 1 to client 3.
data_to_client3 = "3" * 8000
client1.send_data(data_to_client3, client3_address)
client1.send_fin(client3_address)
test_utils.verify_data_sent_equals_data_received(
data_to_client3, client3_output_filename)
# And finally, send data from client 1 to client 4.
data_to_client4 = "4" * 8000
client1.send_data(data_to_client4, client4_address)
client1.send_fin(client4_address)
test_utils.verify_data_sent_equals_data_received(
data_to_client4, client4_output_filename)
def send_just_over_one_block(middlebox_module, testing_part_1):
""" Sends a file that contains a bit over on block.
Verifies that when you send the last packet of this file
you send out the packetized block and then send out the
remaining bytes in the buffer.
To demonstrate, we must make sure that if you sent out
the 8000 bytes below, you do not send a fin on the packet
with bytes 7500 to 8000 but you do send one for the byte
with the last 500 bytes of the file.
|____|____|____|____|____|__| |__|
1500 3000 4500 6000 7500 8000 500
"""
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Send a file from client 1 to client 2.
if testing_part_1:
filename = "8500a.txt"
else:
filename = "just_over_block_pt_2.txt"
client1.send_file(filename, client2_address)
# Make sure that the files have the same contents.
with open(filename, "rb") as input_file:
input_data = input_file.read()
output_file_name = "{}-{}".format("client2", filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
# Remove the output file just created.
os.remove(output_file_name)
if input_data != result_data:
raise Exception(
("The file received did not match the file sent. File sent (size {}):\n{}\nFile " +
"received (size {}):\n{}\n").format(len(input_data), input_data, len(result_data), result_data))
def test_delimiter_at_end_without_fin(middlebox_module, is_testing_part1):
if is_testing_part1:
print("test_delimiter_at_end_without_fin does not test simple_test")
return
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize 2 clients connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
client3_address = "5.6.7.9"
client3_output_filename = "{}_output".format(client3_address)
client3 = simple_client.SimpleClient(client3_address, middlebox2,
client3_output_filename)
# Send part of a block from client 1 to client 2.
first_client2_block = "a" * 2000
client1.send_data(first_client2_block, client2_address)
#tests whether we have not recieved anything
test_utils.verify_data_sent_equals_data_received("",
client2_output_filename)
# Now send some more data to client 2.
second_client2_block = " straight chin suggestive of resolution pushed t"
client1.send_data(second_client2_block, client2_address)
if client2.received_fin:
raise Exception("Client 2 received the fin too early")
#make sure data is correct, we do not have buffer, we just send everything
test_utils.verify_data_sent_equals_data_received(
first_client2_block + second_client2_block, client2_output_filename)
# Close the client 2 stream.
client1.send_fin(client2_address)
if not client2.received_fin:
raise Exception("Client 2 didnt't receive the fin")
test_utils.verify_data_sent_equals_data_received(
first_client2_block + second_client2_block, client2_output_filename)
def verify_data_is_sent_incrementally(middlebox_module, is_testing_part1):
""" Verifies that data is sent incrementally over the WAN.
This test makes sure that the WAN optimizer doesn't wait for a FIN
packet to send data over the WAN.
"""
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
# Send exactly one block from client 1 to client 2.
if is_testing_part1:
single_block = "a" * 8000
else:
single_block = (
"From the lower part of the face he appeared" +
" to be a man of strong character, with a thick, hanging lip," +
" and a long, straight chin suggestive of resolution pushed t")
client1.send_data(single_block, client2_address)
wan_bytes_sent = wide_area_network.get_total_bytes_sent()
if wan_bytes_sent != len(single_block):
raise Exception(
"Since a complete block was sent to the WAN " +
"optimizer, a complete block should have been sent over the " +
"WAN, but only {} bytes were sent.".format(wan_bytes_sent))
# Send a second block, and then close the stream.
second_block = "b" * 8000
client1.send_data(second_block, client2_address)
client1.send_fin(client2_address)
test_utils.verify_data_sent_equals_data_received(
single_block + second_block, client2_output_filename)
def verify_data_is_sent_incrementally(middlebox_module, is_testing_part1):
""" Verifies that data is sent incrementally over the WAN.
This test makes sure that the WAN optimizer doesn't wait for a FIN
packet to send data over the WAN.
"""
if not is_testing_part1:
# This test is only valid for part 1 (because part 2
# splits blocks differently).
return
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
# Send exactly one block from client 1 to client 2.
single_block = "a" * 8000
client1.send_data(single_block, client2_address)
wan_bytes_sent = wide_area_network.get_total_bytes_sent()
if wan_bytes_sent != 8000:
raise Exception(
"Since a complete block was sent to the WAN " +
"optimizer, a complete block should have been sent over the " +
"WAN, but only {} bytes were sent.".format(wan_bytes_sent))
# Send a second block, and then close the stream.
second_block = "b" * 8000
client1.send_data(second_block, client2_address)
client1.send_fin(client2_address)
test_utils.verify_data_sent_equals_data_received(
single_block + second_block, client2_output_filename)
def send_48_bytes(middlebox_module, testing_part_1):
SPECIAL_BYTES = "flamed face and disreputable clothes, walked int"
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
client1.send_data(SPECIAL_BYTES, client2_address)
client1.send_fin(client2_address)
# Verify that the correct data was received.
if not client2.received_fin:
raise Exception("Client 2 never received a fin")
test_utils.verify_data_sent_equals_data_received(SPECIAL_BYTES,
client2_output_filename)
os.remove(client2_output_filename)
client2.send_data(SPECIAL_BYTES, client1_address)
client2.send_fin(client1_address)
if not client1.received_fin:
raise Exception("Client 1 never received a fin")
test_utils.verify_data_sent_equals_data_received(SPECIAL_BYTES,
client1_output_filename)
os.remove(client1_output_filename)
achieved_reduction = compute_reduction(len(SPECIAL_BYTES),
wide_area_network)
expected_reduction = 0.29
if achieved_reduction < expected_reduction:
raise Exception(
"Too many bytes were sent over the network. Expected reduction rate was {} but achieved only {}."
.format(expected_reduction, achieved_reduction))
def send_fin_overload_buffer(middlebox_module, testing_part_1):
""" Sends a single large file and verifies that it's received correctly.
This test only verifies that the correct data is received, and does not
check the optimizer's data compression.
"""
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Create a file to send from client 1 to client 2.
filename = "fin-overload-input.txt"
f = open(filename, "w")
f.write("a" * 8500)
f.close()
# Send file from client 1 to client 2.
client1.send_file(filename, client2_address)
client1.send_file(filename, client2_address)
# Make sure that the files have the same contents.
with open(filename, "r") as input_file:
input_data = input_file.read()
os.remove(filename)
output_file_name = "{}-{}".format("client2", filename)
with open(output_file_name, "r") as output_file:
result_data = output_file.read()
# Remove the output file just created.
os.remove(output_file_name)
if input_data != result_data:
raise Exception(
"The file received did not match the file sent. File received had: "
+ "{}\n and file sent had: {}\n".format(result_data, input_data))
def cache_is_not_flow_specific(middlebox_module, testing_part_1):
""" Checks that a given block appears in the cache at most once.
First, client 1 sends a file to client 2 and client 3.
Then both client 2 and client 3 send the file back to client 1.
If your cache is flow-specific, you will end up with duplicate blocks.
"""
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client A, connected to middlebox 2.
client2_address = "5.5.5.5"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Initialize client B, connected to middlebox 2.
client3_address = "6.6.6.6"
client3 = client.EndHost("client3", client3_address, middlebox2)
filename = "8000B.txt"
client1.send_file(filename, client2_address)
client1.send_file(filename, client3_address)
if len(set(middlebox1.buffers.values())) > len(
middlebox1.buffers.values()):
raise Exception("SRC middlebox has duplicate cache entries: %s" %
middlebox1.cache)
client2.send_file(filename, client1_address)
client3.send_file(filename, client1_address)
if len(set(middlebox2.buffers.values())) > len(
middlebox2.buffers.values()):
raise Exception("DST middlebox has duplicate cache entries: %s" %
middlebox2.cache)
def send_empty_file(middlebox_module, testing_part_1):
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Send a file from client 1 to client 2.
filename = "send_small_files_1.txt"
client1.send_file(filename, client2_address)
# Make sure that the files have the same contents.
with open(filename, "rb") as input_file:
input_data = input_file.read()
output_file_name = "{}-{}".format("client2", filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
# Remove the output file just created.
os.remove(output_file_name)
if input_data != result_data:
raise Exception((
"The file received did not match the file sent. File sent (size {}):\n{}\nFile "
+ "received (size {}):\n{}\n").format(len(input_data), input_data,
len(result_data),
result_data))
# Make sure no extra bytes are sent.
expected_bytes = compute_bytes_in_file(filename)
sent_bytes = wide_area_network.get_total_bytes_sent()
if expected_bytes != sent_bytes:
raise Exception(
"An incorrect number of bytes were sent over the WAN. Expected {} bytes, but got {} bytes."
.format(expected_bytes, sent_bytes))
def simple_send_test(middlebox_module, data_to_send, data_to_send_in_reverse):
""" Sends data between two clients.
Verifies that the data received equals the data sent, and throw an Exception
if this is not the case.
"""
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1, client1_output_filename)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2, client2_output_filename)
if data_to_send:
client1.send_data(data_to_send, client2_address)
client1.send_fin(client2_address)
# Verify that the correct data was received.
if not client2.received_fin:
raise Exception("Client 2 never received a fin")
test_utils.verify_data_sent_equals_data_received(
data_to_send,
client2_output_filename)
if data_to_send_in_reverse:
client2.send_data(data_to_send_in_reverse, client1_address)
client2.send_fin(client1_address)
if not client1.received_fin:
raise Exception("Client 1 never received a fin")
test_utils.verify_data_sent_equals_data_received(
data_to_send_in_reverse,
client1_output_filename)
def hash_partial_blocks(middlebox_module, testing_part_1):
expected_value_1 = 0.995 #For both parts.
expected_value_2 = 0.9985 #For both parts.
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
# Define names of files.
filenames = ["hash_partial_sample_1.txt", "hash_partial_sample_2.txt"]
# Send both files back and forth.
for file_id in range(len(filenames)):
filename = filenames[file_id]
for count in range(2):
client1.send_file(filename, client2_address)
output_file_name = "{}-{}".format("client2", filename)
if not check_file_integrity(filename, output_file_name):
return
os.remove(output_file_name)
if count == 0:
# This is a hack! Don't do this!
wide_area_network._Wan__total_bytes_sent = 0
reduction_rate = compute_reduction(filename, wide_area_network)
expected_value = expected_value_1 if file_id == 0 else expected_value_2
if (reduction_rate < expected_value):
raise Exception(
"Reduction ratio should be greater than " +
" {}, was {}.".format(expected_value, reduction_rate))
def data_reduction_with_jumbled_files(middlebox_module, testing_part_1):
""" Tests whether sending files with the same blocks in different orders
results in proper data compression.
"""
filename = "Diamond_top.txt"
block1 = "a" * 8000
block2 = "b" * 8000
block3 = "c" * 8000
expected_value = .65
if testing_part_1:
delimeter = ""
combo1 = [block1[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block2, block3]
combo2 = [block1[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block3, block2]
combo3 = [block2[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block1, block3]
combo4 = [block2[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block3, block1]
combo5 = [block3[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block1, block2]
combo6 = [block3[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block2, block1]
else:
delimeter = " straight chin suggestive of resolution pushed t"
combo1 = [delimeter, block1, block2, block3]
combo2 = [delimeter, block1, block3, block2]
combo3 = [delimeter, block2, block1, block3]
combo4 = [delimeter, block2, block3, block1]
combo5 = [delimeter, block3, block1, block2]
combo6 = [delimeter, block3, block2, block1]
combos = [combo1, combo2, combo3, combo4, combo5, combo6]
filename = "Diamond_top.txt"
files = []
for combo in combos:
files.append(delimeter.join(combo))
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
bytes_in_sent_files = 0
for data in files:
f = open(filename, 'w')
f.write(data)
f.close()
past2 = wide_area_network.get_total_bytes_sent()
client1.send_file(filename, client2_address)
output_file_name = "{}-{}".format("client2", filename)
# Removing the output file just created
os.remove(output_file_name)
past = bytes_in_sent_files
bytes_in_sent_files += len(data) + len(filename) + len(client.FILENAME_DELIMITER)
bytes_sent = wide_area_network.get_total_bytes_sent()
reduction = (float(bytes_in_sent_files - bytes_sent)
/ float(bytes_in_sent_files))
if (reduction < expected_value):
raise Exception("data_reduction_random_edit_file failed," +
" because reduciton ratio should be greater than " +
" {}, was {}.".format(expected_value, reduction))
def data_reduction_suffixed_files_2(middlebox_module, testing_part_1):
""" Tests that the WAN optimizer reduces data sent over the WAN.
This test sends a file and then sends the same file with extra data
appended at the end. Both tests have a filename of the same length,
so that all of the data sent will be the same, until the suffix at
the end. For both types of WAN optimizer, this should result in the first
blocks being the same when the file is sent again (so there should be
significant compression). The test checks that the reduction
ratio:
(bytes sent from client - bytes sent over wan) /
bytes sent from client
is as expected. The reduction ratios in the test are hardcoded based on
a reference solution.
"""
if testing_part_1:
expected_value = 0
else:
expected_value = 0.49
# if testing_part_1:
# expected_value = 0.40
# else:
# expected_value = 0.63
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
client3_address = "5.6.7.9"
client3 = client.EndHost("client3", client3_address, middlebox2)
# filename = ["good_text_with_limiter.txt", "good_text_with_limiter.txt"]
filename = ["suffix_sample3_var_1.txt", "suffix_sample3_var_1.txt"]
# Send a file from client 1 to client 2.
client1.send_file(filename[0], client2_address)
output_file_name = "{}-{}".format("client2", filename[0])
# Removing the output file just created
os.remove(output_file_name)
# Send a file prefixed with some data in the beginning
# of the same file
client2.send_file(filename[1], client1_address)
output_file_name = "{}-{}".format("client1", filename[1])
# Removing the output file just created
os.remove(output_file_name)
bytes_in_sent_files = 0
for f in filename:
with open(f, "rb") as input_file:
input_data = input_file.read()
extra_data_length = len(f) + len(client.FILENAME_DELIMITER)
bytes_in_sent_files += len(input_data) + extra_data_length
bytes_sent = wide_area_network.get_total_bytes_sent()
# print bytes_sent, bytes_in_sent_files
reduction = (float(bytes_in_sent_files - bytes_sent) /
float(bytes_in_sent_files))
print(reduction)
if (reduction < expected_value):
raise Exception("data_reduction_suffixed_files failed," +
" because reduciton ratio should be less than " +
" {}, was {}.".format(expected_value, reduction))
def one_sender_multiple_sends(middlebox_module, testing_part_1):
total_count = 0
sent_files = 2
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize clients 1 which is connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize clients 2 which is connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
filename = "sample.txt"
with open(filename, "rb") as input_file:
input_data = input_file.read()
# Send the sample file from client 1 to both clients 3 and 4.
client1.send_file(filename, client2_address)
# Make sure that the files have the same contents.
receiver = "client2"
output_file_name = "{}-{}".format(receiver, filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
# Remove the output file just created.
os.remove(output_file_name)
if input_data == result_data:
total_count += 1
# Send same file to client 2 again.
client1.send_file(filename, client2_address)
# Make sure that the files have the same contents.
receiver = "client2"
output_file_name = "{}-{}".format(receiver, filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
# Removing the output file just created
os.remove(output_file_name)
if input_data == result_data:
total_count += 1
if total_count != sent_files:
raise Exception(
"send_mutiple_files failed, because the all files" +
"received did not match the file sent. Files received correctly:" +
" {} and files sent are: {}\n".format(total_count, sent_files))
# Compute the data reduction ratio
expected_value = 0.49
with open(filename, "rb") as input_file:
input_data = input_file.read()
extra_data_length = len(filename) + len(client.FILENAME_DELIMITER)
bytes_in_sent_file = len(input_data) + extra_data_length
bytes_sent = wide_area_network.get_total_bytes_sent()
reduction = (float(bytes_in_sent_file * 2 - bytes_sent) /
float(bytes_in_sent_file * 2))
if (reduction < expected_value):
raise Exception("data_reduction_same_files failed," +
" because reduction ratio should be greater than " +
" {}, was {}.".format(expected_value, reduction))
def data_reduction_prefixed_files(middlebox_module, testing_part_1):
""" Tests that the WAN optimizer reduces data sent over the WAN.
This test sends a file and then sends the same file with extra data
at the beginning. Because the data is offset, this will result in no
blocks being the same for the part 1 middlebox. However, the part 2
middlebox should be able to handle this, and still significantly reduce
data sent over the WAN. The test checks that the reduction
ratio:
(bytes sent from client - bytes sent over wan) /
bytes sent from client
is as expected. The reduction ratios in the test are hardcoded based on
a reference solution.
"""
if testing_part_1:
expected_value = 0.0
else:
expected_value = 0.45
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1 = client.EndHost("client1", client1_address, middlebox1)
# Initialize client connected to middlebox 2.
client2_address = "5.6.7.8"
client2 = client.EndHost("client2", client2_address, middlebox2)
filename = ["sample.txt", "prefix_sample.txt"]
# Send a file from client 1 to client 2.
client1.send_file(filename[0], client2_address)
output_file_name = "{}-{}".format("client2", filename[0])
# Removing the output file just created
os.remove(output_file_name)
# Send a file prefixed with some data in the beginning
# of the same file
client1.send_file(filename[1], client2_address)
output_file_name = "{}-{}".format("client2", filename[1])
# Removing the output file just created
os.remove(output_file_name)
bytes_in_sent_files = 0
for f in filename:
with open(f, "r") as input_file:
input_data = input_file.read()
extra_data_length = len(f) + len(client.FILENAME_DELIMITER)
bytes_in_sent_files += len(input_data) + extra_data_length
bytes_sent = wide_area_network.get_total_bytes_sent()
# print bytes_sent, bytes_in_sent_file, bytes_in_received_file
reduction = (float(bytes_in_sent_files - bytes_sent) /
float(bytes_in_sent_files))
if (reduction < expected_value):
raise Exception("data_reduction_prefixed_files failed," +
" because reduction ratio should be greater than " +
" {}, was {}.".format(expected_value, reduction))
def test_right_partitions(middlebox_module, is_testing_part1):
"""
This function is a test of Aisha's example
So in example you've given, the window will progress like this...
[0:4] ---> found a block
[5:9] ----> found a block
and I would have the following two blocks. 'aaaaa' and 'aaaaa'
let's say if the string was 'bbbaaaaaaaaaaaaaaa'
the window will progress as follows:
[0:4]
[1:5]
[2:6]
[3:7] ---> found a block = 'bbbaaaaa'
[8:12] ---> found a block = 'aaaaa'
[13:17] ---->found a block. = 'aaaaa'
except that I will use the delimiter " straight chin suggestive of resolution pushed t"
instead.
"""
if is_testing_part1:
delimit = "b" * 8000
first_expected_bytes = 16020 #one 8000 byte block for
second_expected_bytes = 16080
third_expected_bytes = 16100
first_client2_block = "a" * 2000 + delimit[:6000] + delimit + delimit
second_client2_block = delimit
else:
delimit = " straight chin suggestive of resolution pushed t"
first_expected_bytes = 2116
second_expected_bytes = 2176
third_expected_bytes = 2196
first_client2_block = "a" * 2000 + delimit + delimit + delimit
second_client2_block = delimit
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Iniitialize client connected to middlebox 1.
client1_address = "1.2.3.4"
client1_output_filename = "{}_output".format(client1_address)
client1 = simple_client.SimpleClient(client1_address, middlebox1,
client1_output_filename)
# Initialize 2 clients connected to middlebox 2.
client2_address = "5.6.7.8"
client2_output_filename = "{}_output".format(client2_address)
client2 = simple_client.SimpleClient(client2_address, middlebox2,
client2_output_filename)
# Send part of a block from client 1 to client 2.
client1.send_data(first_client2_block, client2_address)
#tests whether we recieved right thing
test_utils.verify_data_sent_equals_data_received(first_client2_block,
client2_output_filename)
bytes_sent = wide_area_network.get_total_bytes_sent()
print('bytes sent 1')
print(bytes_sent)
if bytes_sent != first_expected_bytes:
raise Exception("not correct number of bytes")
#send again
client1.send_data(first_client2_block, client2_address)
#tests whether we recieved right thing
test_utils.verify_data_sent_equals_data_received(
first_client2_block + first_client2_block, client2_output_filename)
bytes_sent = wide_area_network.get_total_bytes_sent()
print('bytes sent 2')
print(bytes_sent)
if bytes_sent != second_expected_bytes:
raise Exception("not correct number of bytes")
# Now send some more data to client 2.
client1.send_data_with_fin(second_client2_block, client2_address)
# Close the client 2 stream.
if not client2.received_fin:
raise Exception("Client 2 didnt't receive the fin")
#make sure data is correct, we do not have buffer, we just send everything
test_utils.verify_data_sent_equals_data_received(
first_client2_block + first_client2_block + second_client2_block,
client2_output_filename)
bytes_sent = wide_area_network.get_total_bytes_sent()
print('bytes sent 3')
print(bytes_sent)
if bytes_sent != third_expected_bytes:
raise Exception("not correct number of bytes")
def send_50_random_files(middlebox_module, testing_part_1):
""" Generates 50 random files that have between 6000-32000 bytes
of text.
For each file, between 1 and 6 times (random), chooses a random
pair of hosts on opposite sides of the WAN, and sends the
random file between them.
Makes sure that all files are sent properly.
"""
random.seed("168isthebesttho")
NUM_FILES = 50
total_count = 0
send_count = 0
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
middleboxes = [middlebox1, middlebox2]
# Base names for clients on either side of the WAN
client_address_bases = ["1.2.3.", "9.8.7."]
# Initialize and connect all clients:
mb1_clients = []
mb1_client_addr = []
mb2_clients = []
mb2_client_addr = []
for addr_base, middlebox in zip(client_address_bases, middleboxes):
for i in range(0, 8):
client_address = addr_base + str(i)
if middlebox == middlebox1:
client_i = client.EndHost(client_address, client_address,
middlebox1)
mb1_clients.append(client_i)
mb1_client_addr.append(client_address)
else:
client_i = client.EndHost(client_address, client_address,
middlebox2)
mb2_clients.append(client_i)
mb2_client_addr.append(client_address)
filename = "random.txt"
for i in range(NUM_FILES):
generate_random_file(filename)
with open(filename, "rb") as input_file:
input_data = input_file.read()
input_file.close()
for num_send in range(0, random.randint(1, 6)):
clientA_index = random.randint(0, 7)
clientB_index = random.randint(0, 7)
client_pair = [
mb1_clients[clientA_index], mb2_clients[clientB_index]
]
client_addr_pair = [
mb1_client_addr[clientA_index], mb2_client_addr[clientB_index]
]
# zipped_pair = list(zip(client_pair, client_addr_pair))
zipped_pair = [(client_pair[i], client_addr_pair[i])
for i in range(2)]
random.shuffle(zipped_pair)
sender = zipped_pair[0][0]
senderAddr = zipped_pair[0][1]
receiver = zipped_pair[1][0]
receiverAddr = zipped_pair[1][1]
sender.send_file(filename, receiverAddr)
# Make sure that the files have the same contents.
output_file_name = "{}-{}".format(receiverAddr, filename)
with open(output_file_name, "rb") as output_file:
result_data = output_file.read()
output_file.close()
# Remove the output file just created.
os.remove(output_file_name)
send_count += 1
if input_data == result_data:
total_count += 1
if total_count != send_count:
raise Exception(
"send_mutiple_files failed, because the all files" +
"received did not match the file sent. Files received correctly:" +
" {} and files sent are: {}\n".format(total_count, NUM_FILES))
### IMPLEMENTATION-SPECIFIC ###
# You should change the variable names here to match the class variables
# that cache the hash blocks' keys!
if middlebox1.buffers != middlebox2.buffers:
raise Exception(
"The WAN Optimizers don't have the same state at the end!")
def cross_sending(middlebox_module, testing_part_1):
""" Tests that a large file without a delimeter will be sent correctly
Only works for a a
"""
filename = "Diamond_top.txt"
block1 = "a" * 8000
block2 = "b" * 8000
block3 = "c" * 8000
expected_value = .88
if testing_part_1:
delimeter = ""
combo1 = [block1[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block2, block3]
combo2 = [block1[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block3, block2]
combo3 = [block2[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block1, block3]
combo4 = [block2[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block3, block1]
combo5 = [block3[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block1, block2]
combo6 = [block3[:8000 - len(filename) - len(client.FILENAME_DELIMITER)], block2, block1]
else:
delimeter = " straight chin suggestive of resolution pushed t"
combo1 = [delimeter, block1, block2, block3]
combo2 = [delimeter, block1, block3, block2]
combo3 = [delimeter, block2, block1, block3]
combo4 = [delimeter, block2, block3, block1]
combo5 = [delimeter, block3, block1, block2]
combo6 = [delimeter, block3, block2, block1]
combos = [combo1, combo2, combo3, combo4, combo5, combo6]
filename = "Diamond_top.txt"
files = []
for combo in combos:
files.append(delimeter.join(combo))
middlebox1 = middlebox_module.WanOptimizer()
middlebox2 = middlebox_module.WanOptimizer()
wide_area_network = wan.Wan(middlebox1, middlebox2)
# Initialize clients connected to middlebox 1.
client1_address = "1.2.3.4"
client2_address = "1.3.4.2"
client3_address = "1.4.2.3"
client1 = client.EndHost("client1", client1_address, middlebox1)
client2 = client.EndHost("client2", client2_address, middlebox1)
client3 = client.EndHost("client3", client3_address, middlebox1)
# Initialize clients connected to middlebox 2.
client4_address = "5.6.7.8"
client5_address = "5.8.7.6"
client6_address = "5.7.6.8"
client4 = client.EndHost("client4", client4_address, middlebox2)
client5 = client.EndHost("client5", client5_address, middlebox2)
client6 = client.EndHost("client6", client6_address, middlebox2)
bytes_in_sent_files = 0
for data in files:
f = open(filename, 'w')
f.write(data)
f.close()
past2 = wide_area_network.get_total_bytes_sent()
client1.send_file(filename, client4_address)
client5.send_file(filename, client2_address)
client3.send_file(filename, client6_address)
output_file_name = "{}-{}".format("client2", filename)
# Removing the output file just created
os.remove(output_file_name)
output_file_name = "{}-{}".format("client4", filename)
# Removing the output file just created
os.remove(output_file_name)
output_file_name = "{}-{}".format("client6", filename)
# Removing the output file just created
os.remove(output_file_name)
past = bytes_in_sent_files
bytes_in_sent_files += (len(data) + len(filename) + len(client.FILENAME_DELIMITER)) * 3
bytes_sent = wide_area_network.get_total_bytes_sent()
reduction = (float(bytes_in_sent_files - bytes_sent)
/ float(bytes_in_sent_files))
if (reduction < expected_value):
raise Exception("data_reduction_random_edit_file failed," +
" because reduciton ratio should be greater than " +
" {}, was {}.".format(expected_value, reduction))
