def run(self):
print("Running experiment 2")
assert check_if_all_servers_are_up(), ("There seems to be a problem that not all servers are up!")
# Check if servers are up!
self._test_if_server_is_up(
[
CLIENT['Client1'],
SERVER['Server1'], SERVER['Server2']
]
)
print("Stopping any memcached instances")
self.stop_memcached_servers(
memcached_addresses=[SERVER['Server1'], SERVER['Server2']]
)
time.sleep(2)
print("Setting up..")
memcached_servers = self.start_memcached_servers(
memcached_addresses=[
SERVER['Server1'], SERVER['Server2']
],
port=self.port
)
time.sleep(2)
print("Prepopulating")
self.prepopulate_memcached_servers([CLIENT['Client1']], SERVER_IP['Server1'], port=self.port, time=self.population_time)
self.prepopulate_memcached_servers([CLIENT['Client1']], SERVER_IP['Server2'], port=self.port, time=self.population_time)
time.sleep(self.population_time)
self._create_log_directories([CLIENT['Client1']], self.remote_logdir)
# Setting the reads to writes ratio
for self.writes in ["0", "1"]:
self.reads = "1" if self.writes == "0" else "0" # Exactly only reads or writes
for virtual_client_threads in self.virtual_clients_per_thread:
for i in range(self.repetitions):
print("CONFIGURATION VC!: ", virtual_client_threads)
start_time = time.time()
all_client_processes = []
for client_number in ['Client1']:
def runner_function():
logfile = self.remote_logdir + self.name + "virtualclients_{}__rep_{}_client_{}__writes_{}.txt".format(
virtual_client_threads, i, client_number, self.writes)
self._create_log_files([CLIENT[client_number]], logfile)
print("CONFIGURATION : ", logfile)
start_time = time.time()
# print("Running right now (first) ...", )
print("Running memtier on the server 1!")
# TODO: Testing this with one-to-one to check if replicate is the case
self.run_memtiered_servers(
client_ips=[CLIENT[client_number]],
server_ip=SERVER_IP['Server1'],
port=self.port,
logfile=logfile + "_Server1",
time=self.time,
write_ratio=self.writes,
read_ratio=self.reads,
virtual_clients_per_thread=virtual_client_threads,
threads=self.threads_per_memtier_instance
)
print("Running memtier on the server 2!")
self.run_memtiered_servers(
client_ips=[CLIENT[client_number]],
server_ip=SERVER_IP['Server2'],
port=self.port,
logfile=logfile + "_Server2",
time=self.time,
write_ratio=self.writes,
read_ratio=self.reads,
virtual_clients_per_thread=virtual_client_threads,
threads=self.threads_per_memtier_instance
)
print("Running right now (second) ...", time.time() - start_time)
print("Done running memtier on the server!")
p = Process(target=runner_function)
p.start()
all_client_processes.append(p)
time.sleep(self.time + self.time // 5) # TODO: Timeout for as long as the experiments run
print("Total runtime is: ", time.time() - start_time)
# Join all processes again!
[p.join() for p in all_client_processes]
self.copy_logs_to_local(
[CLIENT['Client1']],
self.remote_logdir,
self.local_logdir
)
time.sleep(10)
# kill all memcached services
self.stop_memcached_servers(
memcached_addresses=[SERVER['Server1'], SERVER['Server2']]
)
def run(self):
print("Running experiment 1")
assert check_if_all_servers_are_up(), ("There seems to be a problem that not all servers are up!")
self._test_if_server_is_up([CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3'], SERVER['Server1']])
print("Stopping any memcached instances")
self.stop_memcached_servers(
memcached_addresses=[SERVER['Server1']]
)
time.sleep(2)
print("Setting up..")
memcached_servers = self.start_memcached_servers(
memcached_addresses=[SERVER['Server1']],
port=self.port
)
time.sleep(2)
print("Prepopulating")
self.prepopulate_memcached_servers([CLIENT['Client1']], SERVER_IP['Server1'], port=self.port, time=self.population_time)
time.sleep(self.population_time)
self._create_log_directories([CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3']], self.remote_logdir)
for virtual_client_threads in self.virtual_clients_per_thread:
for i in range(self.repetitions):
# FORK JOIN for all individual clients
print("CONFIGURATION VC!: ", virtual_client_threads)
all_client_processes = []
for client_number in ['Client1', 'Client2', 'Client3']:
def runner_function():
logfile = self.remote_logdir + "virtualclients_{}__rep_{}_client_{}_writes_{}.txt".format(virtual_client_threads, i, client_number, self.writes)
self._create_log_files([CLIENT[client_number]], logfile)
print("CONFIGURATION : ", logfile)
print("Running memtier on the server!")
self.run_memtiered_servers(
client_ips=[CLIENT[client_number]],
server_ip=SERVER_IP['Server1'],
port=self.port,
logfile=logfile,
time=self.time,
write_ratio=self.writes,
read_ratio=self.reads,
virtual_clients_per_thread=virtual_client_threads,
threads=self.threads_per_memtier_instance
)
print("Done running memtier on the server!")
p = Process(target=runner_function)
p.start()
all_client_processes.append(p)
time.sleep(self.time) # TODO: Timeout for as long as the experiments run
# Join all processes again!
[p.join() for p in all_client_processes]
self.copy_logs_to_local(
[CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3']],
self.remote_logdir,
self.local_logdir
)
time.sleep(20)
# kill all memcached services
self.stop_memcached_servers(
memcached_addresses=[SERVER['Server1']]
)
def run(self):
print("Running experiment 1")
assert check_if_all_servers_are_up(), ("There seems to be a problem that not all servers are up!")
self._test_if_server_is_up(
[
CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3'],
MIDDLEWARE['Middleware1'], MIDDLEWARE['Middleware2'],
SERVER['Server1'], SERVER['Server2'], SERVER['Server3']
]
)
# TODO: Compile, and upload the java jar to the middleware!
print("Compile and upload middleware to VM...")
upload_jar([MIDDLEWARE['Middleware1']], compile=True)
upload_jar([MIDDLEWARE['Middleware2']], compile=True)
print("Stopping any memcached instances")
self.stop_memcached_servers(
memcached_addresses=[SERVER['Server1'], SERVER['Server2'], SERVER['Server3']]
)
time.sleep(2)
print("Setting up memcached..")
self.start_memcached_servers(
memcached_addresses=[SERVER['Server1'], SERVER['Server2'], SERVER['Server3']],
port=self.port
)
time.sleep(5)
server_strings = SERVER_IP['Server1'] + ":" + str(self.port) + " " + \
SERVER_IP['Server2'] + ":" + str(self.port) + " " + \
SERVER_IP['Server3'] + ":" + str(self.port)
print("Server strings are: (USED BY MIDDLEWARE LATER)", server_strings)
print("Prepopulating")
self.prepopulate_memcached_servers([CLIENT['Client1']], SERVER_IP['Server1'], port=self.port, time=self.population_time)
self.prepopulate_memcached_servers([CLIENT['Client2']], SERVER_IP['Server2'], port=self.port, time=self.population_time)
self.prepopulate_memcached_servers([CLIENT['Client3']], SERVER_IP['Server3'], port=self.port, time=self.population_time)
time.sleep(self.population_time)
self._create_log_directories(
[CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3']],
self.remote_logdir
)
# Iterate through keysizes
for keysize in self.reads:
print("Keysize is: ", keysize)
for sharding in self.sharding:
print("Sharding: ", sharding)
assert type(self.worker_threads_per_middleware) == int, ("NO", self.worker_threads_per_middleware)
assert type(self.virtual_clients_per_thread) == int, ("NO 2", self.virtual_clients_per_thread)
for i in range(self.repetitions):
# FORK JOIN for all individual clients
print("CONFIGURATION VC!: ", self.virtual_clients_per_thread)
print("Stopping any middleware instances")
self.stop_middleware_servers(
middleware_addresses=[MIDDLEWARE['Middleware1']]
)
self.stop_middleware_servers(
middleware_addresses=[MIDDLEWARE['Middleware2']]
)
time.sleep(5)
# Generating the server strings:
print("Setting up the middleware...")
self.start_middleware_servers(
middleware_addresses=[MIDDLEWARE['Middleware1'], MIDDLEWARE['Middleware2']],
middleware_localips=[MIDDLEWARE_IP['Middleware1'], MIDDLEWARE_IP['Middleware2']],
listening_port=self.middleware_port,
server_strings=server_strings,
middleware_threads=self.worker_threads_per_middleware,
sharding=sharding
)
time.sleep(10)
self._create_log_directories(
[MIDDLEWARE['Middleware1']],
self.remote_logdir
)
self._create_log_directories(
[MIDDLEWARE['Middleware2']],
self.remote_logdir
)
time.sleep(4)
start_time = time.time()
all_client_processes = []
for client_number in ['Client1', 'Client2', 'Client3']:
def runner_function():
logfile = self.remote_logdir + self.name + "_" + "multikeysize_{}_middleware1__rep_{}_client_{}_sharding_{}_middlewarethreads_{}.txt".format(
keysize, i, client_number, sharding, self.worker_threads_per_middleware)
self._create_log_files([CLIENT[client_number]], logfile)
print("CONFIGURATION : ", logfile)
print("Running memtier on the server 1!")
self.run_memtiered_servers(
client_ips=[CLIENT[client_number]],
server_ip=MIDDLEWARE_IP['Middleware1'],
port=self.middleware_port,
logfile=logfile,
time=self.time,
write_ratio=self.writes,
read_ratio=keysize,
virtual_clients_per_thread=self.virtual_clients_per_thread,
threads=self.threads_per_memtier_instance
)
logfile = self.remote_logdir + "_" + self.name + "_" + "multikeysize_{}_middleware2__rep_{}_client_{}_sharding_{}_middlewarethreads_{}.txt".format(
keysize, i, client_number, sharding, self.worker_threads_per_middleware)
self._create_log_files([CLIENT[client_number]], logfile)
print("CONFIGURATION : ", logfile)
print("Running memtier on the server 2!")
self.run_memtiered_servers(
client_ips=[CLIENT[client_number]],
server_ip=MIDDLEWARE_IP['Middleware2'],
port=self.middleware_port,
logfile=logfile,
time=self.time,
write_ratio=self.writes,
read_ratio=keysize,
virtual_clients_per_thread=self.virtual_clients_per_thread,
threads=self.threads_per_memtier_instance
)
print("Done running memtier on the server!")
p = Process(target=runner_function)
p.start()
all_client_processes.append(p)
time.sleep(1)
time.sleep(self.time + self.time // 3)
# Join all processes again!
print("Starting to copy... Took so many seconds", time.time() - start_time)
self.copy_logs_to_local(
[
MIDDLEWARE['Middleware1']
],
self.remote_logdir,
self.local_logdir + "Middleware1_multikeygetsize_{}_rep_{}__sharding_{}__middlewareworkerthreads_{}/".format(
keysize, i, sharding, self.worker_threads_per_middleware)
)
self.copy_logs_to_local(
[
MIDDLEWARE['Middleware2']
],
self.remote_logdir,
self.local_logdir + "Middleware2_multikeygetsize_{}_rep_{}__sharding_{}_middlewareworkerthreads_{}/".format(
keysize, i, sharding, self.worker_threads_per_middleware)
)
time.sleep(10)
# Join all processes again!
[p.join(self.time // 3) for p in all_client_processes]
print("Starting to copy... Took so many seconds", time.time() - start_time)
self.copy_logs_to_local(
[
CLIENT['Client1'],
CLIENT['Client2'],
CLIENT['Client3']
],
self.remote_logdir,
self.local_logdir
)
time.sleep(30)
# kill all memcached services
self.stop_memcached_servers(
memcached_addresses=[SERVER['Server1']]
)
def run(self):
print("Running experiment 1")
assert check_if_all_servers_are_up(), (
"There seems to be a problem that not all servers are up!")
self._test_if_server_is_up([
CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3'],
MIDDLEWARE['Middleware1'], MIDDLEWARE['Middleware2'],
SERVER['Server1']
])
# TODO: Compile, and upload the java jar to the middleware!
print("Compile and upload middleware to VM...")
upload_jar([MIDDLEWARE['Middleware1']], compile=True)
upload_jar([MIDDLEWARE['Middleware2']], compile=True)
for number_of_servers in self.number_of_servers:
# All the server stuff
print("Prepopulating...")
self.stop_servers(number_of_servers)
self.start_servers(number_of_servers)
self.prepopulate_servers(
number_of_servers) # Simply prepopulate all servers
print("Prepopulated")
server_strings = self.get_server_string(number_of_servers)
self._create_log_directories(
[CLIENT['Client1'], CLIENT['Client2'], CLIENT['Client3']],
self.remote_logdir)
for middlewares in self.number_of_middlewares:
for self.writes in ["0", "1"]: # "0"
self.reads = "1" if self.writes == "0" else "0" # Exactly only reads or writes
for middleware_workerthread in self.worker_threads_per_middleware:
assert type(middleware_workerthread) == int, (
"NO", middleware_workerthread)
assert type(self.virtual_clients_per_thread) == int, (
"NO 2", self.virtual_client_threads)
for i in range(self.repetitions):
# FORK JOIN for all individual clients
print("CONFIGURATION VC!: ",
self.virtual_clients_per_thread)
self.stop_middlewares(middlewares)
time.sleep(10)
print("Remote logdir is: ", self.remote_logdir)
self._create_log_directories([
MIDDLEWARE['Middleware1'],
MIDDLEWARE['Middleware2']
], self.remote_logdir)
time.sleep(10)
# Generating the server strings:
self.start_middlwares(middlewares, server_strings,
middleware_workerthread)
time.sleep(20)
start_time = time.time()
all_client_processes = []
for client_number in [
'Client1', 'Client2', 'Client3'
]:
def runner_function():
self.run_client_on_middleware(
middlewares,
client_number,
middlewares=middlewares,
number_of_servers=number_of_servers,
middleware_workerthread=
middleware_workerthread,
i=i,
writes=self.writes)
p = Process(target=runner_function)
p.start()
all_client_processes.append(p)
time.sleep(5)
time.sleep(self.time + self.time // 4)
# Join all processes again!
print("Starting to copy... Took so many seconds",
time.time() - start_time)
self.copy_logs_to_local(
[MIDDLEWARE['Middleware1']],
self.remote_logdir, self.local_logdir +
"Middleware1_threads_{}_middlewares_{}_servers_{}_rep_{}__writes_{}/"
.format(middleware_workerthread, middlewares,
number_of_servers, i, self.writes))
if middlewares == 2:
self.copy_logs_to_local(
[MIDDLEWARE['Middleware2']],
self.remote_logdir, self.local_logdir +
"Middleware2_threads_{}_middlewares_{}_servers_{}_rep_{}__writes_{}/"
.format(middleware_workerthread,
middlewares, number_of_servers, i,
self.writes))
time.sleep(10)
[
p.join(self.time // 4)
for p in all_client_processes
]
self.copy_logs_to_local([
CLIENT['Client1'], CLIENT['Client2'],
CLIENT['Client3']
], self.remote_logdir, self.local_logdir)
time.sleep(20)
# kill all memcached services
self.stop_servers(number_of_servers)