def run_one_experiment(local_log_folder, local_config_file): time1 = time.time() if os.path.exists(local_log_folder): os.system('rm -rf ' + local_log_folder) # print '***** QUITTING!!! Output folder already exists:', output_folder, '*******' # sys.exit(2) if local_log_folder.endswith('/'): local_log_folder = local_log_folder[:-1] # copy config files to record experiment configuration print 'Creating local log folder: ' + local_log_folder os.system('mkdir -p ' + local_log_folder) # ns_ids, lns_ids = get_node_ids(exp_config.local_ns_file, exp_config.local_lns_file) ns_ids, lns_ids = get_ns_lns_ids_config_file() remote_workload_config = os.path.split(exp_config.wfile)[1] REMOTE_NODE_CONFIG = 'node_config' REMOTE_UPDATE_TRACE = 'request_trace' remote_config_file = os.path.split(local_config_file)[1] os.system('date') os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file) os.system('./rmLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file + ' ' + exp_config.remote_gns_logs) if exp_config.clean_start: print 'Doing clean start of GNS ... ' os.system('./rmPaxosLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file + ' ' + exp_config.paxos_log_folder) os.system('./kill_mongodb.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.db_folder) os.system('./run_mongodb.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.remote_mongo_bin + ' ' + exp_config.db_folder + ' ' + str(exp_config.mongo_port)) print "Waiting for mongod process to start fully ..." os.system('sleep ' + str(exp_config.mongo_sleep)) # ensures mongo is fully running else: print "\nRestarting GNS from previous run ...\n" print 'Copy scripts and config files ...' from cp_scripts_configs import cp_scripts_configs cp_scripts_configs(exp_config.user, exp_config.ssh_key, ns_ids, lns_ids, exp_config.remote_gns_logs, local_config_file, exp_config.wfile, exp_config.node_config_folder, REMOTE_NODE_CONFIG) # print 'Copy workload to LNS ...' # copy workload for local name servers .... copy_workload.copy_workload(exp_config.user, exp_config.ssh_key, lns_ids, exp_config.update_trace, exp_config.remote_gns_logs, REMOTE_UPDATE_TRACE) # copy GNS jar file .... if exp_config.copy_jar: print 'Copy jar to remote folder ...' remote_jar_folder = os.path.split(exp_config.remote_jar_file)[0] print remote_jar_folder os.system('./rmcpJar.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file + ' ' + exp_config.local_jar_file + ' ' + remote_jar_folder + ' ' + exp_config.remote_jar_file) elif exp_config.download_jar: # url of jar file is hardcoded # location of jar file is hard coded os.system('./getJarS3.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file) # run name servers .... run_all_ns(exp_config.user, exp_config.ssh_key, ns_ids, exp_config.remote_gns_logs, remote_config_file, REMOTE_NODE_CONFIG) print 'Name servers running ...' try: #os.system('sleep ' + str(exp_config.ns_sleep)) print 'Waiting for name servers to start ' + str(exp_config.ns_sleep) + 'sec ..' sleep_for_time(exp_config.ns_sleep) except: print 'NS sleep interrupted. Starting LNS ...' # run local name servers .... run_all_lns(exp_config.user, exp_config.ssh_key, lns_ids, exp_config.remote_gns_logs, remote_config_file, REMOTE_NODE_CONFIG, REMOTE_UPDATE_TRACE, remote_workload_config) if exp_config.experiment_run_time == -1: print 'Experiment run time == -1. All name servers and local name servers started.' return elif exp_config.event_file is None: wait_exp_over() else: # NOTE: not sure if this works in emulation assert False handle_events(exp_config.event_file, exp_config.experiment_run_time, ns_ids, lns_ids) print 'Ending experiment ..' # ok os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file) from get_log import get_log get_log(exp_config.user, exp_config.ssh_key, ns_ids, lns_ids, local_log_folder, exp_config.remote_gns_logs) # ensure that this does not change print 'Output stats ...' stats_folder = local_log_folder + '_stats' if local_log_folder.endswith('/'): stats_folder = local_log_folder[:-1] + '_stats' parse_log(local_log_folder, stats_folder) time2 = time.time() diff = time2 - time1 print 'TOTAL EXPERIMENT TIME:', int(diff / 60), 'minutes' sys.exit(2)
def run_one_experiment(output_folder, exp_time_sec, lookupTrace, updateTrace): time1 = time.time() if os.path.exists(output_folder): os.system('rm -rf ' + output_folder) # print '***** QUITTING!!! Output folder already exists:', output_folder, '*******' # sys.exit(2) if output_folder.endswith('/'): output_folder = output_folder[:-1] # copy config files to record experiment configuration print 'Creating local output folder: ' + output_folder os.system('mkdir -p ' + output_folder) #os.system(generate_config_script + ' ' + str(exp_config.load)) # os.system('cp local-name-server.py name-server.py exp_config.py ' + output_folder) # used for workload generation generate_multinode_config_file(exp_config.load) os.system('date') os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file) os.system('./rmLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + exp_config.paxos_log_folder + ' ' + exp_config.gns_output_logs) # ./cpPl.sh os.system('./cpPl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + exp_config.config_folder + ' ' + exp_config.gns_output_logs) # # ./cpWorkload.sh os.system('./cpWorkload.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.lns_file + ' ' + exp_config.gns_output_logs + ' ' + lookupTrace + ' ' + updateTrace) #os.system('./cpNameActives.sh ' + name_actives_local + ' ' + name_actives_remote + ' ') if exp_config.copy_jar: remote_jar_folder = os.path.split(exp_config.jar_file_remote)[0] print remote_jar_folder os.system('./rmcpJar.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + exp_config.jar_file + ' ' + remote_jar_folder + ' ' + exp_config.jar_file_remote) elif exp_config.download_jar: # url of jar file is hardcoded # location of jar file is hard coded os.system('./getJarS3.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file) #run mongo db if exp_config.run_db: os.system('./kill_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.db_folder) os.system('./run_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.mongo_bin + ' ' + exp_config.db_folder) os.system( 'sleep ' + str(exp_config.mongo_sleep)) # ensures mongo is fully running print "Waiting for mongod process to start fully ..." elif exp_config.restore_db: os.system('./kill_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.db_folder) os.system('./restore_backup.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.db_folder_backup + ' ' + exp_config.db_folder) os.system('./run_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.mongo_bin + ' ' + exp_config.db_folder) os.system( 'sleep ' + str(exp_config.mongo_sleep)) # ensures mongo is fully running # ./name-server.sh #os.system('./name-server.sh') run_all_ns(exp_config.gns_output_logs) print 'Name servers running ...' try: #os.system('sleep ' + str(exp_config.ns_sleep)) print 'Waiting for NS to load all records for ' + str( exp_config.ns_sleep) + 'sec ..' sleep_for_time(exp_config.ns_sleep) except: print 'NS sleep interrupted. Starting LNS ...' # ./local-name-server.sh run_all_lns(exp_config.gns_output_logs) # this is the excess wait after given duration of experiment excess_wait = exp_config.extra_wait print 'LNS running. Now wait for ', (exp_time_sec + excess_wait) / 60, 'min ...' # sleep for experiment_time if exp_time_sec == -1: return sleep_time = 0 while sleep_time < exp_time_sec + excess_wait: os.system('sleep 60') sleep_time += 60 if sleep_time % 60 == 0: print 'Time = ', sleep_time / 60, 'min /', ( exp_time_sec + excess_wait) / 60, 'min' # ./endExperiment.sh output_folder print 'Ending experiment ..' os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file) os.system('./getLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + output_folder + ' ' + exp_config.gns_output_logs) # ./parse_log.py output_folder #os.system('logparse/parse_log.py ' + output_folder) stats_folder = output_folder + '_stats' if output_folder.endswith('/'): stats_folder = output_folder[:-1] + '_stats' parse_log(output_folder, stats_folder, False) time2 = time.time() diff = time2 - time1 print 'TOTAL EXPERIMENT TIME:', int(diff / 60), 'minutes' sys.exit(2)
def run_one_experiment(local_log_folder, local_config_file): time1 = time.time() if os.path.exists(local_log_folder): os.system('rm -rf ' + local_log_folder) # print '***** QUITTING!!! Output folder already exists:', output_folder, '*******' # sys.exit(2) if local_log_folder.endswith('/'): local_log_folder = local_log_folder[:-1] # copy config files to record experiment configuration print 'Creating local log folder: ' + local_log_folder os.system('mkdir -p ' + local_log_folder) # ns_ids, lns_ids = get_node_ids(exp_config.local_ns_file, exp_config.local_lns_file) ns_ids, lns_ids = get_ns_lns_ids_config_file() remote_workload_config = os.path.split(exp_config.wfile)[1] REMOTE_NODE_CONFIG = 'node_config' REMOTE_UPDATE_TRACE = 'request_trace' remote_config_file = os.path.split(local_config_file)[1] os.system('date') os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file) os.system('./rmLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file + ' ' + exp_config.remote_gns_logs) if exp_config.clean_start: print 'Doing clean start of GNS ... ' os.system('./rmPaxosLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file + ' ' + exp_config.paxos_log_folder) os.system('./kill_mongodb.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.db_folder) os.system('./run_mongodb.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.remote_mongo_bin + ' ' + exp_config.db_folder + ' ' + str(exp_config.mongo_port)) print "Waiting for mongod process to start fully ..." os.system( 'sleep ' + str(exp_config.mongo_sleep)) # ensures mongo is fully running else: print "\nRestarting GNS from previous run ...\n" print 'Copy scripts and config files ...' from cp_scripts_configs import cp_scripts_configs cp_scripts_configs(exp_config.user, exp_config.ssh_key, ns_ids, lns_ids, exp_config.remote_gns_logs, local_config_file, exp_config.wfile, exp_config.node_config_folder, REMOTE_NODE_CONFIG) # print 'Copy workload to LNS ...' # copy workload for local name servers .... copy_workload.copy_workload(exp_config.user, exp_config.ssh_key, lns_ids, exp_config.update_trace, exp_config.remote_gns_logs, REMOTE_UPDATE_TRACE) # copy GNS jar file .... if exp_config.copy_jar: print 'Copy jar to remote folder ...' remote_jar_folder = os.path.split(exp_config.remote_jar_file)[0] print remote_jar_folder os.system('./rmcpJar.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file + ' ' + exp_config.local_jar_file + ' ' + remote_jar_folder + ' ' + exp_config.remote_jar_file) elif exp_config.download_jar: # url of jar file is hardcoded # location of jar file is hard coded os.system('./getJarS3.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file) # run name servers .... run_all_ns(exp_config.user, exp_config.ssh_key, ns_ids, exp_config.remote_gns_logs, remote_config_file, REMOTE_NODE_CONFIG) print 'Name servers running ...' try: #os.system('sleep ' + str(exp_config.ns_sleep)) print 'Waiting for name servers to start ' + str( exp_config.ns_sleep) + 'sec ..' sleep_for_time(exp_config.ns_sleep) except: print 'NS sleep interrupted. Starting LNS ...' # run local name servers .... run_all_lns(exp_config.user, exp_config.ssh_key, lns_ids, exp_config.remote_gns_logs, remote_config_file, REMOTE_NODE_CONFIG, REMOTE_UPDATE_TRACE, remote_workload_config) if exp_config.experiment_run_time == -1: print 'Experiment run time == -1. All name servers and local name servers started.' return elif exp_config.event_file is None: wait_exp_over() else: # NOTE: not sure if this works in emulation assert False handle_events(exp_config.event_file, exp_config.experiment_run_time, ns_ids, lns_ids) print 'Ending experiment ..' # ok os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.local_ns_file + ' ' + exp_config.local_lns_file) from get_log import get_log get_log(exp_config.user, exp_config.ssh_key, ns_ids, lns_ids, local_log_folder, exp_config.remote_gns_logs) # ensure that this does not change print 'Output stats ...' stats_folder = local_log_folder + '_stats' if local_log_folder.endswith('/'): stats_folder = local_log_folder[:-1] + '_stats' parse_log(local_log_folder, stats_folder) time2 = time.time() diff = time2 - time1 print 'TOTAL EXPERIMENT TIME:', int(diff / 60), 'minutes' sys.exit(2)
def run_exp(): from kill_local import kill_local_gnrs kill_local_gnrs() node_config = exp_config.node_config # folder where GNS logs are output log_output_folder = os.path.join(exp_config.output_folder, exp_config.DEFAULT_GNS_OUTPUT_FOLDER) # folder where GNS stats are output stats_folder = os.path.join(exp_config.output_folder, exp_config.DEFAULT_STATS_FOLDER) print 'Clearing old GNS logs ..' if not os.path.exists(log_output_folder): os.system('mkdir -p ' + log_output_folder) os.system('rm -rf ' + log_output_folder + '/*') if exp_config.clean_start: print 'Doing clean start ...' os.system('rm -rf ' + exp_config.paxos_log_folder + '/*') os.system(script_folder + '/kill_mongodb_local.sh ' + exp_config.mongodb_data_folder) os.system(script_folder + '/run_mongodb_local.sh ' + exp_config.mongo_bin_folder + ' ' + exp_config.mongodb_data_folder + ' ' + str(exp_config.mongo_port)) else: print 'Resuming gns ...' # generate config file write_local_config_file(exp_config.node_config, exp_config.num_ns, exp_config.num_lns, exp_config.const_latency_value, exp_config.random_node_ids) # generate workloads # todo move workload generation from here # generate_all_traces() #os.system('sleep 100') #os.system('pssh -h hosts.txt "killall -9 java"') f = open(node_config) first_lns = False for line in f: tokens = line.split() id = tokens[0] hostname = tokens[2] if tokens[1] == 'yes': if is_failed_node(id): print 'NODE FAILED = ', id continue name_server.run_name_server(id, log_output_folder) else: if first_lns is False: time.sleep(exp_config.ns_sleep) # sleep so that name servers load name records into DB first_lns = True update_trace = None if exp_config.is_experiment_mode and exp_config.experiment_run_time >= 0: update_trace = get_update_trace(exp_config.trace_folder, id) local_name_server.run_local_name_server(id, log_output_folder, update_trace) # kill local if not exp_config.is_experiment_mode or exp_config.experiment_run_time < 0: print 'Not experiment mode. GNS running.' return print 'Sleeping until experiment finishes ...' time.sleep(exp_config.experiment_run_time + exp_config.extra_wait) kill_local_gnrs() # if exp_config.output_folder.endswith('/'): # stats_folder = exp_config.output_folder[:-1] + '_stats' parse_log(log_output_folder, stats_folder) # parse logs and generate output # not doing restart stuff right now sys.exit(2) restart_period = 20 number_restarts = 1 restart_node = '2' for i in range(number_restarts): time.sleep(restart_period) print 'Restarting node ' + restart_node run_name_server(restart_node, log_output_folder, node_config) print 'Done'
def run_one_experiment(output_folder, exp_time_sec, lookupTrace, updateTrace): time1 = time.time() if os.path.exists(output_folder): os.system('rm -rf ' + output_folder) # print '***** QUITTING!!! Output folder already exists:', output_folder, '*******' # sys.exit(2) if output_folder.endswith('/'): output_folder = output_folder[:-1] # copy config files to record experiment configuration print 'Creating local output folder: ' + output_folder os.system('mkdir -p ' + output_folder) #os.system(generate_config_script + ' ' + str(exp_config.load)) # os.system('cp local-name-server.py name-server.py exp_config.py ' + output_folder) # used for workload generation generate_multinode_config_file(exp_config.load) os.system('date') os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file) os.system('./rmLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + exp_config.paxos_log_folder + ' ' + exp_config.gns_output_logs) # ./cpPl.sh os.system('./cpPl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + exp_config.config_folder + ' ' + exp_config.gns_output_logs) # # ./cpWorkload.sh os.system('./cpWorkload.sh '+ exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.lns_file + ' ' + exp_config.gns_output_logs + ' ' + lookupTrace + ' ' + updateTrace) #os.system('./cpNameActives.sh ' + name_actives_local + ' ' + name_actives_remote + ' ') if exp_config.copy_jar: remote_jar_folder = os.path.split(exp_config.jar_file_remote)[0] print remote_jar_folder os.system('./rmcpJar.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + exp_config.jar_file + ' ' + remote_jar_folder + ' ' + exp_config.jar_file_remote) elif exp_config.download_jar: # url of jar file is hardcoded # location of jar file is hard coded os.system('./getJarS3.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file) #run mongo db if exp_config.run_db: os.system('./kill_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.db_folder) os.system('./run_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.mongo_bin + ' ' + exp_config.db_folder) os.system('sleep ' + str(exp_config.mongo_sleep)) # ensures mongo is fully running print "Waiting for mongod process to start fully ..." elif exp_config.restore_db: os.system('./kill_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.db_folder) os.system('./restore_backup.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.db_folder_backup + ' ' + exp_config.db_folder) os.system('./run_mongodb_pl.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.mongo_bin + ' ' + exp_config.db_folder) os.system('sleep ' + str(exp_config.mongo_sleep)) # ensures mongo is fully running # ./name-server.sh #os.system('./name-server.sh') run_all_ns(exp_config.gns_output_logs) print 'Name servers running ...' try: #os.system('sleep ' + str(exp_config.ns_sleep)) print 'Waiting for NS to load all records for ' + str(exp_config.ns_sleep) + 'sec ..' sleep_for_time(exp_config.ns_sleep) except: print 'NS sleep interrupted. Starting LNS ...' # ./local-name-server.sh run_all_lns(exp_config.gns_output_logs) # this is the excess wait after given duration of experiment excess_wait = exp_config.extra_wait print 'LNS running. Now wait for ', (exp_time_sec + excess_wait)/60, 'min ...' # sleep for experiment_time if exp_time_sec == -1: return sleep_time = 0 while sleep_time < exp_time_sec + excess_wait: os.system('sleep 60') sleep_time += 60 if sleep_time % 60 == 0: print 'Time = ', sleep_time/60, 'min /', (exp_time_sec + excess_wait)/60, 'min' # ./endExperiment.sh output_folder print 'Ending experiment ..' os.system('./killJava.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file) os.system('./getLog.sh ' + exp_config.user + ' ' + exp_config.ssh_key + ' ' + exp_config.ns_file + ' ' + exp_config.lns_file + ' ' + output_folder + ' ' + exp_config.gns_output_logs) # ./parse_log.py output_folder #os.system('logparse/parse_log.py ' + output_folder) stats_folder = output_folder + '_stats' if output_folder.endswith('/'): stats_folder = output_folder[:-1] + '_stats' parse_log(output_folder, stats_folder, False) time2 = time.time() diff = time2 - time1 print 'TOTAL EXPERIMENT TIME:', int(diff/60), 'minutes' sys.exit(2)