def run_hadoop_baseline(pm, nodes_used, workload, schedule):
''' This is a single run to gather the baseline resource
and performance profile for Cassandra '''
#Spawn and setup two clusters of hadoop
for i in range(1,3):
###### SPAWN VMS ##########
hadoop_utils.spawn_hadoop_vms(pm["hadoop:num_hadoop"],
pm["exp_number"],
pm["hadoop:placement"])
time.sleep(120)
####### START PROCESSES ##########
hadoopStartProcess = Process(target=hadoop_utils.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"], schedule))
hadoopStartProcess.start()
hadoopStartProcess.join()
time.sleep(120)
####### LOAD DATA ##########
hadoopLoadProcess = Process(target=hadoop_utils.hadoop_load_workload, args=(pm, pm["exp_number"], workload ))
hadoopLoadProcess.start()
hadoopLoadProcess.join()
time.sleep(30)
# Next Cluster Experment ID
pm["exp_number"] = pm["exp_number"] + 1
####### RUN WORKLOAD ##########
processList = []
run_mpstat(nodes_used, pm["exp_number"])
run_bwmon(nodes_used, pm["exp_number"])
run_iostat(nodes_used, pm["exp_number"])
hadoopRunProcess_1 = Process(target=multi_hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"] -1, workload))
time.sleep(40)
hadoopRunProcess_2 = Process(target=multi_hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"] , workload))
time.sleep(40)
hadoopRunProcess_1.start()
processList.append(hadoopRunProcess_1)
hadoopRunProcess_2.start()
processList.append(hadoopRunProcess_2)
#hadoopRunProcess.join()
time.sleep(180)
for process in processList:
process.join()
retreive_mpstat_results(nodes_used, pm["exp_number"])
retreive_bwmon_results(nodes_used, pm["exp_number"])
retreive_iostat_results(nodes_used, pm["exp_number"])
def run_hadoop_baseline(pm, nodes_used, workload, schedule):
''' This is a single run to gather the baseline resource
and performance profile for Cassandra '''
###### SPAWN VMS ##########
hadoop_utils.spawn_hadoop_vms(pm["hadoop:num_hadoop"],
pm["exp_number"],
pm["hadoop:placement"])
time.sleep(120)
####### START PROCESSES ##########
hadoopStartProcess = Process(target=hadoop_utils.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"], schedule))
hadoopStartProcess.start()
hadoopStartProcess.join()
time.sleep(120)
####### LOAD DATA ##########
hadoopLoadProcess = Process(target=hadoop_utils.hadoop_load_workload, args=(pm, pm["exp_number"], workload ))
hadoopLoadProcess.start()
hadoopLoadProcess.join()
time.sleep(30)
####### RUN WORKLOAD ##########
run = raw_input("Press Enter to run experiment ")
run_mpstat(nodes_used, pm["exp_number"])
run_bwmon(nodes_used, pm["exp_number"])
run_iostat(nodes_used, pm["exp_number"])
hadoopRunProcess = Process(target=hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"], workload))
time.sleep(40)
hadoopRunProcess.start()
hadoopRunProcess.join()
time.sleep(180)
retreive_mpstat_results(nodes_used, pm["exp_number"])
retreive_bwmon_results(nodes_used, pm["exp_number"])
retreive_iostat_results(nodes_used, pm["exp_number"])
def run_hadoop_baseline(pm, nodes_used, workload, schedule):
''' This is a single run to gather the baseline resource
and performance profile for hadoop '''
###### SPAWN VMS ##########
spawnProcessList = []
hadoopSpawnProcess1 = Process (target=hadoop_utils.spawn_hadoop_vms(pm["hadoop:num_hadoop"],
pm["exp_number"],
pm["hadoop:placement"]))
hadoopSpawnProcess1.start()
spawnProcessList.append(hadoopSpawnProcess1)
hadoopSpawnProcess2 = Process (target=hadoop_utils_2.spawn_hadoop_vms(pm["hadoop:num_hadoop"],
pm["exp_number"]+1,
pm["hadoop:placement"]))
hadoopSpawnProcess2.start()
spawnProcessList.append(hadoopSpawnProcess2)
for process in spawnProcessList:
process.join()
time.sleep(200)
####### START PROCESSES ##########
startProcessList = []
hadoopStartProcess1 = Process(target=hadoop_utils.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"],pm, schedule))
hadoopStartProcess1.start()
startProcessList.append(hadoopStartProcess1)
hadoopStartProcess2 = Process(target=hadoop_utils_2.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"]+1,pm, schedule))
hadoopStartProcess2.start()
startProcessList.append(hadoopStartProcess2)
for process in startProcessList:
process.join()
time.sleep(20)
####### LOAD DATA ##########
loadProcessList = []
hadoopLoadProcess1 = Process(target=hadoop_utils.hadoop_load_workload, args=(pm, pm["exp_number"], workload ))
hadoopLoadProcess1.start()
loadProcessList.append(hadoopLoadProcess1)
hadoopLoadProcess2 = Process(target=hadoop_utils_2.hadoop_load_workload, args=(pm, pm["exp_number"]+1, workload ))
hadoopLoadProcess2.start()
loadProcessList.append(hadoopLoadProcess2)
for process in loadProcessList:
process.join()
time.sleep(10)
####### RUN WORKLOAD ##########
run_mpstat(nodes_used, pm["exp_number"])
run_bwmon(nodes_used, pm["exp_number"])
run_iostat(nodes_used, pm["exp_number"])
runProcessList = []
hadoopRunProcess1 = Process(target=hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"], workload))
hadoopRunProcess1.start()
runProcessList.append(hadoopRunProcess1)
hadoopRunProcess2 = Process(target=hadoop_utils_2.hadoop_run_workload, args=(pm, pm["exp_number"]+1, workload))
hadoopRunProcess2.start()
runProcessList.append(hadoopRunProcess2)
for process in runProcessList:
process.join()
retreive_mpstat_results(nodes_used, pm["exp_number"])
retreive_bwmon_results(nodes_used, pm["exp_number"])
retreive_iostat_results(nodes_used, pm["exp_number"])
