def run_hadoop_baseline(pm, nodes_used, workload, schedule): ''' This is a single run to gather the baseline resource and performance profile for Cassandra ''' #Spawn and setup two clusters of hadoop for i in range(1,3): ###### SPAWN VMS ########## hadoop_utils.spawn_hadoop_vms(pm["hadoop:num_hadoop"], pm["exp_number"], pm["hadoop:placement"]) time.sleep(120) ####### START PROCESSES ########## hadoopStartProcess = Process(target=hadoop_utils.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"], schedule)) hadoopStartProcess.start() hadoopStartProcess.join() time.sleep(120) ####### LOAD DATA ########## hadoopLoadProcess = Process(target=hadoop_utils.hadoop_load_workload, args=(pm, pm["exp_number"], workload )) hadoopLoadProcess.start() hadoopLoadProcess.join() time.sleep(30) # Next Cluster Experment ID pm["exp_number"] = pm["exp_number"] + 1 ####### RUN WORKLOAD ########## processList = [] run_mpstat(nodes_used, pm["exp_number"]) run_bwmon(nodes_used, pm["exp_number"]) run_iostat(nodes_used, pm["exp_number"]) hadoopRunProcess_1 = Process(target=multi_hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"] -1, workload)) time.sleep(40) hadoopRunProcess_2 = Process(target=multi_hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"] , workload)) time.sleep(40) hadoopRunProcess_1.start() processList.append(hadoopRunProcess_1) hadoopRunProcess_2.start() processList.append(hadoopRunProcess_2) #hadoopRunProcess.join() time.sleep(180) for process in processList: process.join() retreive_mpstat_results(nodes_used, pm["exp_number"]) retreive_bwmon_results(nodes_used, pm["exp_number"]) retreive_iostat_results(nodes_used, pm["exp_number"])
def run_hadoop_baseline(pm, nodes_used, workload, schedule): ''' This is a single run to gather the baseline resource and performance profile for Cassandra ''' ###### SPAWN VMS ########## hadoop_utils.spawn_hadoop_vms(pm["hadoop:num_hadoop"], pm["exp_number"], pm["hadoop:placement"]) time.sleep(120) ####### START PROCESSES ########## hadoopStartProcess = Process(target=hadoop_utils.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"], schedule)) hadoopStartProcess.start() hadoopStartProcess.join() time.sleep(120) ####### LOAD DATA ########## hadoopLoadProcess = Process(target=hadoop_utils.hadoop_load_workload, args=(pm, pm["exp_number"], workload )) hadoopLoadProcess.start() hadoopLoadProcess.join() time.sleep(30) ####### RUN WORKLOAD ########## run = raw_input("Press Enter to run experiment ") run_mpstat(nodes_used, pm["exp_number"]) run_bwmon(nodes_used, pm["exp_number"]) run_iostat(nodes_used, pm["exp_number"]) hadoopRunProcess = Process(target=hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"], workload)) time.sleep(40) hadoopRunProcess.start() hadoopRunProcess.join() time.sleep(180) retreive_mpstat_results(nodes_used, pm["exp_number"]) retreive_bwmon_results(nodes_used, pm["exp_number"]) retreive_iostat_results(nodes_used, pm["exp_number"])
def run_hadoop_baseline(pm, nodes_used, workload, schedule): ''' This is a single run to gather the baseline resource and performance profile for hadoop ''' ###### SPAWN VMS ########## spawnProcessList = [] hadoopSpawnProcess1 = Process (target=hadoop_utils.spawn_hadoop_vms(pm["hadoop:num_hadoop"], pm["exp_number"], pm["hadoop:placement"])) hadoopSpawnProcess1.start() spawnProcessList.append(hadoopSpawnProcess1) hadoopSpawnProcess2 = Process (target=hadoop_utils_2.spawn_hadoop_vms(pm["hadoop:num_hadoop"], pm["exp_number"]+1, pm["hadoop:placement"])) hadoopSpawnProcess2.start() spawnProcessList.append(hadoopSpawnProcess2) for process in spawnProcessList: process.join() time.sleep(200) ####### START PROCESSES ########## startProcessList = [] hadoopStartProcess1 = Process(target=hadoop_utils.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"],pm, schedule)) hadoopStartProcess1.start() startProcessList.append(hadoopStartProcess1) hadoopStartProcess2 = Process(target=hadoop_utils_2.setup_hadoop, args=(pm["hadoop:num_hadoop"], pm["exp_number"]+1,pm, schedule)) hadoopStartProcess2.start() startProcessList.append(hadoopStartProcess2) for process in startProcessList: process.join() time.sleep(20) ####### LOAD DATA ########## loadProcessList = [] hadoopLoadProcess1 = Process(target=hadoop_utils.hadoop_load_workload, args=(pm, pm["exp_number"], workload )) hadoopLoadProcess1.start() loadProcessList.append(hadoopLoadProcess1) hadoopLoadProcess2 = Process(target=hadoop_utils_2.hadoop_load_workload, args=(pm, pm["exp_number"]+1, workload )) hadoopLoadProcess2.start() loadProcessList.append(hadoopLoadProcess2) for process in loadProcessList: process.join() time.sleep(10) ####### RUN WORKLOAD ########## run_mpstat(nodes_used, pm["exp_number"]) run_bwmon(nodes_used, pm["exp_number"]) run_iostat(nodes_used, pm["exp_number"]) runProcessList = [] hadoopRunProcess1 = Process(target=hadoop_utils.hadoop_run_workload, args=(pm, pm["exp_number"], workload)) hadoopRunProcess1.start() runProcessList.append(hadoopRunProcess1) hadoopRunProcess2 = Process(target=hadoop_utils_2.hadoop_run_workload, args=(pm, pm["exp_number"]+1, workload)) hadoopRunProcess2.start() runProcessList.append(hadoopRunProcess2) for process in runProcessList: process.join() retreive_mpstat_results(nodes_used, pm["exp_number"]) retreive_bwmon_results(nodes_used, pm["exp_number"]) retreive_iostat_results(nodes_used, pm["exp_number"])