def run(self): print "ReduceTask: starting (%s)" % self.task.name self.control.start_reduce_task(self.task) # Sleep to simulate the initial delay time.sleep(self.task.initialDelay) self.task.waitFinished = time.time() mode = get_operation_mode() if mode == REPLAY_MODE: # Replay execution from original traces self.scheduleTransfersFromTraces() else: # Hadoop Emulation self.fetchOutpus() time.sleep(self.task.mergingTime) self.task.shuffleFinished = time.time() # Sleep to simulate the tasks exectution time time.sleep(self.task.sortingTime) self.task.sortFinished = time.time() time.sleep(self.task.processingTime) self.control.end_reduce_task(self.task) print "ReduceTask: finished (%s)" % self.task.name
def copyOutput(self, partition): self.copierControl.start_copy() partition.startTime = time.time() print "MapOutputCopier: copiando partitcao - %s" % str(partition) print "VEIGA: vou dormir por ", partition.initTime, " antes de comecar a enviar" mode = get_operation_mode() if mode == HADOOP_MODE: time.sleep(partition.initTime) self.copierControl.control.cond.acquire() global IPERF_PORT self.port = IPERF_PORT IPERF_PORT += 1 self.copierControl.control.cond.release() partition.dstPort = self.port print "MapOutputCopier: starting iperf server at %s:%d" % ( partition.dstAddress, partition.dstPort) cmd = "./trafficgen/iperf-server %s %d" % (partition.dstAddress, partition.dstPort) print "MapOutputCopier: run %s" % cmd p = Popen(cmd, shell=True) ok = False while not ok: ok = self.contactShuffleServer(partition) if not ok: time.sleep(0.01) # Wait for the shuffle transfer to finish p.wait() if mode == HADOOP_MODE: time.sleep(partition.postTime) partition.finishTime = time.time() partition.duration = partition.finishTime - partition.startTime self.copierControl.control.end_shuffle(partition) print "MapOutputCopier: finished %s:%d" % (partition.dstAddress, partition.dstPort) self.copierControl.end_copy()
def copyOutput(self, partition): self.copierControl.start_copy() partition.startTime = time.time() print "MapOutputCopier: copiando partitcao - %s" % str(partition) print "VEIGA: vou dormir por ",partition.initTime," antes de comecar a enviar" mode = get_operation_mode() if mode == HADOOP_MODE: time.sleep(partition.initTime) self.copierControl.control.cond.acquire() global IPERF_PORT self.port = IPERF_PORT IPERF_PORT += 1 self.copierControl.control.cond.release() partition.dstPort = self.port print "MapOutputCopier: starting iperf server at %s:%d" % (partition.dstAddress, partition.dstPort) cmd = "./trafficgen/iperf-server %s %d" % (partition.dstAddress, partition.dstPort) print "MapOutputCopier: run %s" % cmd p = Popen(cmd, shell=True) ok = False while not ok: ok = self.contactShuffleServer(partition) if not ok: time.sleep(0.01) # Wait for the shuffle transfer to finish p.wait() if mode == HADOOP_MODE: time.sleep(partition.postTime); partition.finishTime = time.time() partition.duration = partition.finishTime - partition.startTime self.copierControl.control.end_shuffle(partition) print "MapOutputCopier: finished %s:%d" % (partition.dstAddress, partition.dstPort) self.copierControl.end_copy()
def _read_transfers(self): mode = get_operation_mode() if mode == REPLAY_MODE: self.trace["transfers"] = self.trace["transfersMapper"] else: self.trace["transfers"] = self.computeTransferTimes() transfers = sorted(self.trace["transfers"],key=lambda x:x['startTime']) for task in self.maps: for transfer in transfers: if transfer["dstPort"] != 0 and transfer["mapper"] == task.name: partition = DataPartition(transfer["mapper"], transfer["reducer"], self.task2host[transfer["mapper"]], transfer["srcPort"], self.task2host[transfer["reducer"]], transfer["dstPort"], transfer["size"]) if mode == REPLAY_MODE: partition.initTime = 0 partition.postTime = 0 else: partition.initTime = float(transfer["initTime"]) partition.postTime = float(transfer["postTime"]) task.addPartition(partition) for task in self.reduces: copia = copy.deepcopy(self.trace["transfers"]) transfersReducer = [x for x in copia if x["reducer"] == task.name] result = self.computeTransfersWaitTime(transfersReducer, int(self.trace["config"]["maxParallelCopies"])) delay = 0.1 startTime = result[0]["startTime"] - delay for transfer in result: partition = DataPartition(transfer["mapper"], transfer["reducer"], self.task2host[transfer["mapper"]], transfer["srcPort"], self.task2host[transfer["reducer"]], transfer["dstPort"], transfer["size"]) task.addPartition(partition) partition.waitTime = float(transfer["waitTime"]) partition.eventArrival = float(transfer["eventArrival"]-delay-startTime) if mode == REPLAY_MODE: partition.initTime = 0 partition.postTime = 0 else: partition.initTime = float(transfer["initTime"]) partition.postTime = float(transfer["postTime"])