def run(self):
print "ReduceTask: starting (%s)" % self.task.name
self.control.start_reduce_task(self.task)
# Sleep to simulate the initial delay
time.sleep(self.task.initialDelay)
self.task.waitFinished = time.time()
mode = get_operation_mode()
if mode == REPLAY_MODE:
# Replay execution from original traces
self.scheduleTransfersFromTraces()
else:
# Hadoop Emulation
self.fetchOutpus()
time.sleep(self.task.mergingTime)
self.task.shuffleFinished = time.time()
# Sleep to simulate the tasks exectution time
time.sleep(self.task.sortingTime)
self.task.sortFinished = time.time()
time.sleep(self.task.processingTime)
self.control.end_reduce_task(self.task)
print "ReduceTask: finished (%s)" % self.task.name
def run(self):
print "ReduceTask: starting (%s)" % self.task.name
self.control.start_reduce_task(self.task)
# Sleep to simulate the initial delay
time.sleep(self.task.initialDelay)
self.task.waitFinished = time.time()
mode = get_operation_mode()
if mode == REPLAY_MODE:
# Replay execution from original traces
self.scheduleTransfersFromTraces()
else:
# Hadoop Emulation
self.fetchOutpus()
time.sleep(self.task.mergingTime)
self.task.shuffleFinished = time.time()
# Sleep to simulate the tasks exectution time
time.sleep(self.task.sortingTime)
self.task.sortFinished = time.time()
time.sleep(self.task.processingTime)
self.control.end_reduce_task(self.task)
print "ReduceTask: finished (%s)" % self.task.name
def copyOutput(self, partition):
self.copierControl.start_copy()
partition.startTime = time.time()
print "MapOutputCopier: copiando partitcao - %s" % str(partition)
print "VEIGA: vou dormir por ", partition.initTime, " antes de comecar a enviar"
mode = get_operation_mode()
if mode == HADOOP_MODE:
time.sleep(partition.initTime)
self.copierControl.control.cond.acquire()
global IPERF_PORT
self.port = IPERF_PORT
IPERF_PORT += 1
self.copierControl.control.cond.release()
partition.dstPort = self.port
print "MapOutputCopier: starting iperf server at %s:%d" % (
partition.dstAddress, partition.dstPort)
cmd = "./trafficgen/iperf-server %s %d" % (partition.dstAddress,
partition.dstPort)
print "MapOutputCopier: run %s" % cmd
p = Popen(cmd, shell=True)
ok = False
while not ok:
ok = self.contactShuffleServer(partition)
if not ok:
time.sleep(0.01)
# Wait for the shuffle transfer to finish
p.wait()
if mode == HADOOP_MODE:
time.sleep(partition.postTime)
partition.finishTime = time.time()
partition.duration = partition.finishTime - partition.startTime
self.copierControl.control.end_shuffle(partition)
print "MapOutputCopier: finished %s:%d" % (partition.dstAddress,
partition.dstPort)
self.copierControl.end_copy()
def copyOutput(self, partition):
self.copierControl.start_copy()
partition.startTime = time.time()
print "MapOutputCopier: copiando partitcao - %s" % str(partition)
print "VEIGA: vou dormir por ",partition.initTime," antes de comecar a enviar"
mode = get_operation_mode()
if mode == HADOOP_MODE:
time.sleep(partition.initTime)
self.copierControl.control.cond.acquire()
global IPERF_PORT
self.port = IPERF_PORT
IPERF_PORT += 1
self.copierControl.control.cond.release()
partition.dstPort = self.port
print "MapOutputCopier: starting iperf server at %s:%d" % (partition.dstAddress, partition.dstPort)
cmd = "./trafficgen/iperf-server %s %d" % (partition.dstAddress, partition.dstPort)
print "MapOutputCopier: run %s" % cmd
p = Popen(cmd, shell=True)
ok = False
while not ok:
ok = self.contactShuffleServer(partition)
if not ok:
time.sleep(0.01)
# Wait for the shuffle transfer to finish
p.wait()
if mode == HADOOP_MODE:
time.sleep(partition.postTime);
partition.finishTime = time.time()
partition.duration = partition.finishTime - partition.startTime
self.copierControl.control.end_shuffle(partition)
print "MapOutputCopier: finished %s:%d" % (partition.dstAddress, partition.dstPort)
self.copierControl.end_copy()
def _read_transfers(self):
mode = get_operation_mode()
if mode == REPLAY_MODE:
self.trace["transfers"] = self.trace["transfersMapper"]
else:
self.trace["transfers"] = self.computeTransferTimes()
transfers = sorted(self.trace["transfers"],key=lambda x:x['startTime'])
for task in self.maps:
for transfer in transfers:
if transfer["dstPort"] != 0 and transfer["mapper"] == task.name:
partition = DataPartition(transfer["mapper"], transfer["reducer"],
self.task2host[transfer["mapper"]], transfer["srcPort"], self.task2host[transfer["reducer"]], transfer["dstPort"], transfer["size"])
if mode == REPLAY_MODE:
partition.initTime = 0
partition.postTime = 0
else:
partition.initTime = float(transfer["initTime"])
partition.postTime = float(transfer["postTime"])
task.addPartition(partition)
for task in self.reduces:
copia = copy.deepcopy(self.trace["transfers"])
transfersReducer = [x for x in copia if x["reducer"] == task.name]
result = self.computeTransfersWaitTime(transfersReducer, int(self.trace["config"]["maxParallelCopies"]))
delay = 0.1
startTime = result[0]["startTime"] - delay
for transfer in result:
partition = DataPartition(transfer["mapper"], transfer["reducer"],
self.task2host[transfer["mapper"]], transfer["srcPort"], self.task2host[transfer["reducer"]], transfer["dstPort"], transfer["size"])
task.addPartition(partition)
partition.waitTime = float(transfer["waitTime"])
partition.eventArrival = float(transfer["eventArrival"]-delay-startTime)
if mode == REPLAY_MODE:
partition.initTime = 0
partition.postTime = 0
else:
partition.initTime = float(transfer["initTime"])
partition.postTime = float(transfer["postTime"])
