def finishJob(self):
self.logger.job_finish(self.name)
self.finishTime = time.time()
output = TraceOutput(self.config.host)
output.saveJobTracker(self.config, self.submitTime, self.startTime,
self.finishTime, self.name, self.numMaps,
self.numReduces)
def run(self):
ok = False
while not ok:
ok = self.ready()
self.waitForJobTracker()
if len(self.reduces) > 0:
# Start Partition Listener
partitionQueue = dict()
for task in self.reduces:
partitionQueue[task.name] = []
reduceListener = ReduceListener(self.reduces, self.control,
self.config, partitionQueue)
reduceListener.start()
# Schedule map tasks
if len(self.maps) > 0:
shuffleServer = ShuffleServer(self.config.host, self.numTransfers)
shuffleServer.start()
# Start Map Task Launcher
initialDelay = self.trace.getInitialMapDelay(self.config.host)
mapLauncher = MapTaskLauncher(self.maps, self.control, self.config,
initialDelay)
mapLauncher.start()
# Schedule reduce tasks
if len(self.reduces) > 0:
# Start Reduce Task Launcher
initialDelay = self.trace.getInitialReduceDelay(self.config.host)
reduceLauncher = ReduceTaskLauncher(self.reduces, self.control,
self.config, partitionQueue,
initialDelay)
reduceLauncher.start()
if len(self.maps) > 0:
mapLauncher.join()
if len(self.reduces) > 0:
reduceListener.join()
if len(self.maps) > 0:
shuffleServer.join()
if len(self.reduces) > 0:
reduceLauncher.join()
self.finish()
output = TraceOutput(self.config.host)
output.saveTaskTracker(self.control.completedMaps,
self.control.completedReduces,
self.control.completedTransfers)
def run(self):
ok = False
while not ok:
ok = self.ready()
self.waitForJobTracker()
if len(self.reduces) > 0:
# Start Partition Listener
partitionQueue = dict()
for task in self.reduces:
partitionQueue[task.name] = []
reduceListener = ReduceListener(self.reduces, self.control, self.config, partitionQueue)
reduceListener.start()
# Schedule map tasks
if len(self.maps) > 0:
shuffleServer = ShuffleServer(self.config.host, self.numTransfers)
shuffleServer.start()
# Start Map Task Launcher
initialDelay = self.trace.getInitialMapDelay(self.config.host)
mapLauncher = MapTaskLauncher(self.maps, self.control, self.config, initialDelay)
mapLauncher.start()
# Schedule reduce tasks
if len(self.reduces) > 0:
# Start Reduce Task Launcher
initialDelay = self.trace.getInitialReduceDelay(self.config.host)
reduceLauncher = ReduceTaskLauncher(self.reduces, self.control, self.config, partitionQueue, initialDelay)
reduceLauncher.start()
if len(self.maps) > 0:
mapLauncher.join()
if len(self.reduces) > 0:
reduceListener.join()
if len(self.maps) > 0:
shuffleServer.join()
if len(self.reduces) > 0:
reduceLauncher.join()
self.finish()
output = TraceOutput(self.config.host)
output.saveTaskTracker(self.control.completedMaps, self.control.completedReduces, self.control.completedTransfers)
def finishJob(self):
self.logger.job_finish(self.name)
self.finishTime = time.time()
output = TraceOutput(self.config.host)
output.saveJobTracker(self.config, self.submitTime, self.startTime, self.finishTime, self.name, self.numMaps, self.numReduces)