def __getattr__(self, name): if name != 'value': return getattr(self.value, name) # in the executor process, Broadcast is not initialized if not self.initialized: raise AttributeError(name) uuid = self.uuid value = self.cache.get(uuid) if value is not None: self.value = value return value oldtitle = getproctitle() setproctitle('dpark worker: broadcasting ' + uuid) value = self.recv() if value is None: raise Exception("recv broadcast failed") self.value = value self.cache.put(uuid, value) setproctitle(oldtitle) return value
def run_task(task_data): try: gc.disable() task, ntry = cPickle.loads(decompress(task_data)) setproctitle('dpark worker %s: run task %s' % (Script, task)) Accumulator.clear() result = task.run(ntry) accUpdate = Accumulator.values() if marshalable(result): flag, data = 0, marshal.dumps(result) else: flag, data = 1, cPickle.dumps(result, -1) data = compress(data) if len(data) > TASK_RESULT_LIMIT: workdir = env.get('WORKDIR') name = 'task_%s_%s.result' % (task.id, ntry) path = os.path.join(workdir, name) f = open(path, 'w') f.write(data) f.close() data = LocalFileShuffle.getServerUri() + '/' + name flag += 2 return mesos_pb2.TASK_FINISHED, cPickle.dumps((task.id, Success(), (flag, data), accUpdate), -1) except Exception, e: import traceback msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, cPickle.dumps((task.id, OtherFailure(msg), None, None), -1)
def __getattr__(self, name): if name != 'value': return getattr(self.value, name) if self.stopped: raise SystemExit("broadcast has been cleared") # in the executor process, Broadcast is not initialized if not self.initialized: raise AttributeError(name) uuid = self.uuid value = self.cache.get(uuid) if value is not None: self.value = value return value oldtitle = getproctitle() setproctitle('dpark worker: broadcasting ' + uuid) value = self.recv() if value is None: raise Exception("recv broadcast failed") self.value = value self.cache.put(uuid, value) setproctitle(oldtitle) return value
def run_task(task, ntry): try: setproctitle('dpark worker %s: run task %s' % (Script, task)) Accumulator.clear() gc.disable() result = task.run(ntry) accUpdate = Accumulator.values() if marshalable(result): flag, data = 0, marshal.dumps(result) else: flag, data = 1, cPickle.dumps(result, -1) data = compress(data) if len(data) > TASK_RESULT_LIMIT: workdir = env.get('WORKDIR') name = 'task_%s_%s.result' % (task.id, ntry) path = os.path.join(workdir, name) f = open(path, 'w') f.write(data) f.close() data = LocalFileShuffle.getServerUri() + '/' + name flag += 2 return mesos_pb2.TASK_FINISHED, cPickle.dumps( (task.id, Success(), (flag, data), accUpdate), -1) except Exception, e: import traceback msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, cPickle.dumps( (task.id, OtherFailure(msg), None, None), -1)
def run_task(task_data): try: gc.disable() task, ntry = cPickle.loads(decompress(task_data)) setproctitle('dpark worker %s: run task %s' % (Script, task)) Accumulator.clear() result = task.run(ntry) accUpdate = Accumulator.values() if marshalable(result): flag, data = 0, marshal.dumps(result) else: flag, data = 1, cPickle.dumps(result, -1) data = compress(data) if len(data) > TASK_RESULT_LIMIT: path = LocalFileShuffle.getOutputFile(0, ntry, task.id, len(data)) f = open(path, 'w') f.write(data) f.close() data = '/'.join([LocalFileShuffle.getServerUri()] + path.split('/')[-3:]) flag += 2 return mesos_pb2.TASK_FINISHED, cPickle.dumps((Success(), (flag, data), accUpdate), -1) except FetchFailed, e: return mesos_pb2.TASK_FAILED, cPickle.dumps((e, None, None), -1)
def run_task(task_data): try: gc.disable() task, ntry = cPickle.loads(decompress(task_data)) setproctitle('dpark worker %s: run task %s' % (Script, task)) Accumulator.clear() result = task.run(ntry) accUpdate = Accumulator.values() if marshalable(result): try: flag, data = 0, marshal.dumps(result) except Exception, e: flag, data = 1, cPickle.dumps(result, -1) else:
def setup_cleaner_process(workdir): ppid = os.getpid() pid = os.fork() if pid == 0: os.setsid() pid = os.fork() if pid == 0: try: import psutil except ImportError: os._exit(1) try: setproctitle('dpark cleaner %s wait(%d)' % (workdir, ppid)) psutil.Process(ppid).wait() os.killpg(ppid, signal.SIGKILL) # kill workers except Exception, e: pass # make sure to exit finally:
def clean_work_dir(path): setproctitle('dpark cleaner %s' % path) threading.Thread(target=cleanup, args=[path]).start()
def init_env(args, workdir): setproctitle('dpark worker: idle') env.start(False, args) threading.Thread(target=cleanup, args=[workdir]).start()
workdir = env.get('WORKDIR') name = 'task_%s_%s.result' % (task.id, ntry) path = os.path.join(workdir, name) f = open(path, 'w') f.write(data) f.close() data = LocalFileShuffle.getServerUri() + '/' + name flag += 2 return mesos_pb2.TASK_FINISHED, cPickle.dumps((task.id, Success(), (flag, data), accUpdate), -1) except Exception, e: import traceback msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, cPickle.dumps((task.id, OtherFailure(msg), None, None), -1) finally: setproctitle('dpark worker: idle') gc.collect() gc.enable() def cleanup(workdir): while os.getppid() > 1: time.sleep(1) while os.path.exists(workdir): try: shutil.rmtree(workdir, True) except: pass os._exit(0) def init_env(args, workdir): setproctitle('dpark worker: idle')
def init_env(args): setproctitle('dpark worker: idle') env.start(False, args)
path = LocalFileShuffle.getOutputFile(0, ntry, task.id, len(data)) f = open(path, 'w') f.write(data) f.close() data = '/'.join([LocalFileShuffle.getServerUri()] + path.split('/')[-3:]) flag += 2 return mesos_pb2.TASK_FINISHED, cPickle.dumps((Success(), (flag, data), accUpdate), -1) except FetchFailed, e: return mesos_pb2.TASK_FAILED, cPickle.dumps((e, None, None), -1) except : import traceback msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, cPickle.dumps((OtherFailure(msg), None, None), -1) finally: setproctitle('dpark worker: idle') gc.collect() gc.enable() def init_env(args): setproctitle('dpark worker: idle') env.start(False, args) class LocalizedHTTP(SimpleHTTPServer.SimpleHTTPRequestHandler): basedir = None def translate_path(self, path): out = SimpleHTTPServer.SimpleHTTPRequestHandler.translate_path(self, path) return self.basedir + '/' + os.path.relpath(out) def log_message(self, format, *args):
path = os.path.join(workdir, name) f = open(path, 'w') f.write(data) f.close() data = LocalFileShuffle.getServerUri() + '/' + name flag += 2 return mesos_pb2.TASK_FINISHED, cPickle.dumps( (task.id, Success(), (flag, data), accUpdate), -1) except Exception, e: import traceback msg = traceback.format_exc() return mesos_pb2.TASK_FAILED, cPickle.dumps( (task.id, OtherFailure(msg), None, None), -1) finally: setproctitle('dpark worker: idle') gc.collect() gc.enable() def init_env(args): setproctitle('dpark worker: idle') env.start(False, args) basedir = None class LocalizedHTTP(SimpleHTTPServer.SimpleHTTPRequestHandler): def translate_path(self, path): out = SimpleHTTPServer.SimpleHTTPRequestHandler.translate_path(
if pid == 0: os.setsid() pid = os.fork() if pid == 0: try: import psutil except ImportError: os._exit(1) try: setproctitle('dpark cleaner %s wait(%d)' % (workdir, ppid)) psutil.Process(ppid).wait() os.killpg(ppid, signal.SIGKILL) # kill workers except Exception, e: pass # make sure to exit finally: setproctitle('dpark cleaning %s ' % workdir) for d in workdir: while os.path.exists(d): try: shutil.rmtree(d, True) except: pass os._exit(0) os.wait() class MyExecutor(mesos.Executor): def __init__(self): self.workdir = [] self.idle_workers = [] self.busy_workers = {}
if pid == 0: os.setsid() pid = os.fork() if pid == 0: try: import psutil except ImportError: os._exit(1) try: setproctitle('dpark cleaner %s wait(%d)' % (workdir, ppid)) psutil.Process(ppid).wait() os.killpg(ppid, signal.SIGKILL) # kill workers except Exception, e: pass # make sure to exit finally: setproctitle('dpark cleaning %s ' % workdir) for d in workdir: while os.path.exists(d): try: shutil.rmtree(d, True) except: pass os._exit(0) os.wait() class MyExecutor(mesos.Executor): def __init__(self): self.workdir = [] self.idle_workers = [] self.busy_workers = {} self.lock = threading.RLock() def check_memory(self, driver):