def __getattr__(self, name):
if name != 'value':
return getattr(self.value, name)
# in the executor process, Broadcast is not initialized
if not self.initialized:
raise AttributeError(name)
uuid = self.uuid
value = self.cache.get(uuid)
if value is not None:
self.value = value
return value
oldtitle = getproctitle()
setproctitle('dpark worker: broadcasting ' + uuid)
value = self.recv()
if value is None:
raise Exception("recv broadcast failed")
self.value = value
self.cache.put(uuid, value)
setproctitle(oldtitle)
return value
def run_task(task_data):
try:
gc.disable()
task, ntry = cPickle.loads(decompress(task_data))
setproctitle('dpark worker %s: run task %s' % (Script, task))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
if marshalable(result):
flag, data = 0, marshal.dumps(result)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
workdir = env.get('WORKDIR')
name = 'task_%s_%s.result' % (task.id, ntry)
path = os.path.join(workdir, name)
f = open(path, 'w')
f.write(data)
f.close()
data = LocalFileShuffle.getServerUri() + '/' + name
flag += 2
return mesos_pb2.TASK_FINISHED, cPickle.dumps((task.id, Success(), (flag, data), accUpdate), -1)
except Exception, e:
import traceback
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, cPickle.dumps((task.id, OtherFailure(msg), None, None), -1)
def __getattr__(self, name):
if name != 'value':
return getattr(self.value, name)
if self.stopped:
raise SystemExit("broadcast has been cleared")
# in the executor process, Broadcast is not initialized
if not self.initialized:
raise AttributeError(name)
uuid = self.uuid
value = self.cache.get(uuid)
if value is not None:
self.value = value
return value
oldtitle = getproctitle()
setproctitle('dpark worker: broadcasting ' + uuid)
value = self.recv()
if value is None:
raise Exception("recv broadcast failed")
self.value = value
self.cache.put(uuid, value)
setproctitle(oldtitle)
return value
def run_task(task, ntry):
try:
setproctitle('dpark worker %s: run task %s' % (Script, task))
Accumulator.clear()
gc.disable()
result = task.run(ntry)
accUpdate = Accumulator.values()
if marshalable(result):
flag, data = 0, marshal.dumps(result)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
workdir = env.get('WORKDIR')
name = 'task_%s_%s.result' % (task.id, ntry)
path = os.path.join(workdir, name)
f = open(path, 'w')
f.write(data)
f.close()
data = LocalFileShuffle.getServerUri() + '/' + name
flag += 2
return mesos_pb2.TASK_FINISHED, cPickle.dumps(
(task.id, Success(), (flag, data), accUpdate), -1)
except Exception, e:
import traceback
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, cPickle.dumps(
(task.id, OtherFailure(msg), None, None), -1)
def run_task(task_data):
try:
gc.disable()
task, ntry = cPickle.loads(decompress(task_data))
setproctitle('dpark worker %s: run task %s' % (Script, task))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
if marshalable(result):
flag, data = 0, marshal.dumps(result)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
path = LocalFileShuffle.getOutputFile(0, ntry, task.id, len(data))
f = open(path, 'w')
f.write(data)
f.close()
data = '/'.join([LocalFileShuffle.getServerUri()] + path.split('/')[-3:])
flag += 2
return mesos_pb2.TASK_FINISHED, cPickle.dumps((Success(), (flag, data), accUpdate), -1)
except FetchFailed, e:
return mesos_pb2.TASK_FAILED, cPickle.dumps((e, None, None), -1)
def run_task(task_data):
try:
gc.disable()
task, ntry = cPickle.loads(decompress(task_data))
setproctitle('dpark worker %s: run task %s' % (Script, task))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception, e:
flag, data = 1, cPickle.dumps(result, -1)
else:
def run_task(task_data):
try:
gc.disable()
task, ntry = cPickle.loads(decompress(task_data))
setproctitle('dpark worker %s: run task %s' % (Script, task))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception, e:
flag, data = 1, cPickle.dumps(result, -1)
else:
def setup_cleaner_process(workdir):
ppid = os.getpid()
pid = os.fork()
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
try:
import psutil
except ImportError:
os._exit(1)
try:
setproctitle('dpark cleaner %s wait(%d)' % (workdir, ppid))
psutil.Process(ppid).wait()
os.killpg(ppid, signal.SIGKILL) # kill workers
except Exception, e:
pass # make sure to exit
finally:
def clean_work_dir(path):
setproctitle('dpark cleaner %s' % path)
threading.Thread(target=cleanup, args=[path]).start()
def init_env(args, workdir):
setproctitle('dpark worker: idle')
env.start(False, args)
threading.Thread(target=cleanup, args=[workdir]).start()
workdir = env.get('WORKDIR')
name = 'task_%s_%s.result' % (task.id, ntry)
path = os.path.join(workdir, name)
f = open(path, 'w')
f.write(data)
f.close()
data = LocalFileShuffle.getServerUri() + '/' + name
flag += 2
return mesos_pb2.TASK_FINISHED, cPickle.dumps((task.id, Success(), (flag, data), accUpdate), -1)
except Exception, e:
import traceback
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, cPickle.dumps((task.id, OtherFailure(msg), None, None), -1)
finally:
setproctitle('dpark worker: idle')
gc.collect()
gc.enable()
def cleanup(workdir):
while os.getppid() > 1:
time.sleep(1)
while os.path.exists(workdir):
try: shutil.rmtree(workdir, True)
except: pass
os._exit(0)
def init_env(args, workdir):
setproctitle('dpark worker: idle')
def init_env(args):
setproctitle('dpark worker: idle')
env.start(False, args)
path = LocalFileShuffle.getOutputFile(0, ntry, task.id, len(data))
f = open(path, 'w')
f.write(data)
f.close()
data = '/'.join([LocalFileShuffle.getServerUri()] + path.split('/')[-3:])
flag += 2
return mesos_pb2.TASK_FINISHED, cPickle.dumps((Success(), (flag, data), accUpdate), -1)
except FetchFailed, e:
return mesos_pb2.TASK_FAILED, cPickle.dumps((e, None, None), -1)
except :
import traceback
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, cPickle.dumps((OtherFailure(msg), None, None), -1)
finally:
setproctitle('dpark worker: idle')
gc.collect()
gc.enable()
def init_env(args):
setproctitle('dpark worker: idle')
env.start(False, args)
class LocalizedHTTP(SimpleHTTPServer.SimpleHTTPRequestHandler):
basedir = None
def translate_path(self, path):
out = SimpleHTTPServer.SimpleHTTPRequestHandler.translate_path(self, path)
return self.basedir + '/' + os.path.relpath(out)
def log_message(self, format, *args):
path = os.path.join(workdir, name)
f = open(path, 'w')
f.write(data)
f.close()
data = LocalFileShuffle.getServerUri() + '/' + name
flag += 2
return mesos_pb2.TASK_FINISHED, cPickle.dumps(
(task.id, Success(), (flag, data), accUpdate), -1)
except Exception, e:
import traceback
msg = traceback.format_exc()
return mesos_pb2.TASK_FAILED, cPickle.dumps(
(task.id, OtherFailure(msg), None, None), -1)
finally:
setproctitle('dpark worker: idle')
gc.collect()
gc.enable()
def init_env(args):
setproctitle('dpark worker: idle')
env.start(False, args)
basedir = None
class LocalizedHTTP(SimpleHTTPServer.SimpleHTTPRequestHandler):
def translate_path(self, path):
out = SimpleHTTPServer.SimpleHTTPRequestHandler.translate_path(
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
try:
import psutil
except ImportError:
os._exit(1)
try:
setproctitle('dpark cleaner %s wait(%d)' % (workdir, ppid))
psutil.Process(ppid).wait()
os.killpg(ppid, signal.SIGKILL) # kill workers
except Exception, e:
pass # make sure to exit
finally:
setproctitle('dpark cleaning %s ' % workdir)
for d in workdir:
while os.path.exists(d):
try:
shutil.rmtree(d, True)
except:
pass
os._exit(0)
os.wait()
class MyExecutor(mesos.Executor):
def __init__(self):
self.workdir = []
self.idle_workers = []
self.busy_workers = {}
if pid == 0:
os.setsid()
pid = os.fork()
if pid == 0:
try:
import psutil
except ImportError:
os._exit(1)
try:
setproctitle('dpark cleaner %s wait(%d)' % (workdir, ppid))
psutil.Process(ppid).wait()
os.killpg(ppid, signal.SIGKILL) # kill workers
except Exception, e:
pass # make sure to exit
finally:
setproctitle('dpark cleaning %s ' % workdir)
for d in workdir:
while os.path.exists(d):
try: shutil.rmtree(d, True)
except: pass
os._exit(0)
os.wait()
class MyExecutor(mesos.Executor):
def __init__(self):
self.workdir = []
self.idle_workers = []
self.busy_workers = {}
self.lock = threading.RLock()
def check_memory(self, driver):
