def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(
Script, cwd, python_path, osenv, self.parallel,
out_logger, err_logger, logLevel, use_color, dpark_env
) = marshal.loads(decode_data(executorInfo.data))
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = formatter_message(
'{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10),
use_color
)
init_dpark_logger(logLevel, use_color=use_color)
logging.root.setLevel(logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
spawn_rconsole(locals())
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
env.workdir.init(dpark_env.get(env.DPARK_ID))
self._try_flock(env.workdir.main)
dpark_env['SERVER_URI'] = startWebServer(env.workdir.main)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
env.workdir.setup_cleaner_process()
spawn(self.check_alive, driver)
spawn(self.replier, driver)
env.environ.update(dpark_env)
from dpark.broadcast import start_download_manager
start_download_manager()
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise
def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(Script, cwd, python_path, osenv, self.parallel, out_logger,
err_logger, logLevel, use_color,
dpark_env) = marshal.loads(decode_data(executorInfo.data))
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = formatter_message(
'{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10),
use_color)
init_dpark_logger(logLevel, use_color=use_color)
logging.root.setLevel(logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
spawn_rconsole(locals())
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
env.workdir.init(dpark_env.get(env.DPARK_ID))
self._try_flock(env.workdir.main)
dpark_env['SERVER_URI'] = startWebServer(env.workdir.main)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
env.workdir.setup_cleaner_process()
spawn(self.check_alive, driver)
spawn(self.replier, driver)
env.environ.update(dpark_env)
from dpark.broadcast import start_download_manager
start_download_manager()
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise
def submitTasks(self, tasks):
if not tasks:
return
logger.info('Got a job with %d tasks: %s', len(tasks), tasks[0].rdd)
total, self.finished, start = len(tasks), 0, time.time()
def initializer():
# when called on subprocess of multiprocessing's Pool,
# default sighandler of SIGTERM will be called to quit gracefully,
# and we ignore other signals to prevent dead lock.
import signal
from .context import _signals
for sig in _signals:
if sig == signal.SIGTERM:
signal.signal(sig, signal.SIG_DFL)
else:
signal.signal(sig, signal.SIG_IGN)
def callback(args):
logger.debug('got answer: %s', args)
tid, reason, result, update = args
task = self.tasks.pop(tid)
self.finished += 1
logger.info('Task %s finished (%d/%d) \x1b[1A',
tid, self.finished, total)
if self.finished == total:
logger.info(
'Job finished in %.1f seconds' + ' ' * 20,
time.time() - start)
self.taskEnded(task, reason, result, update)
for task in tasks:
logger.debug('put task async: %s', task)
self.tasks[task.id] = task
if not self.pool:
# daemonic processes are not allowed to have children
from dpark.broadcast import start_download_manager
start_download_manager()
self.pool = multiprocessing.Pool(
self.threads or 2,
initializer = initializer
)
self.pool.apply_async(run_task_in_process,
[task, self.nextAttempId(), env.environ],
callback=callback)