def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(
Script, cwd, python_path, osenv, self.parallel,
out_logger, err_logger, logLevel, use_color, dpark_env
) = marshal.loads(decode_data(executorInfo.data))
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = formatter_message(
'{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10),
use_color
)
init_dpark_logger(logLevel, use_color=use_color)
logging.root.setLevel(logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
spawn_rconsole(locals())
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
env.workdir.init(dpark_env.get(env.DPARK_ID))
self._try_flock(env.workdir.main)
dpark_env['SERVER_URI'] = startWebServer(env.workdir.main)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
env.workdir.setup_cleaner_process()
spawn(self.check_alive, driver)
spawn(self.replier, driver)
env.environ.update(dpark_env)
from dpark.broadcast import start_download_manager
start_download_manager()
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise
def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(Script, cwd, python_path, osenv, self.parallel, out_logger,
err_logger, logLevel, use_color,
dpark_env) = marshal.loads(decode_data(executorInfo.data))
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = formatter_message(
'{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10),
use_color)
init_dpark_logger(logLevel, use_color=use_color)
logging.root.setLevel(logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
spawn_rconsole(locals())
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
env.workdir.init(dpark_env.get(env.DPARK_ID))
self._try_flock(env.workdir.main)
dpark_env['SERVER_URI'] = startWebServer(env.workdir.main)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
env.workdir.setup_cleaner_process()
spawn(self.check_alive, driver)
spawn(self.replier, driver)
env.environ.update(dpark_env)
from dpark.broadcast import start_download_manager
start_download_manager()
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise
def start(self):
def shutdown():
self.stop()
try:
import dpark.web
dpark.web.stop(self.web_port)
except ImportError:
pass
if self.started:
return
self.init()
env.start()
self.scheduler.start()
self.started = True
_shutdown_handlers.append(shutdown)
spawn_rconsole(locals())
def start(self):
def shutdown():
self.stop()
try:
import dpark.web
dpark.web.stop(self.web_port)
except ImportError:
pass
if self.started:
return
self.init()
env.start_master()
if not isinstance(self.scheduler, MesosScheduler):
env.start_slave()
self.scheduler.start()
self.started = True
_shutdown_handlers.append(shutdown)
spawn_rconsole(locals())
logger.debug('Connecting to mesos master %s', options.master)
driver = MesosSchedulerDriver(sched,
sched.framework,
options.master,
use_addict=True)
def handler(signm, frame):
logger.warning('got signal %d, exit now', signm)
sched.stop(EXIT_SIGNAL)
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
spawn_rconsole(locals())
try:
driver.start()
sched.run(driver)
except KeyboardInterrupt:
logger.warning('stopped by KeyboardInterrupt')
sched.stop(EXIT_KEYBORAD)
except Exception as e:
import traceback
logger.warning('catch unexpected Exception, exit now. %s',
traceback.format_exc())
sched.stop(EXIT_EXCEPTION)
finally:
try:
driver = MesosSchedulerDriver(
sched, sched.framework, options.master, use_addict=True
)
def handler(signm, frame):
logger.warning('got signal %d, exit now', signm)
sched.stop(EXIT_SIGNAL)
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGHUP, handler)
signal.signal(signal.SIGABRT, handler)
signal.signal(signal.SIGQUIT, handler)
spawn_rconsole(locals())
try:
driver.start()
sched.run(driver)
except KeyboardInterrupt:
logger.warning('stopped by KeyboardInterrupt')
sched.stop(EXIT_KEYBORAD)
except Exception as e:
import traceback
logger.warning('catch unexpected Exception, exit now. %s',
traceback.format_exc())
sched.stop(EXIT_EXCEPTION)
finally:
try: