def parse_options():
options, args = parser.parse_args()
setup_conf(options)
options.logLevel = (options.quiet and logging.ERROR
or options.verbose and logging.DEBUG or logging.INFO)
init_dpark_logger(options.logLevel)
if any(arg.startswith('-') for arg in args):
logger.warning('unknown args found in command-line: %s', ' '.join(args))
return options
def parse_options():
options, args = parser.parse_args()
if options.mem is None:
options.mem = conf.MEM_PER_TASK
else:
options.mem = memory_str_to_mb(options.mem)
options.logLevel = (options.quiet and logging.ERROR
or options.verbose and logging.DEBUG or logging.INFO)
if options.color is None:
options.color = getattr(sys.stderr, 'isatty', lambda: False)()
init_dpark_logger(options.logLevel, use_color=options.color)
if any(arg.startswith('-') for arg in args):
logger.warning('unknown args found in command-line: %s',
' '.join(args))
return options
def registered(self, driver, executorInfo, frameworkInfo, agent_info):
try:
global Script
(Script, cwd, python_path, osenv, self.parallel, out_logger,
err_logger, logLevel, use_color,
args) = marshal.loads(decode_data(executorInfo.data))
sys.path = python_path
os.environ.update(osenv)
setproctitle('[Executor]' + Script)
prefix = formatter_message(
'{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10),
use_color)
init_dpark_logger(logLevel, use_color=use_color)
logging.root.setLevel(logLevel)
r1 = self.stdout_redirect = Redirect(1, out_logger, prefix)
sys.stdout = r1.pipe_wfile
r2 = self.stderr_redirect = Redirect(2, err_logger, prefix)
sys.stderr = r2.pipe_wfile
if os.path.exists(cwd):
try:
os.chdir(cwd)
except Exception as e:
logger.warning('change cwd to %s failed: %s', cwd, e)
else:
logger.warning('cwd (%s) not exists', cwd)
self.workdir = args['WORKDIR']
main_workdir = self.workdir[0]
root = os.path.dirname(main_workdir)
if not os.path.exists(root):
os.mkdir(root)
os.chmod(root, 0o777) # because umask
mkdir_p(main_workdir)
self._try_flock(main_workdir)
args['SERVER_URI'] = startWebServer(main_workdir)
if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy
setup_cleaner_process(self.workdir)
spawn(self.check_memory, driver)
spawn(self.replier, driver)
env.environ.update(args)
from dpark.broadcast import start_download_manager
start_download_manager()
logger.debug('executor started at %s', agent_info.hostname)
except Exception as e:
import traceback
msg = traceback.format_exc()
logger.error('init executor failed: %s', msg)
raise