def registered(self, driver, executorInfo, frameworkInfo, agent_info): try: global Script ( Script, cwd, python_path, osenv, self.parallel, out_logger, err_logger, logLevel, use_color, dpark_env ) = marshal.loads(decode_data(executorInfo.data)) sys.path = python_path os.environ.update(osenv) setproctitle('[Executor]' + Script) prefix = formatter_message( '{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10), use_color ) init_dpark_logger(logLevel, use_color=use_color) logging.root.setLevel(logLevel) r1 = self.stdout_redirect = Redirect(1, out_logger, prefix) sys.stdout = r1.pipe_wfile r2 = self.stderr_redirect = Redirect(2, err_logger, prefix) sys.stderr = r2.pipe_wfile spawn_rconsole(locals()) if os.path.exists(cwd): try: os.chdir(cwd) except Exception as e: logger.warning('change cwd to %s failed: %s', cwd, e) else: logger.warning('cwd (%s) not exists', cwd) env.workdir.init(dpark_env.get(env.DPARK_ID)) self._try_flock(env.workdir.main) dpark_env['SERVER_URI'] = startWebServer(env.workdir.main) if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy env.workdir.setup_cleaner_process() spawn(self.check_alive, driver) spawn(self.replier, driver) env.environ.update(dpark_env) from dpark.broadcast import start_download_manager start_download_manager() logger.debug('executor started at %s', agent_info.hostname) except Exception as e: import traceback msg = traceback.format_exc() logger.error('init executor failed: %s', msg) raise
def registered(self, driver, executorInfo, frameworkInfo, agent_info): try: global Script (Script, cwd, python_path, osenv, self.parallel, out_logger, err_logger, logLevel, use_color, dpark_env) = marshal.loads(decode_data(executorInfo.data)) sys.path = python_path os.environ.update(osenv) setproctitle('[Executor]' + Script) prefix = formatter_message( '{MAGENTA}[%s]{RESET} ' % socket.gethostname().ljust(10), use_color) init_dpark_logger(logLevel, use_color=use_color) logging.root.setLevel(logLevel) r1 = self.stdout_redirect = Redirect(1, out_logger, prefix) sys.stdout = r1.pipe_wfile r2 = self.stderr_redirect = Redirect(2, err_logger, prefix) sys.stderr = r2.pipe_wfile spawn_rconsole(locals()) if os.path.exists(cwd): try: os.chdir(cwd) except Exception as e: logger.warning('change cwd to %s failed: %s', cwd, e) else: logger.warning('cwd (%s) not exists', cwd) env.workdir.init(dpark_env.get(env.DPARK_ID)) self._try_flock(env.workdir.main) dpark_env['SERVER_URI'] = startWebServer(env.workdir.main) if 'MESOS_SLAVE_PID' in os.environ: # make unit test happy env.workdir.setup_cleaner_process() spawn(self.check_alive, driver) spawn(self.replier, driver) env.environ.update(dpark_env) from dpark.broadcast import start_download_manager start_download_manager() logger.debug('executor started at %s', agent_info.hostname) except Exception as e: import traceback msg = traceback.format_exc() logger.error('init executor failed: %s', msg) raise