def _cmdline(self): params = dict( log_dir=LogOptions.log_dir(), log_to_disk='DEBUG', checkpoint_root=self._checkpoint_root, sandbox=self._sandbox.root, container_sandbox=self._sandbox.container_root, task_id=self._task_id, thermos_json=self._task_filename, hostname=self._hostname, process_logger_destination=self._process_logger_destination, process_logger_mode=self._process_logger_mode, rotate_log_size_mb=self._rotate_log_size_mb, rotate_log_backups=self._rotate_log_backups) if getpass.getuser() == 'root' and self._role: params.update(setuid=self._role) cmdline_args = [sys.executable, self._runner_pex] cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items() if value is not None) if self._enable_chroot: cmdline_args.extend(['--enable_chroot']) if self._preserve_env: cmdline_args.extend(['--preserve_env']) if self._sandbox.is_filesystem_image: cmdline_args.extend([ '--mesos_containerizer_path=%s' % self._mesos_containerizer_path ]) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args
def init(filebase=None): """ Sets up default stderr logging and, if filebase is supplied, sets up disk logging using: {--log_dir}/filebase.{INFO,WARNING,...} """ logging._acquireLock() # set up permissive logger root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) # clear existing handlers teardown_stderr_logging() teardown_disk_logging() for handler in root_logger.handlers: root_logger.removeHandler(handler) # setup INFO...FATAL handlers if filebase: for handler in _setup_disk_logging(filebase): root_logger.addHandler(handler) _DISK_LOGGERS.append(handler) for handler in _setup_stderr_logging(): root_logger.addHandler(handler) _STDERR_LOGGERS.append(handler) logging._releaseLock() if len(_DISK_LOGGERS) > 0 and LogOptions.stderr_log_level( ) != LogOptions.LOG_LEVEL_NONE: print('Writing log files to disk in %s' % LogOptions.log_dir(), file=sys.stderr) return root_logger
def init(filebase): """ Set up default logging using: {--log_dir}/filebase.{INFO,WARNING,...} """ logging._acquireLock() # set up permissive logger root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) # clear existing handlers teardown_stderr_logging() teardown_disk_logging() # setup INFO...FATAL handlers num_disk_handlers = 0 for handler in _setup_disk_logging(filebase): root_logger.addHandler(handler) _DISK_LOGGERS.append(handler) for handler in _setup_stderr_logging(): root_logger.addHandler(handler) _STDERR_LOGGERS.append(handler) logging._releaseLock() if len(_DISK_LOGGERS) > 0 and LogOptions.stderr_log_level() != LogOptions.LOG_LEVEL_NONE: print('Writing log files to disk in %s' % LogOptions.log_dir(), file=sys.stderr) return root_logger
def _cmdline(self): host_sandbox = None if os.environ.get('MESOS_DIRECTORY'): host_sandbox = os.path.join(os.environ.get('MESOS_DIRECTORY'), 'sandbox') params = dict(log_dir=LogOptions.log_dir(), log_to_disk='DEBUG', checkpoint_root=self._checkpoint_root, sandbox=host_sandbox or self._root, task_id=self._task_id, thermos_json=self._task_filename, hostname=self._hostname, process_logger_destination=self._process_logger_destination, process_logger_mode=self._process_logger_mode, rotate_log_size_mb=self._rotate_log_size_mb, rotate_log_backups=self._rotate_log_backups) if getpass.getuser() == 'root' and self._role: params.update(setuid=self._role) cmdline_args = [sys.executable, self._runner_pex] cmdline_args.extend( '--%s=%s' % (flag, value) for flag, value in params.items() if value is not None) if self._enable_chroot: cmdline_args.extend(['--enable_chroot']) if self._preserve_env: cmdline_args.extend(['--preserve_env']) if self._sandbox.is_filesystem_image: cmdline_args.extend( ['--mesos_containerizer_path=%s' % self._mesos_containerizer_path]) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args
def _cmdline(self): host_sandbox = None if os.environ.get('MESOS_DIRECTORY'): host_sandbox = os.path.join(os.environ.get('MESOS_DIRECTORY'), 'sandbox') params = dict(log_dir=LogOptions.log_dir(), log_to_disk='DEBUG', checkpoint_root=self._checkpoint_root, sandbox=host_sandbox or self._root, task_id=self._task_id, thermos_json=self._task_filename, hostname=self._hostname) if getpass.getuser() == 'root' and self._role: params.update(setuid=self._role) cmdline_args = [sys.executable, self._runner_pex] cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items()) if self._enable_chroot: cmdline_args.extend(['--enable_chroot']) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args
def _cmdline(self): params = dict( log_dir=LogOptions.log_dir(), log_to_disk="DEBUG", checkpoint_root=self._checkpoint_root, sandbox=self._sandbox.root, container_sandbox=self._sandbox.container_root, task_id=self._task_id, thermos_json=self._task_filename, hostname=self._hostname, process_logger_destination=self._process_logger_destination, process_logger_mode=self._process_logger_mode, rotate_log_size_mb=self._rotate_log_size_mb, rotate_log_backups=self._rotate_log_backups, ) if getpass.getuser() == "root" and self._role: params.update(setuid=self._role) cmdline_args = [sys.executable, self._runner_pex] cmdline_args.extend("--%s=%s" % (flag, value) for flag, value in params.items() if value is not None) if self._enable_chroot: cmdline_args.extend(["--enable_chroot"]) if self._preserve_env: cmdline_args.extend(["--preserve_env"]) if self._sandbox.is_filesystem_image: cmdline_args.extend(["--mesos_containerizer_path=%s" % self._mesos_containerizer_path]) for name, port in self._ports.items(): cmdline_args.extend(["--port=%s:%s" % (name, port)]) return cmdline_args
def _setup_aggregated_disk_logging(filebase): filename = os.path.join(LogOptions.log_dir(), filebase + '.log') formatter = ProxyFormatter(LogOptions.disk_log_scheme) file_handler = PreambleFileHandler(filename, formatter.preamble()) file_handler.setFormatter(formatter) file_handler.addFilter(GenericFilter(lambda level: level >= LogOptions.disk_log_level())) return [file_handler]
def _cmdline(self): host_sandbox = None if os.environ.get("MESOS_DIRECTORY"): host_sandbox = os.path.join(os.environ.get("MESOS_DIRECTORY"), "sandbox") params = dict( log_dir=LogOptions.log_dir(), log_to_disk="DEBUG", checkpoint_root=self._checkpoint_root, sandbox=host_sandbox or self._root, task_id=self._task_id, thermos_json=self._task_filename, hostname=self._hostname, process_logger_mode=self._process_logger_mode, rotate_log_size_mb=self._rotate_log_size_mb, rotate_log_backups=self._rotate_log_backups, ) if getpass.getuser() == "root" and self._role: params.update(setuid=self._role) cmdline_args = [sys.executable, self._runner_pex] cmdline_args.extend("--%s=%s" % (flag, value) for flag, value in params.items() if value is not None) if self._enable_chroot: cmdline_args.extend(["--enable_chroot"]) for name, port in self._ports.items(): cmdline_args.extend(["--port=%s:%s" % (name, port)]) return cmdline_args
def _setup_aggregated_disk_logging(filebase): filename = os.path.join(LogOptions.log_dir(), filebase + '.log') formatter = ProxyFormatter(LogOptions.disk_log_scheme) file_handler = PreambleFileHandler(filename, formatter.preamble()) file_handler.setFormatter(formatter) file_handler.addFilter( GenericFilter(lambda level: level >= LogOptions.disk_log_level())) return [file_handler]
def init(filebase=None): """ Sets up default stderr logging and, if filebase is supplied, sets up disk logging using: {--log_dir}/filebase.{INFO,WARNING,...} If '--log_simple' is specified, logs are written into a single file: {--log_dir}/filebase.log """ logging._acquireLock() # set up permissive logger root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) # clear existing handlers teardown_scribe_logging() teardown_stderr_logging() teardown_disk_logging() for handler in root_logger.handlers: root_logger.removeHandler(handler) # setup INFO...FATAL handlers if filebase: _initialize_disk_logging() initializer = _setup_aggregated_disk_logging if LogOptions.simple( ) else _setup_disk_logging for handler in initializer(filebase): root_logger.addHandler(handler) _DISK_LOGGERS.append(handler) if LogOptions._is_scribe_logging_required(): try: for handler in _setup_scribe_logging(): root_logger.addHandler(handler) _SCRIBE_LOGGERS.append(handler) except ScribeHandler.ScribeHandlerException as err: print_stderr(err) for handler in _setup_stderr_logging(): root_logger.addHandler(handler) _STDERR_LOGGERS.append(handler) logging._releaseLock() if len(_DISK_LOGGERS) > 0: print_stderr('Writing log files to disk in %s' % LogOptions.log_dir()) if len(_SCRIBE_LOGGERS) > 0: print_stderr('Sending log messages to scribe host=%s:%d category=%s' % (LogOptions.scribe_host(), LogOptions.scribe_port(), LogOptions.scribe_category())) return root_logger
def init(filebase=None): """ Sets up default stderr logging and, if filebase is supplied, sets up disk logging using: {--log_dir}/filebase.{INFO,WARNING,...} If '--log_simple' is specified, logs are written into a single file: {--log_dir}/filebase.log """ logging._acquireLock() # set up permissive logger root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) # clear existing handlers teardown_scribe_logging() teardown_stderr_logging() teardown_disk_logging() for handler in root_logger.handlers: root_logger.removeHandler(handler) # setup INFO...FATAL handlers if filebase: _initialize_disk_logging() initializer = _setup_aggregated_disk_logging if LogOptions.simple() else _setup_disk_logging for handler in initializer(filebase): root_logger.addHandler(handler) _DISK_LOGGERS.append(handler) if LogOptions._is_scribe_logging_required(): try: for handler in _setup_scribe_logging(): root_logger.addHandler(handler) _SCRIBE_LOGGERS.append(handler) except ScribeHandler.ScribeHandlerException as err: print_stderr(err) for handler in _setup_stderr_logging(): root_logger.addHandler(handler) _STDERR_LOGGERS.append(handler) logging._releaseLock() if len(_DISK_LOGGERS) > 0: print_stderr('Writing log files to disk in %s' % LogOptions.log_dir()) if len(_SCRIBE_LOGGERS) > 0: print_stderr('Sending log messages to scribe host=%s:%d category=%s' % (LogOptions.scribe_host(), LogOptions.scribe_port(), LogOptions.scribe_category())) return root_logger
def _cmdline(self): params = dict(log_dir=LogOptions.log_dir(), log_to_disk='DEBUG', checkpoint_root=self._checkpoint_root, sandbox=self._root, task_id=self._task_id, thermos_json=self._task_filename) if getpass.getuser() == 'root': params.update(setuid=self._role) cmdline_args = [self._runner_pex] cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items()) if self._enable_chroot: cmdline_args.extend(['--enable_chroot']) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args
def _cmdline(self): params = dict(log_dir=LogOptions.log_dir(), log_to_disk='DEBUG', checkpoint_root=self._checkpoint_root, sandbox=self._root, task_id=self._task_id, thermos_json=self._task_filename) if getpass.getuser() == 'root': params.update(setuid=self._role) cmdline_args = [self._runner_pex] cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items()) if self._enable_chroot: cmdline_args.extend(['--enable_chroot']) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args
def _setup_disk_logging(filebase): handlers = [] logroot = LogOptions.log_dir() safe_mkdir(logroot) now = time.localtime() def gen_filter(level): return GenericFilter( lambda record_level: record_level == level and level >= LogOptions.disk_log_level()) def gen_link_filename(filebase, level): return '%(filebase)s.%(level)s' % { 'filebase': filebase, 'level': level, } hostname = gethostname() username = getpass.getuser() pid = os.getpid() datestring = time.strftime('%Y%m%d-%H%M%S', time.localtime()) def gen_verbose_filename(filebase, level): return '%(filebase)s.%(hostname)s.%(user)s.log.%(level)s.%(date)s.%(pid)s' % { 'filebase': filebase, 'hostname': hostname, 'user': username, 'level': level, 'date': datestring, 'pid': pid } for filter_type, filter_name in _FILTER_TYPES.items(): formatter = ProxyFormatter(LogOptions.disk_log_scheme) filter = gen_filter(filter_type) full_filebase = os.path.join(logroot, filebase) logfile_link = gen_link_filename(full_filebase, filter_name) logfile_full = gen_verbose_filename(full_filebase, filter_name) file_handler = logging.FileHandler(logfile_full) file_handler.setFormatter(formatter) file_handler.addFilter(filter) handlers.append(file_handler) _safe_setup_link(logfile_link, logfile_full) return handlers
def _setup_disk_logging(filebase): handlers = [] logroot = LogOptions.log_dir() safe_mkdir(logroot) def gen_filter(level): return GenericFilter(lambda record_level: record_level == level and level >= LogOptions.disk_log_level()) def gen_link_filename(filebase, level): return '%(filebase)s.%(level)s' % { 'filebase': filebase, 'level': level, } hostname = gethostname() username = getpass.getuser() pid = os.getpid() datestring = time.strftime('%Y%m%d-%H%M%S', time.localtime()) def gen_verbose_filename(filebase, level): return '%(filebase)s.%(hostname)s.%(user)s.log.%(level)s.%(date)s.%(pid)s' % { 'filebase': filebase, 'hostname': hostname, 'user': username, 'level': level, 'date': datestring, 'pid': pid } for filter_type, filter_name in _FILTER_TYPES.items(): formatter = ProxyFormatter(LogOptions.disk_log_scheme) filter = gen_filter(filter_type) full_filebase = os.path.join(logroot, filebase) logfile_link = gen_link_filename(full_filebase, filter_name) logfile_full = gen_verbose_filename(full_filebase, filter_name) file_handler = logging.FileHandler(logfile_full) file_handler.setFormatter(formatter) file_handler.addFilter(filter) handlers.append(file_handler) _safe_setup_link(logfile_link, logfile_full) return handlers
def _cmdline(self): host_sandbox = None if os.environ.get('MESOS_DIRECTORY'): host_sandbox = os.path.join(os.environ.get('MESOS_DIRECTORY'), 'sandbox') params = dict(log_dir=LogOptions.log_dir(), log_to_disk='DEBUG', checkpoint_root=self._checkpoint_root, sandbox=host_sandbox or self._root, task_id=self._task_id, thermos_json=self._task_filename, hostname=self._hostname) if getpass.getuser() == 'root' and self._role: params.update(setuid=self._role) cmdline_args = [sys.executable, self._runner_pex] cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items()) if self._enable_chroot: cmdline_args.extend(['--enable_chroot']) for name, port in self._ports.items(): cmdline_args.extend(['--port=%s:%s' % (name, port)]) return cmdline_args
def _initialize_disk_logging(): safe_mkdir(LogOptions.log_dir())
def _initialize_disk_logging(): safe_mkdir(LogOptions.log_dir())