def _cmdline(self):
params = dict(
log_dir=LogOptions.log_dir(),
log_to_disk='DEBUG',
checkpoint_root=self._checkpoint_root,
sandbox=self._sandbox.root,
container_sandbox=self._sandbox.container_root,
task_id=self._task_id,
thermos_json=self._task_filename,
hostname=self._hostname,
process_logger_destination=self._process_logger_destination,
process_logger_mode=self._process_logger_mode,
rotate_log_size_mb=self._rotate_log_size_mb,
rotate_log_backups=self._rotate_log_backups)
if getpass.getuser() == 'root' and self._role:
params.update(setuid=self._role)
cmdline_args = [sys.executable, self._runner_pex]
cmdline_args.extend('--%s=%s' % (flag, value)
for flag, value in params.items()
if value is not None)
if self._enable_chroot:
cmdline_args.extend(['--enable_chroot'])
if self._preserve_env:
cmdline_args.extend(['--preserve_env'])
if self._sandbox.is_filesystem_image:
cmdline_args.extend([
'--mesos_containerizer_path=%s' %
self._mesos_containerizer_path
])
for name, port in self._ports.items():
cmdline_args.extend(['--port=%s:%s' % (name, port)])
return cmdline_args
def init(filebase=None):
"""
Sets up default stderr logging and, if filebase is supplied, sets up disk logging using:
{--log_dir}/filebase.{INFO,WARNING,...}
"""
logging._acquireLock()
# set up permissive logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# clear existing handlers
teardown_stderr_logging()
teardown_disk_logging()
for handler in root_logger.handlers:
root_logger.removeHandler(handler)
# setup INFO...FATAL handlers
if filebase:
for handler in _setup_disk_logging(filebase):
root_logger.addHandler(handler)
_DISK_LOGGERS.append(handler)
for handler in _setup_stderr_logging():
root_logger.addHandler(handler)
_STDERR_LOGGERS.append(handler)
logging._releaseLock()
if len(_DISK_LOGGERS) > 0 and LogOptions.stderr_log_level(
) != LogOptions.LOG_LEVEL_NONE:
print('Writing log files to disk in %s' % LogOptions.log_dir(),
file=sys.stderr)
return root_logger
def init(filebase):
"""
Set up default logging using:
{--log_dir}/filebase.{INFO,WARNING,...}
"""
logging._acquireLock()
# set up permissive logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# clear existing handlers
teardown_stderr_logging()
teardown_disk_logging()
# setup INFO...FATAL handlers
num_disk_handlers = 0
for handler in _setup_disk_logging(filebase):
root_logger.addHandler(handler)
_DISK_LOGGERS.append(handler)
for handler in _setup_stderr_logging():
root_logger.addHandler(handler)
_STDERR_LOGGERS.append(handler)
logging._releaseLock()
if len(_DISK_LOGGERS) > 0 and LogOptions.stderr_log_level() != LogOptions.LOG_LEVEL_NONE:
print('Writing log files to disk in %s' % LogOptions.log_dir(), file=sys.stderr)
return root_logger
def _cmdline(self):
host_sandbox = None
if os.environ.get('MESOS_DIRECTORY'):
host_sandbox = os.path.join(os.environ.get('MESOS_DIRECTORY'), 'sandbox')
params = dict(log_dir=LogOptions.log_dir(),
log_to_disk='DEBUG',
checkpoint_root=self._checkpoint_root,
sandbox=host_sandbox or self._root,
task_id=self._task_id,
thermos_json=self._task_filename,
hostname=self._hostname,
process_logger_destination=self._process_logger_destination,
process_logger_mode=self._process_logger_mode,
rotate_log_size_mb=self._rotate_log_size_mb,
rotate_log_backups=self._rotate_log_backups)
if getpass.getuser() == 'root' and self._role:
params.update(setuid=self._role)
cmdline_args = [sys.executable, self._runner_pex]
cmdline_args.extend(
'--%s=%s' % (flag, value) for flag, value in params.items() if value is not None)
if self._enable_chroot:
cmdline_args.extend(['--enable_chroot'])
if self._preserve_env:
cmdline_args.extend(['--preserve_env'])
if self._sandbox.is_filesystem_image:
cmdline_args.extend(
['--mesos_containerizer_path=%s' % self._mesos_containerizer_path])
for name, port in self._ports.items():
cmdline_args.extend(['--port=%s:%s' % (name, port)])
return cmdline_args
def _cmdline(self):
host_sandbox = None
if os.environ.get('MESOS_DIRECTORY'):
host_sandbox = os.path.join(os.environ.get('MESOS_DIRECTORY'),
'sandbox')
params = dict(log_dir=LogOptions.log_dir(),
log_to_disk='DEBUG',
checkpoint_root=self._checkpoint_root,
sandbox=host_sandbox or self._root,
task_id=self._task_id,
thermos_json=self._task_filename,
hostname=self._hostname)
if getpass.getuser() == 'root' and self._role:
params.update(setuid=self._role)
cmdline_args = [sys.executable, self._runner_pex]
cmdline_args.extend('--%s=%s' % (flag, value)
for flag, value in params.items())
if self._enable_chroot:
cmdline_args.extend(['--enable_chroot'])
for name, port in self._ports.items():
cmdline_args.extend(['--port=%s:%s' % (name, port)])
return cmdline_args
def _cmdline(self):
params = dict(
log_dir=LogOptions.log_dir(),
log_to_disk="DEBUG",
checkpoint_root=self._checkpoint_root,
sandbox=self._sandbox.root,
container_sandbox=self._sandbox.container_root,
task_id=self._task_id,
thermos_json=self._task_filename,
hostname=self._hostname,
process_logger_destination=self._process_logger_destination,
process_logger_mode=self._process_logger_mode,
rotate_log_size_mb=self._rotate_log_size_mb,
rotate_log_backups=self._rotate_log_backups,
)
if getpass.getuser() == "root" and self._role:
params.update(setuid=self._role)
cmdline_args = [sys.executable, self._runner_pex]
cmdline_args.extend("--%s=%s" % (flag, value) for flag, value in params.items() if value is not None)
if self._enable_chroot:
cmdline_args.extend(["--enable_chroot"])
if self._preserve_env:
cmdline_args.extend(["--preserve_env"])
if self._sandbox.is_filesystem_image:
cmdline_args.extend(["--mesos_containerizer_path=%s" % self._mesos_containerizer_path])
for name, port in self._ports.items():
cmdline_args.extend(["--port=%s:%s" % (name, port)])
return cmdline_args
def _setup_aggregated_disk_logging(filebase): filename = os.path.join(LogOptions.log_dir(), filebase + '.log') formatter = ProxyFormatter(LogOptions.disk_log_scheme) file_handler = PreambleFileHandler(filename, formatter.preamble()) file_handler.setFormatter(formatter) file_handler.addFilter(GenericFilter(lambda level: level >= LogOptions.disk_log_level())) return [file_handler]
def _cmdline(self):
host_sandbox = None
if os.environ.get("MESOS_DIRECTORY"):
host_sandbox = os.path.join(os.environ.get("MESOS_DIRECTORY"), "sandbox")
params = dict(
log_dir=LogOptions.log_dir(),
log_to_disk="DEBUG",
checkpoint_root=self._checkpoint_root,
sandbox=host_sandbox or self._root,
task_id=self._task_id,
thermos_json=self._task_filename,
hostname=self._hostname,
process_logger_mode=self._process_logger_mode,
rotate_log_size_mb=self._rotate_log_size_mb,
rotate_log_backups=self._rotate_log_backups,
)
if getpass.getuser() == "root" and self._role:
params.update(setuid=self._role)
cmdline_args = [sys.executable, self._runner_pex]
cmdline_args.extend("--%s=%s" % (flag, value) for flag, value in params.items() if value is not None)
if self._enable_chroot:
cmdline_args.extend(["--enable_chroot"])
for name, port in self._ports.items():
cmdline_args.extend(["--port=%s:%s" % (name, port)])
return cmdline_args
def _setup_aggregated_disk_logging(filebase):
filename = os.path.join(LogOptions.log_dir(), filebase + '.log')
formatter = ProxyFormatter(LogOptions.disk_log_scheme)
file_handler = PreambleFileHandler(filename, formatter.preamble())
file_handler.setFormatter(formatter)
file_handler.addFilter(
GenericFilter(lambda level: level >= LogOptions.disk_log_level()))
return [file_handler]
def init(filebase=None):
"""
Sets up default stderr logging and, if filebase is supplied, sets up disk logging using:
{--log_dir}/filebase.{INFO,WARNING,...}
If '--log_simple' is specified, logs are written into a single file:
{--log_dir}/filebase.log
"""
logging._acquireLock()
# set up permissive logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# clear existing handlers
teardown_scribe_logging()
teardown_stderr_logging()
teardown_disk_logging()
for handler in root_logger.handlers:
root_logger.removeHandler(handler)
# setup INFO...FATAL handlers
if filebase:
_initialize_disk_logging()
initializer = _setup_aggregated_disk_logging if LogOptions.simple(
) else _setup_disk_logging
for handler in initializer(filebase):
root_logger.addHandler(handler)
_DISK_LOGGERS.append(handler)
if LogOptions._is_scribe_logging_required():
try:
for handler in _setup_scribe_logging():
root_logger.addHandler(handler)
_SCRIBE_LOGGERS.append(handler)
except ScribeHandler.ScribeHandlerException as err:
print_stderr(err)
for handler in _setup_stderr_logging():
root_logger.addHandler(handler)
_STDERR_LOGGERS.append(handler)
logging._releaseLock()
if len(_DISK_LOGGERS) > 0:
print_stderr('Writing log files to disk in %s' % LogOptions.log_dir())
if len(_SCRIBE_LOGGERS) > 0:
print_stderr('Sending log messages to scribe host=%s:%d category=%s' %
(LogOptions.scribe_host(), LogOptions.scribe_port(),
LogOptions.scribe_category()))
return root_logger
def init(filebase=None):
"""
Sets up default stderr logging and, if filebase is supplied, sets up disk logging using:
{--log_dir}/filebase.{INFO,WARNING,...}
If '--log_simple' is specified, logs are written into a single file:
{--log_dir}/filebase.log
"""
logging._acquireLock()
# set up permissive logger
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
# clear existing handlers
teardown_scribe_logging()
teardown_stderr_logging()
teardown_disk_logging()
for handler in root_logger.handlers:
root_logger.removeHandler(handler)
# setup INFO...FATAL handlers
if filebase:
_initialize_disk_logging()
initializer = _setup_aggregated_disk_logging if LogOptions.simple() else _setup_disk_logging
for handler in initializer(filebase):
root_logger.addHandler(handler)
_DISK_LOGGERS.append(handler)
if LogOptions._is_scribe_logging_required():
try:
for handler in _setup_scribe_logging():
root_logger.addHandler(handler)
_SCRIBE_LOGGERS.append(handler)
except ScribeHandler.ScribeHandlerException as err:
print_stderr(err)
for handler in _setup_stderr_logging():
root_logger.addHandler(handler)
_STDERR_LOGGERS.append(handler)
logging._releaseLock()
if len(_DISK_LOGGERS) > 0:
print_stderr('Writing log files to disk in %s' % LogOptions.log_dir())
if len(_SCRIBE_LOGGERS) > 0:
print_stderr('Sending log messages to scribe host=%s:%d category=%s'
% (LogOptions.scribe_host(), LogOptions.scribe_port(), LogOptions.scribe_category()))
return root_logger
def _cmdline(self):
params = dict(log_dir=LogOptions.log_dir(),
log_to_disk='DEBUG',
checkpoint_root=self._checkpoint_root,
sandbox=self._root,
task_id=self._task_id,
thermos_json=self._task_filename)
if getpass.getuser() == 'root':
params.update(setuid=self._role)
cmdline_args = [self._runner_pex]
cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items())
if self._enable_chroot:
cmdline_args.extend(['--enable_chroot'])
for name, port in self._ports.items():
cmdline_args.extend(['--port=%s:%s' % (name, port)])
return cmdline_args
def _cmdline(self):
params = dict(log_dir=LogOptions.log_dir(),
log_to_disk='DEBUG',
checkpoint_root=self._checkpoint_root,
sandbox=self._root,
task_id=self._task_id,
thermos_json=self._task_filename)
if getpass.getuser() == 'root':
params.update(setuid=self._role)
cmdline_args = [self._runner_pex]
cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items())
if self._enable_chroot:
cmdline_args.extend(['--enable_chroot'])
for name, port in self._ports.items():
cmdline_args.extend(['--port=%s:%s' % (name, port)])
return cmdline_args
def _setup_disk_logging(filebase):
handlers = []
logroot = LogOptions.log_dir()
safe_mkdir(logroot)
now = time.localtime()
def gen_filter(level):
return GenericFilter(
lambda record_level: record_level == level and level >= LogOptions.disk_log_level())
def gen_link_filename(filebase, level):
return '%(filebase)s.%(level)s' % {
'filebase': filebase,
'level': level,
}
hostname = gethostname()
username = getpass.getuser()
pid = os.getpid()
datestring = time.strftime('%Y%m%d-%H%M%S', time.localtime())
def gen_verbose_filename(filebase, level):
return '%(filebase)s.%(hostname)s.%(user)s.log.%(level)s.%(date)s.%(pid)s' % {
'filebase': filebase,
'hostname': hostname,
'user': username,
'level': level,
'date': datestring,
'pid': pid
}
for filter_type, filter_name in _FILTER_TYPES.items():
formatter = ProxyFormatter(LogOptions.disk_log_scheme)
filter = gen_filter(filter_type)
full_filebase = os.path.join(logroot, filebase)
logfile_link = gen_link_filename(full_filebase, filter_name)
logfile_full = gen_verbose_filename(full_filebase, filter_name)
file_handler = logging.FileHandler(logfile_full)
file_handler.setFormatter(formatter)
file_handler.addFilter(filter)
handlers.append(file_handler)
_safe_setup_link(logfile_link, logfile_full)
return handlers
def _setup_disk_logging(filebase):
handlers = []
logroot = LogOptions.log_dir()
safe_mkdir(logroot)
def gen_filter(level):
return GenericFilter(lambda record_level: record_level == level and
level >= LogOptions.disk_log_level())
def gen_link_filename(filebase, level):
return '%(filebase)s.%(level)s' % {
'filebase': filebase,
'level': level,
}
hostname = gethostname()
username = getpass.getuser()
pid = os.getpid()
datestring = time.strftime('%Y%m%d-%H%M%S', time.localtime())
def gen_verbose_filename(filebase, level):
return '%(filebase)s.%(hostname)s.%(user)s.log.%(level)s.%(date)s.%(pid)s' % {
'filebase': filebase,
'hostname': hostname,
'user': username,
'level': level,
'date': datestring,
'pid': pid
}
for filter_type, filter_name in _FILTER_TYPES.items():
formatter = ProxyFormatter(LogOptions.disk_log_scheme)
filter = gen_filter(filter_type)
full_filebase = os.path.join(logroot, filebase)
logfile_link = gen_link_filename(full_filebase, filter_name)
logfile_full = gen_verbose_filename(full_filebase, filter_name)
file_handler = logging.FileHandler(logfile_full)
file_handler.setFormatter(formatter)
file_handler.addFilter(filter)
handlers.append(file_handler)
_safe_setup_link(logfile_link, logfile_full)
return handlers
def _cmdline(self):
host_sandbox = None
if os.environ.get('MESOS_DIRECTORY'):
host_sandbox = os.path.join(os.environ.get('MESOS_DIRECTORY'), 'sandbox')
params = dict(log_dir=LogOptions.log_dir(),
log_to_disk='DEBUG',
checkpoint_root=self._checkpoint_root,
sandbox=host_sandbox or self._root,
task_id=self._task_id,
thermos_json=self._task_filename,
hostname=self._hostname)
if getpass.getuser() == 'root' and self._role:
params.update(setuid=self._role)
cmdline_args = [sys.executable, self._runner_pex]
cmdline_args.extend('--%s=%s' % (flag, value) for flag, value in params.items())
if self._enable_chroot:
cmdline_args.extend(['--enable_chroot'])
for name, port in self._ports.items():
cmdline_args.extend(['--port=%s:%s' % (name, port)])
return cmdline_args
def _initialize_disk_logging(): safe_mkdir(LogOptions.log_dir())
def _initialize_disk_logging():
safe_mkdir(LogOptions.log_dir())
