def proxy_main():
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_executor = initialize(options)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor and wait until it is stopped.
driver_thread = ExecutorDriverThread(driver)
driver_thread.start()
try:
while driver_thread.isAlive():
driver_thread.join(5)
except (KeyboardInterrupt, SystemExit):
driver.stop()
raise
log.info('MesosExecutorDriver.run() has finished.')
app.register_module(ExceptionTerminationHandler())
app.main()
def proxy_main():
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_executor = initialize(options)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor and wait until it is stopped.
driver_thread = ExecutorDriverThread(driver)
driver_thread.start()
try:
while driver_thread.isAlive():
driver_thread.join(5)
except (KeyboardInterrupt, SystemExit):
driver.stop()
raise
log.info('MesosExecutorDriver.run() has finished.')
app.register_module(ExceptionTerminationHandler())
app.main()
# limitations under the License.
# ==================================================================================================
import logging
from initialize import init
try:
from twitter.common import app
class LoggingSubsystem(app.Module):
def __init__(self):
app.Module.__init__(self, __name__, description="Logging subsystem.")
def setup_function(self):
init(app.name())
app.register_module(LoggingSubsystem())
except ImportError:
# Do not require twitter.common.app
pass
debug = logging.debug
info = logging.info
warning = logging.warning
warn = logging.warning
error = logging.error
fatal = logging.fatal
log = logging.log
logger = logging.getLogger
DEBUG = logging.DEBUG
INFO = logging.INFO
from twitter.common import app
from twitter.common.log.options import LogOptions
class LoggingSubsystem(app.Module):
def __init__(self):
app.Module.__init__(self,
__name__,
description="Logging subsystem.")
def setup_function(self):
if not LogOptions._is_disk_logging_required():
init()
else:
init(app.name())
app.register_module(LoggingSubsystem())
except ImportError:
# Do not require twitter.common.app
pass
debug = logging.debug
info = logging.info
warning = logging.warning
warn = logging.warning
error = logging.error
fatal = logging.fatal
log = logging.log
logger = logging.getLogger
DEBUG = logging.DEBUG
INFO = logging.INFO
path_detector = MesosPathDetector(options.mesos_root)
disk_collector_settings = DiskCollectorSettings(
options.agent_api_url, options.executor_id_json_path,
options.disk_usage_json_path,
Amount(options.task_disk_collection_interval_secs, Time.SECONDS))
return TaskObserver(
path_detector,
Amount(options.polling_interval_secs, Time.SECONDS),
Amount(options.task_process_collection_interval_secs, Time.SECONDS),
enable_mesos_disk_collector=options.enable_mesos_disk_collector,
disk_collector_settings=disk_collector_settings)
def main(_, options):
observer = initialize(options)
observer.start()
root_server = configure_server(observer)
server = ExceptionalThread(
target=lambda: root_server.run(options.ip, options.port, 'cherrypy'))
server.daemon = True
server.start()
sleep_forever()
LogOptions.set_stderr_log_level('google:INFO')
app.register_module(ExceptionTerminationHandler())
app.main()
def __init__(self):
app.Module.__init__(self, __name__, description='Zookeeper logging subsystem.')
def setup_function(self):
if app.get_options().twitter_common_zookeeper_debug:
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_DEBUG)
else:
self._set_default_log_level()
def _set_default_log_level(self):
log_level = LogOptions.stderr_log_level()
zk_log_level = ZookeeperLoggingSubsystem._ZK_LOG_LEVEL_MAP.get(
log_level, zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_debug_level(zk_log_level)
app.register_module(ZookeeperLoggingSubsystem())
class ZooKeeper(object):
"""A convenience wrapper around the low-level ZooKeeper API.
Blocks until the initial connection is established, and proxies method calls
to the corresponding ZK functions, passing the handle.
eg.
zk = ZooKeeper('localhost:2181')
print zk.get_children('/')
See the "zookeeper" module for details on indiviual methods.
:param servers: Comma-separated list of host:ports to connect to. Defaults to
self._set_log_level(log_level_override=log_level_override)
def _set_log_level(self, log_level_override=''):
stderr_log_level = LogOptions.stderr_log_level()
# set default level to FATAL.
# we do this here (instead of add_option) to distinguish when an override is set.
if stderr_log_level == log.INFO and log_level_override != 'INFO':
stderr_log_level = log.FATAL
# default to using stderr logging level, setting override if applicable
log_level = getattr(log, log_level_override, stderr_log_level)
# set the logger
zk_log_level = ZookeeperLoggingSubsystem._ZK_LOG_LEVEL_MAP.get(
log_level, zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_debug_level(zk_log_level)
app.register_module(ZookeeperLoggingSubsystem())
class Perms(object):
READ = zookeeper.PERM_READ
WRITE = zookeeper.PERM_WRITE
CREATE = zookeeper.PERM_CREATE
DELETE = zookeeper.PERM_DELETE
ADMIN = zookeeper.PERM_ADMIN
ALL = zookeeper.PERM_ALL
class Ids(object):
ANYONE_ID_UNSAFE = Id('world', 'anyone')
AUTH_IDS = Id('auth', '')
log.error('Internal error: %s', err)
sys.exit(INTERNAL_ERROR)
except TaskRunner.InvalidTask as err:
log.error('Invalid task: %s', err)
sys.exit(INVALID_TASK)
except TaskRunner.StateError as err:
log.error('Checkpoint error: %s', err)
sys.exit(TERMINAL_TASK)
except Process.UnknownUserError as err:
log.error('User ceased to exist: %s', err)
sys.exit(UNKNOWN_USER)
except KeyboardInterrupt:
log.info('Caught ^C, tearing down runner.')
runner_teardown(task_runner)
except Exception as e:
log.error('Unknown exception: %s', e)
for line in traceback.format_exc().splitlines():
log.error(line)
sys.exit(UNKNOWN_ERROR)
def main(args, opts):
return proxy_main(args, opts)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
app.register_module(ExceptionTerminationHandler())
app.main()