def __init__(self, period=None):
self._metrics = RootMetrics()
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
def __init__(self, period=None):
self._metrics = RootMetrics()
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
class VarsEndpoint(object):
"""
Wrap a MetricSampler to export the /vars endpoint for applications that register
exported variables.
"""
def __init__(self, period=None):
self._metrics = RootMetrics()
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
@HttpServer.route("/vars")
@HttpServer.route("/vars/:var")
def handle_vars(self, var=None):
HttpServer.set_content_type('text/plain; charset=iso-8859-1')
samples = self._monitor.sample()
if var is None:
return '\n'.join('%s %s' % (key, val)
for key, val in sorted(samples.items()))
else:
if var in samples:
return samples[var]
else:
HttpServer.abort(404, 'Unknown exported variable')
@HttpServer.route("/vars.json")
def handle_vars_json(self, var=None, value=None):
return self._monitor.sample()
def shutdown(self):
self._monitor.shutdown()
self._monitor.join()
class VarsEndpoint(object):
"""
Wrap a MetricSampler to export the /vars endpoint for applications that register
exported variables.
"""
def __init__(self, period=None):
self._metrics = RootMetrics()
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
@HttpServer.route("/vars")
@HttpServer.route("/vars/:var")
def handle_vars(self, var=None):
samples = self._monitor.sample()
if var is None:
body = '<br>'.join('%s %s' % (key, val)
for key, val in samples.items())
return '<html><body><pre>%s</pre></body></html>' % body
else:
if var in samples:
return samples[var]
else:
HttpServer.abort(404, 'Unknown exported variable')
@HttpServer.route("/vars.json")
def handle_vars_json(self, var=None, value=None):
return self._monitor.sample()
def shutdown(self):
self._monitor.shutdown()
self._monitor.join()
class VarsEndpoint(object):
"""
Wrap a MetricSampler to export the /vars endpoint for applications that register
exported variables.
"""
def __init__(self, period=None, stats_filter=None):
self._metrics = RootMetrics()
self._stats_filter = stats_filter
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
@HttpServer.route("/vars")
@HttpServer.route("/vars/:var")
def handle_vars(self, var=None):
HttpServer.set_content_type('text/plain; charset=iso-8859-1')
filtered = self._parse_filtered_arg()
samples = self._monitor.sample()
if var is None and filtered and self._stats_filter:
return '\n'.join('%s %s' % (key, val)
for key, val in sorted(samples.items())
if not self._stats_filter.match(key))
elif var is None:
return '\n'.join('%s %s' % (key, val)
for key, val in sorted(samples.items()))
else:
if var in samples:
return samples[var]
else:
HttpServer.abort(404, 'Unknown exported variable')
@HttpServer.route("/vars.json")
def handle_vars_json(self, var=None, value=None):
filtered = self._parse_filtered_arg()
sample = self._monitor.sample()
if filtered and self._stats_filter:
return dict((key, val) for key, val in sample.items()
if not self._stats_filter.match(key))
else:
return sample
def shutdown(self):
self._monitor.shutdown()
self._monitor.join()
def _parse_filtered_arg(self):
return request.GET.get('filtered', '') in ('true', '1')
class VarsEndpoint(object):
"""
Wrap a MetricSampler to export the /vars endpoint for applications that register
exported variables.
"""
def __init__(self, period=None, stats_filter=None):
self._metrics = RootMetrics()
self._stats_filter = stats_filter
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
@HttpServer.route("/vars")
@HttpServer.route("/vars/:var")
def handle_vars(self, var=None):
HttpServer.set_content_type('text/plain; charset=iso-8859-1')
filtered = self._parse_filtered_arg()
samples = self._monitor.sample()
if var is None and filtered and self._stats_filter:
return '\n'.join(
'%s %s' % (key, val) for key, val in sorted(samples.items())
if not self._stats_filter.match(key))
elif var is None:
return '\n'.join(
'%s %s' % (key, val) for key, val in sorted(samples.items()))
else:
if var in samples:
return samples[var]
else:
HttpServer.abort(404, 'Unknown exported variable')
@HttpServer.route("/vars.json")
def handle_vars_json(self, var=None, value=None):
filtered = self._parse_filtered_arg()
sample = self._monitor.sample()
if filtered and self._stats_filter:
return dict((key, val) for key, val in sample.items() if not self._stats_filter.match(key))
else:
return sample
def shutdown(self):
self._monitor.shutdown()
self._monitor.join()
def _parse_filtered_arg(self):
return request.GET.get('filtered', '') in ('true', '1')
class VarsEndpoint(object):
"""
Wrap a MetricSampler to export the /vars endpoint for applications that register
exported variables.
"""
def __init__(self, period=None):
self._metrics = RootMetrics()
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
@HttpServer.route("/vars")
@HttpServer.route("/vars/:var")
def handle_vars(self, var=None):
HttpServer.set_content_type('text/plain; charset=iso-8859-1')
samples = self._monitor.sample()
if var is None:
return '\n'.join(
'%s %s' % (key, val) for key, val in sorted(samples.items()))
else:
if var in samples:
return samples[var]
else:
HttpServer.abort(404, 'Unknown exported variable')
@HttpServer.route("/vars.json")
def handle_vars_json(self, var=None, value=None):
return self._monitor.sample()
def shutdown(self):
self._monitor.shutdown()
self._monitor.join()
class VarsEndpoint(object):
"""
Wrap a MetricSampler to export the /vars endpoint for applications that register
exported variables.
"""
def __init__(self, period=None):
self._metrics = RootMetrics()
if period is not None:
self._monitor = MetricSampler(self._metrics, period)
else:
self._monitor = MetricSampler(self._metrics)
self._monitor.start()
@HttpServer.route("/vars")
@HttpServer.route("/vars/:var")
def handle_vars(self, var = None):
samples = self._monitor.sample()
if var is None:
body='<br>'.join(
'%s %s' % (key, val) for key, val in samples.iteritems())
return '<html><body><pre>%s</pre></body></html>' % body
else:
if var in samples:
return samples[var]
else:
HttpServer.abort(404, 'Unknown exported variable')
@HttpServer.route("/vars.json")
def handle_vars_json(self, var = None, value = None):
return self._monitor.sample()
def shutdown(self):
self._monitor.shutdown()
self._monitor.join()
def main(args, options):
log.info("Options in use: %s", options)
if not options.api_port:
app.error('Must specify --port')
if not options.mesos_master:
app.error('Must specify --mesos_master')
if not options.framework_user:
app.error('Must specify --framework_user')
if not options.executor_uri:
app.error('Must specify --executor_uri')
if not options.executor_cmd:
app.error('Must specify --executor_cmd')
if not options.zk_url:
app.error('Must specify --zk_url')
if not options.admin_keypath:
app.error('Must specify --admin_keypath')
if not options.scheduler_keypath:
app.error('Must specify --scheduler_keypath')
if options.verbose:
LogOptions.set_stderr_log_level('google:DEBUG')
try:
election_timeout = parse_time(options.election_timeout)
framework_failover_timeout = parse_time(options.framework_failover_timeout)
except InvalidTime as e:
app.error(e.message)
try:
_, zk_servers, zk_root = zookeeper.parse(options.zk_url)
except Exception as e:
app.error("Invalid --zk_url: %s" % e.message)
web_assets_dir = os.path.join(options.work_dir, "web")
pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH)
log.info("Extracted web assets into %s" % options.work_dir)
fw_principal = None
fw_secret = None
if options.framework_authentication_file:
try:
with open(options.framework_authentication_file, "r") as f:
cred = yaml.load(f)
fw_principal = cred["principal"]
fw_secret = cred["secret"]
log.info("Loaded credential (principal=%s) for framework authentication" % fw_principal)
except IOError as e:
app.error("Unable to read the framework authentication key file: %s" % e)
except (KeyError, yaml.YAMLError) as e:
app.error("Invalid framework authentication key file format %s" % e)
scheduler_key = None
try:
with open(options.scheduler_keypath, 'rb') as f:
scheduler_key = f.read().strip()
if not scheduler_key:
raise ValueError("The key file is empty")
except Exception as e:
app.error("Cannot read --scheduler_keypath: %s" % e)
log.info("Starting Mysos scheduler")
kazoo = KazooClient(zk_servers)
kazoo.start()
if options.state_storage == 'zk':
log.info("Using ZooKeeper (path: %s) for state storage" % zk_root)
state_provider = ZooKeeperStateProvider(kazoo, zk_root)
else:
log.info("Using local disk for state storage")
state_provider = LocalStateProvider(options.work_dir)
try:
state = state_provider.load_scheduler_state()
except StateProvider.Error as e:
app.error(e.message)
if state:
log.info("Successfully restored scheduler state")
framework_info = state.framework_info
if framework_info.HasField('id'):
log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value)
else:
log.info("No scheduler state to restore")
framework_info = FrameworkInfo(
user=options.framework_user,
name=FRAMEWORK_NAME,
checkpoint=True,
failover_timeout=framework_failover_timeout.as_(Time.SECONDS),
role=options.framework_role,
hostname=options.hostname,
webui_url="http://%s:%s/" % (options.hostname, options.api_port))
if fw_principal:
framework_info.principal = fw_principal
state = Scheduler(framework_info)
state_provider.dump_scheduler_state(state)
scheduler = MysosScheduler(
state,
state_provider,
options.framework_user,
options.executor_uri,
options.executor_cmd,
kazoo,
options.zk_url,
election_timeout,
options.admin_keypath,
scheduler_key,
installer_args=options.installer_args,
backup_store_args=options.backup_store_args,
executor_environ=options.executor_environ,
executor_source_prefix=options.executor_source_prefix,
docker_image=options.docker_image,
framework_role=options.framework_role)
RootMetrics().register_observable('scheduler', scheduler)
if fw_principal and fw_secret:
cred = Credential(principal=fw_principal, secret=fw_secret)
scheduler_driver = mesos.native.MesosSchedulerDriver(
scheduler,
framework_info,
options.mesos_master,
cred)
else:
scheduler_driver = mesos.native.MesosSchedulerDriver(
scheduler,
framework_info,
options.mesos_master)
scheduler_driver.start()
metric_sampler = MetricSampler(RootMetrics())
metric_sampler.start()
server = HttpServer()
server.mount_routes(MysosServer(scheduler, web_assets_dir, metric_sampler))
et = ExceptionalThread(
target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy'))
et.daemon = True
et.start()
try:
# Wait for the scheduler to stop.
# The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the
# process with SIGINT.
while not scheduler.stopped.wait(timeout=0.5):
pass
except KeyboardInterrupt:
log.info('Interrupted, exiting.')
else:
log.info('Scheduler exited.')
app.shutdown(1) # Mysos scheduler is supposed to be long-running thus the use of exit status 1.
def setUp(self):
self._scheduler = FakeScheduler()
self._app = TestApp(
MysosServer(self._scheduler, self.web_assets_dir,
MetricSampler(RootMetrics())).app)
