def run():
def main(args, opts):
"""Main"""
server = RedirServer(opts.zk_basepath,
opts.subdomain,
opts.base_domain)
thread = ExceptionalThread(
target=lambda: server.run(opts.listen,
opts.port,
server='cherrypy'))
thread.daemon = True
thread.start()
wait_forever()
log.LogOptions.set_stderr_log_level('google:INFO')
app.add_option('--port', help='http port', default=8080)
app.add_option('--listen',
help='IP address to listen for http connections.',
default='0.0.0.0')
app.add_option('--zk_basepath',
help='Zookeeper service path root.',
default='/aurora')
app.add_option('--base_domain',
help='Domain name of your site.',
default='example.com')
app.add_option('--subdomain',
help='Subdomain that roots Aurora job namespace.',
default='aurora')
app.main()
def proxy_main():
def main(args, opts):
if args:
print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
app.help()
sys.exit(1)
root_server = HttpServer()
root_server.mount_routes(DiagnosticsEndpoints())
task_observer = TaskObserver(opts.root)
task_observer.start()
bottle_wrapper = BottleObserver(task_observer)
root_server.mount_routes(bottle_wrapper)
def run():
root_server.run('0.0.0.0', opts.port, 'cherrypy')
et = ExceptionalThread(target=run)
et.daemon = True
et.start()
et.join()
app.main()
def proxy_main():
"""Proxy main function.
setuptools entrypoints with twitter.common.app is so awkward.
"""
def main(_, opts):
"""Main"""
if not opts.bucket:
log.error('--bucket is required.')
app.help()
server = S3Web(bucket=opts.bucket,
prefix=opts.prefix,
access_key_id=opts.access_key_id,
secret_key=opts.secret_key)
thread = ExceptionalThread(
target=lambda: server.run(opts.listen,
opts.port,
server='cherrypy'))
thread.daemon = True
thread.start()
log.info('Ready.')
app.wait_forever()
register_opts()
app.set_usage(__doc__)
app.set_option('twitter_common_log_stderr_log_level', 'google:INFO')
app.set_name('s3webfront')
app.main()
def proxy_main():
def main(args, options):
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# status providers:
status_providers = [HealthCheckerProvider()]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error('Must specify --announcer-ensemble if the announcer is enabled.')
status_providers.append(DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path))
# Create executor stub
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def proxy_main():
def main():
runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# Create executor stub
thermos_executor = ThermosExecutor(
runner_provider=runner_provider,
status_providers=(HealthCheckerProvider(),),
)
# Create driver stub
driver = mesos.MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def proxy_main():
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_executor = initialize(options)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor and wait until it is stopped.
driver_thread = ExecutorDriverThread(driver)
driver_thread.start()
try:
while driver_thread.isAlive():
driver_thread.join(5)
except (KeyboardInterrupt, SystemExit):
driver.stop()
raise
log.info('MesosExecutorDriver.run() has finished.')
app.register_module(ExceptionTerminationHandler())
app.main()
def proxy_main():
def main(args, options):
plugins = list_plugins()
if options.list_plugins:
for plugin in plugins:
print('\n%s' % plugin.__name__)
if plugin.__doc__:
for line in plugin.__doc__.splitlines():
print(' %s' % line)
else:
print(' No information')
return
if options.plugins:
plugins_map = dict((plugin.__name__, plugin) for plugin in plugins)
plugins = list(filter(None, map(plugins_map.get, options.plugins)))
if options.skip_plugins:
plugins_map = dict((plugin.__name__, plugin) for plugin in plugins)
for plugin in options.skip_plugins:
plugins_map.pop(plugin, None)
plugins = list(plugins_map.values())
if not args and options.diff is None:
options.diff = DEFAULT_BRANCH
if options.diff:
iterator = git_iterator(args, options)
else:
iterator = path_iterator(args, options)
severity = Nit.COMMENT
for number, name in Nit.SEVERITY.items():
if name == options.severity:
severity = number
should_fail = False
for filename, line_filter in iterator:
try:
python_file = PythonFile.parse(filename)
except SyntaxError as e:
print('%s:SyntaxError: %s' % (filename, e))
continue
for checker in plugins:
for nit in apply_filter(python_file, checker, line_filter):
if nit.severity >= severity:
print(nit)
print()
should_fail |= nit.severity >= Nit.ERROR or (
nit.severity >= Nit.WARNING and options.strict)
return int(should_fail)
app.main()
def proxy_main():
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=options.checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error('Must specify --announcer-ensemble if the announcer is enabled.')
status_providers.append(DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.abspath(CWD)
)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user)
)
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.abspath(CWD)
)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def proxy_main():
def main(args, options):
log.info('Starting testing mysos executor')
executor = MysosExecutor(
FakeTaskRunnerProvider(FakeTaskControlProvider()), Sandbox(SANDBOX_ROOT))
driver = mesos.native.MesosExecutorDriver(executor)
driver.run()
log.info('Exiting executor main')
app.main()
def proxy_main():
def main():
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_gc_executor, metric_writer, driver = initialize()
thermos_gc_executor.start()
metric_writer.start()
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def run():
def main(_, opts):
path_detector = FixedPathDetector(opts.root)
task_observer = TaskObserver(path_detector)
task_observer.start()
server = configure_server(task_observer)
thread = ExceptionalThread(target=lambda: server.run('0.0.0.0', opts.port, 'cherrypy'))
thread.daemon = True
thread.start()
sleep_forever()
app.main()
def proxy_main():
def main():
# Create executor stub
thermos_gc_executor = ThermosGCExecutor(checkpoint_root=TaskPath.DEFAULT_CHECKPOINT_ROOT)
thermos_gc_executor.start()
# Start metrics collection
metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
metric_writer.start()
# Create driver stub
driver = mesos.MesosExecutorDriver(thermos_gc_executor)
# Start GC executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def proxy_main():
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_executor = initialize(options)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def proxy_main():
def main():
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# Create executor stub
thermos_gc_executor = ThermosGCExecutor(FixedPathDetector(DEFAULT_CHECKPOINT_ROOT))
thermos_gc_executor.start()
# Start metrics collection
metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
metric_writer.start()
# Create driver stub
driver = MesosExecutorDriver(thermos_gc_executor)
# Start GC executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
app.main()
def proxy_main():
def main(args, options):
# 'sandbox' directory resides under the working directory assigned by the Mesos slave.
sandbox_root = os.path.join(os.path.realpath('.'), "sandbox")
unpack_assets(sandbox_root, MYSOS_MODULE, ASSET_RELPATH, execute=chmod_scripts)
log.info("Starting Vagrant Mysos executor within sandbox %s" % sandbox_root)
sandbox = Sandbox(sandbox_root)
executor = MysosExecutor(
MysosTaskRunnerProvider(
MySQLTaskControlProvider(),
NoopPackageInstallerProvider(), # Do not install any package.
NoopBackupStoreProvider()), # Do not recover any state.
sandbox)
driver = mesos.native.MesosExecutorDriver(executor)
driver.run()
log.info('Exiting executor main')
app.main()
def run():
def main(args, opts):
"""Main"""
zkconn = kazoo_client.TwitterKazooClient(opts.zk)
zkconn.start()
server = RedirServer(zkconn, opts.zk_basepath, opts.scheduler_url,
opts.subdomain, opts.base_domain)
thread = ExceptionalThread(
target=lambda: server.run(opts.listen,
opts.port,
server='cherrypy'))
thread.daemon = True
thread.start()
# Wait forever, basically.
thread.join()
app.add_option('--port', help='http port', default=8080)
app.add_option('--listen',
help='IP address to listen for http connections.',
default='0.0.0.0')
app.add_option('--zk',
help='Zookeeper ensemble (comma-delimited)',
default='zookeeper:2181')
app.add_option('--zk_basepath',
help='Zookeeper service path root.',
default='/aurora/svc')
app.add_option('--scheduler_url',
help='Aurora scheduler URL')
app.add_option('--base_domain',
help='Domain name of your site.')
app.add_option('--subdomain',
help='Subdomain that roots Aurora job namespace.',
default='aurora')
app.main()
def proxy_main(): app.main()
def proxy_main(): main = runner_proxy_main app.main()
def main(args):
options = app.get_options()
if options.show_help:
app.help()
if options.show_version or options.just_version:
print >> sys.stdout, 'Python NailGun client version 0.0.1'
if options.just_version:
sys.exit(0)
# Assume ng.pex has been aliased to the command name
command = re.compile('.pex$').sub('', os.path.basename(sys.argv[0]))
args_index = 0
# Otherwise the command name is the 1st arg
if command == 'ng':
if not args:
app.help()
command = args[0]
args_index = 1
ng = NailgunClient(host=options.ng_host, port=options.ng_port)
result = ng(command, *args[args_index:], **os.environ)
sys.exit(result)
app.main()
def proxy_main():
main = runner_proxy_main # noqa
app.main()
def proxy_main():
app.add_option(
'--port',
dest='api_port',
type='int',
default=None,
help='Port for the HTTP API server')
app.add_option(
'--mesos_master',
dest='mesos_master',
default=None,
help='Mesos master address. It can be a ZooKeeper URL through which the master can be '
'detected')
app.add_option(
'--framework_user',
dest='framework_user',
help='The Unix user that Mysos executor runs as')
app.add_option(
'--framework_role',
dest='framework_role',
default='*',
help="The role that Mysos framework runs as. If set, Mysos only uses Mesos pool resources "
"with that role. The default value '*' is what Mesos considers as the default role.\n"
"NOTE: Mesos master needs to be configured to allow the specified role. See its --roles "
"flag")
app.add_option(
'--executor_uri',
dest='executor_uri',
default=None,
help='URI for the Mysos executor package')
app.add_option(
'--executor_cmd',
dest='executor_cmd',
default=None,
help='Command to execute the executor package')
app.add_option(
'--executor_environ',
dest='executor_environ',
default=None,
help="Environment variables for the executors (and the tasks) as a list of dicts keyed by "
"{name, value} in JSON. Note that these variables don't affect Mesos slave components "
"such as the fetcher")
app.add_option(
'--zk_url',
dest='zk_url',
default=None,
help='ZooKeeper URL for various Mysos operations, in the form of '
'"zk://*****:*****@servers/path". The sub-directory <zk_url>/discover is used for '
'communicating MySQL cluster information between Mysos scheduler and executors')
# TODO(jyx): This could also be made a per-cluster configuration.
app.add_option(
'--election_timeout',
dest='election_timeout',
default='60s',
help='The amount of time the scheduler waits for all slaves to respond during a MySQL master '
'election, e.g., 60s. After the timeout the master is elected from only the slaves that '
'have responded')
app.add_option(
'--admin_keypath',
dest='admin_keypath',
default=None,
help='The path to the key file with MySQL admin credentials on Mesos slaves')
app.add_option(
'--work_dir',
dest='work_dir',
default=os.path.join(tempfile.gettempdir(), 'mysos'),
help="Directory path to place Mysos work directories, e.g., web assets, state files if "
"--state_storage=local. Default to a system temp directory.")
app.add_option(
'--state_storage',
dest='state_storage',
default='zk',
help="Mechanism to persist scheduler state. Available options are 'zk' and 'local'. If 'zk' "
"is chosen, the scheduler state is stored under <zk_url>/state; see --zk_url. Otherwise "
"'local' is chosen and the state is persisted under <work_dir>/state; see --work_dir")
app.add_option(
'--framework_failover_timeout',
dest='framework_failover_timeout',
default='14d',
help='Time after which Mysos framework is considered deleted. This implies losing all tasks. '
'SHOULD BE VERY HIGH')
# TODO(jyx): Flags like this are generally optional but specific executor implementations may
# require them. Consider adding validators that can be plugged in so configuration errors can be
# caught in the scheduler.
app.add_option(
'--installer_args',
dest='installer_args',
default=None,
help='Arguments for MySQL installer directly passed along to and parsed by the installer. '
'e.g., a serialized JSON string'
)
app.add_option(
'--backup_store_args',
dest='backup_store_args',
default=None,
help="Arguments for the store for MySQL backups. Its use and format are defined by the "
"backup store implementation. e.g., It can be a serialized JSON string"
)
app.add_option(
'--framework_authentication_file',
dest='framework_authentication_file',
default=None,
help="Path to the key file for authenticating the framework against Mesos master. Framework "
"will fail to register with Mesos if authentication is required by Mesos and this "
"option is not provided"
)
def main(args, options):
log.info("Options in use: %s", options)
if not options.api_port:
app.error('Must specify --port')
if not options.mesos_master:
app.error('Must specify --mesos_master')
if not options.framework_user:
app.error('Must specify --framework_user')
if not options.executor_uri:
app.error('Must specify --executor_uri')
if not options.executor_cmd:
app.error('Must specify --executor_cmd')
if not options.zk_url:
app.error('Must specify --zk_url')
if not options.admin_keypath:
app.error('Must specify --admin_keypath')
try:
election_timeout = parse_time(options.election_timeout)
framework_failover_timeout = parse_time(options.framework_failover_timeout)
except InvalidTime as e:
app.error(e.message)
try:
_, zk_servers, zk_root = zookeeper.parse(options.zk_url)
except Exception as e:
app.error("Invalid --zk_url: %s" % e.message)
web_assets_dir = os.path.join(options.work_dir, "web")
pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH)
log.info("Extracted web assets into %s" % options.work_dir)
fw_principal = None
fw_secret = None
if options.framework_authentication_file:
try:
with open(options.framework_authentication_file, "r") as f:
cred = yaml.load(f)
fw_principal = cred["principal"]
fw_secret = cred["secret"]
log.info("Loaded credential (principal=%s) for framework authentication" % fw_principal)
except IOError as e:
app.error("Unable to read the framework authentication key file: %s" % e)
except (KeyError, yaml.YAMLError) as e:
app.error("Invalid framework authentication key file format %s" % e)
log.info("Starting Mysos scheduler")
kazoo = KazooClient(zk_servers)
kazoo.start()
if options.state_storage == 'zk':
log.info("Using ZooKeeper (path: %s) for state storage" % zk_root)
state_provider = ZooKeeperStateProvider(kazoo, zk_root)
else:
log.info("Using local disk for state storage")
state_provider = LocalStateProvider(options.work_dir)
try:
state = state_provider.load_scheduler_state()
except StateProvider.Error as e:
app.error(e.message)
if state:
log.info("Successfully restored scheduler state")
framework_info = state.framework_info
if framework_info.HasField('id'):
log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value)
else:
log.info("No scheduler state to restore")
framework_info = FrameworkInfo(
user=options.framework_user,
name=FRAMEWORK_NAME,
checkpoint=True,
failover_timeout=framework_failover_timeout.as_(Time.SECONDS),
role=options.framework_role)
if fw_principal:
framework_info.principal = fw_principal
state = Scheduler(framework_info)
state_provider.dump_scheduler_state(state)
scheduler = MysosScheduler(
state,
state_provider,
options.framework_user,
options.executor_uri,
options.executor_cmd,
kazoo,
options.zk_url,
election_timeout,
options.admin_keypath,
installer_args=options.installer_args,
backup_store_args=options.backup_store_args,
executor_environ=options.executor_environ,
framework_role=options.framework_role)
if fw_principal and fw_secret:
cred = Credential(principal=fw_principal, secret=fw_secret)
scheduler_driver = mesos.native.MesosSchedulerDriver(
scheduler,
framework_info,
options.mesos_master,
cred)
else:
scheduler_driver = mesos.native.MesosSchedulerDriver(
scheduler,
framework_info,
options.mesos_master)
scheduler_driver.start()
server = HttpServer()
server.mount_routes(MysosServer(scheduler, web_assets_dir))
et = ExceptionalThread(
target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy'))
et.daemon = True
et.start()
try:
# Wait for the scheduler to stop.
# The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the
# process with SIGINT.
while not scheduler.stopped.wait(timeout=0.5):
pass
except KeyboardInterrupt:
log.info('Interrupted, exiting.')
else:
log.info('Scheduler exited.')
app.shutdown(1) # Mysos scheduler is supposed to be long-running thus the use of exit status 1.
app.main()
def proxy_main():
app.main()
def proxy_main(): main = runner_proxy_main # noqa app.main()
if opts.sampling < 0 or opts.sampling > 1:
sys.stdout.write("--sampling takes values within [0, 1]\n")
sys.exit(1)
stats = StatsServer(opts.iface,
opts.zookeeper_port,
opts.aggregation_depth,
opts.max_results,
opts.max_queued_requests,
opts.max_queued_replies,
opts.max_queued_events,
sampling=opts.sampling,
include_bytes=not opts.exclude_bytes)
log.info("Starting with opts: %s" % (opts))
signal.signal(signal.SIGINT, signal.SIG_DFL)
server = Server()
server.mount_routes(DiagnosticsEndpoints())
server.mount_routes(stats)
server.run(opts.http_addr, opts.http_port)
stats.sniffer.join()
if __name__ == '__main__':
setup()
app.main()
def proxy_main():
@app.command
@app.command_option(
'--num_nodes',
dest='num_nodes',
type='int',
help='Number of nodes this cluster should have')
@app.command_option(
'--backup_id',
dest='backup_id',
default=None,
help="The 'backup_id' to restore from")
@app.command_option(
'--cluster_user',
dest='cluster_user',
help='MySQL user name the of cluster')
def create(args, options):
validate_common_options(options)
if not options.num_nodes:
app.error("--num_nodes is required")
if not options.cluster_user:
app.error("--cluster_user is required")
url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name)
values = dict(
num_nodes=int(options.num_nodes),
cluster_user=options.cluster_user,
backup_id=options.backup_id if options.backup_id else '')
req = urllib2.Request(url, urllib.urlencode(values))
try:
response = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
log.error("POST request failed: %s, %s, %s" % (
e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read()))
app.quit(1)
try:
result = json.loads(response)
if not isinstance(result, dict):
raise ValueError()
except ValueError:
log.error("Invalid response: %s" % response)
app.quit(1)
log.info("Cluster created. Cluster info: %s" % str(result))
with open(options.password_file, 'w') as f:
f.write(result["cluster_password"])
log.info("Waiting for the master for this cluster to be elected...")
master_endpoint = wait_for_master(result['cluster_url']).service_endpoint
connection_str = "mysql://%s:%s@%s:%d/" % (
options.cluster_user,
result["cluster_password"],
master_endpoint.host,
master_endpoint.port)
log.info("Connecting to the MySQL cluster master: %s" % connection_str)
engine = create_engine(connection_str)
for i in range(5): # Loop for 5 times/seconds to wait for the master to be promoted.
try:
# TODO(jyx): Test writing to the master and reading from the slave.
result = engine.execute("SELECT 1;").scalar()
assert 1 == int(result), "Expecting result to be 1 but got %s" % result
break
except OperationalError:
if i == 4:
raise
log.debug("MySQL master not ready yet. Sleep for 1 second...")
time.sleep(1)
log.info("Cluster successfully started")
@app.command
def delete(args, options):
validate_common_options(options)
with open(options.password_file, 'r') as f:
password = f.read().strip()
if not password:
app.error("Empty password file")
url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name)
values = dict(password=password)
req = urllib2.Request(url, urllib.urlencode(values))
req.get_method = lambda: 'DELETE'
try:
response = urllib2.urlopen(req).read()
except urllib2.HTTPError as e:
log.error("DELETE request failed: %s, %s, %s" % (
e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read()))
app.quit(1)
try:
result = json.loads(response)
if not isinstance(result, dict):
raise ValueError()
except ValueError:
log.error("Invalid response: %s" % response)
app.quit(1)
log.info("Cluster deletion result: %s" % result)
log.info("Waiting for the cluster to terminate...")
wait_for_termination(result['cluster_url'])
log.info("Cluster terminated/deleted")
def validate_common_options(options):
if not options.api_host:
app.error("--api_host is required")
if not options.api_port:
app.error("--api_port is required")
if not options.cluster_name:
app.error("--cluster is required")
if not options.password_file:
app.error("--password_file is required")
log.info("Using --password_file=%s" % options.password_file)
safe_mkdir(os.path.dirname(options.password_file))
def main(args, options):
app.help()
app.main()
