def add_verbosity_options(): from twitter.common import app from twitter.common.log.options import LogOptions def set_quiet(option, _1, _2, parser): setattr(parser.values, option.dest, 'quiet') LogOptions.set_stderr_log_level('NONE') def set_verbose(option, _1, _2, parser): setattr(parser.values, option.dest, 'verbose') LogOptions.set_stderr_log_level('DEBUG') app.add_option('-v', dest='verbosity', default='normal', action='callback', callback=set_verbose, help='Verbose logging. (default: %default)') app.add_option('-q', dest='verbosity', default='normal', action='callback', callback=set_quiet, help='Quiet logging. (default: %default)')
def add_verbosity_options(): from twitter.common import app from twitter.common.log.options import LogOptions def set_quiet(option, _1, _2, parser): setattr(parser.values, option.dest, 'quiet') LogOptions.set_stderr_log_level('NONE') def set_verbose(option, _1, _2, parser): setattr(parser.values, option.dest, 'verbose') LogOptions.set_stderr_log_level('DEBUG') app.add_option('-v', dest='verbosity', default='normal', action='callback', callback=set_verbose, help='Verbose logging. (default: %default)') app.add_option('-q', dest='verbosity', default='normal', action='callback', callback=set_quiet, help='Quiet logging. (default: %default)')
def add_verbosity_options(): from twitter.common import app from twitter.common.log.options import LogOptions def set_quiet(option, _1, _2, parser): setattr(parser.values, option.dest, "quiet") LogOptions.set_stderr_log_level("NONE") def set_verbose(option, _1, _2, parser): setattr(parser.values, option.dest, "verbose") LogOptions.set_stderr_log_level("DEBUG") app.add_option( "-v", dest="verbosity", default="normal", action="callback", callback=set_verbose, help="Verbose logging. (default: %default)", ) app.add_option( "-q", dest="verbosity", default="normal", action="callback", callback=set_quiet, help="Quiet logging. (default: %default)", )
def run(): def main(args, opts): """Main""" server = RedirServer(opts.zk_basepath, opts.subdomain, opts.base_domain) thread = ExceptionalThread( target=lambda: server.run(opts.listen, opts.port, server='cherrypy')) thread.daemon = True thread.start() wait_forever() log.LogOptions.set_stderr_log_level('google:INFO') app.add_option('--port', help='http port', default=8080) app.add_option('--listen', help='IP address to listen for http connections.', default='0.0.0.0') app.add_option('--zk_basepath', help='Zookeeper service path root.', default='/aurora') app.add_option('--base_domain', help='Domain name of your site.', default='example.com') app.add_option('--subdomain', help='Subdomain that roots Aurora job namespace.', default='aurora') app.main()
def test_app_add_options_with_Option(self): # options.Option opt = options.Option('--option1', dest='option1') app.add_option(opt) app.init(force_args=['--option1', 'option1value', 'extraargs']) assert app.get_options().option1 == 'option1value' assert app.argv() == ['extraargs']
def test_app_add_options_with_Option(self): # options.Option opt = options.Option('--option1', dest='option1') app.add_option(opt) app.init(force_args=['--option1', 'option1value', 'extraargs']) assert app.get_options().option1 == 'option1value' assert app.argv() == ['extraargs']
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str) app.add_option('--port', default=3888, type=int) app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--version', default=False, action='store_true')
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str) app.add_option('--port', default=3888, type=int) app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--version', default=False, action='store_true')
CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) _LOGGER_TYPES = ', '.join(LoggerMode.VALUES) app.add_option( '--announcer-enable', dest='announcer_enable', action='store_true', default=False, help='Enable the ServerSet announcer for this executor. Jobs must still activate using ' 'the Announcer configuration.') app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path',
import atexit import errno import socket import subprocess import time from twitter.common.quantity import Amount, Time try: from twitter.common import app HAS_APP=True app.add_option( '--tunnel_host', type='string', dest='tunnel_host', default='nest1.corp.twitter.com', help='Host to tunnel commands through (default: %default)') except ImportError: HAS_APP=False __all__ = ( 'TunnelHelper', ) def safe_kill(po): """ Given a Popen object, safely kill it without an unexpected exception.
import os import sys import pprint import time from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState from twitter.common import app from twitter.common.recordio import RecordIO, ThriftRecordReader from twitter.thermos.common.ckpt import CheckpointDispatcher app.add_option("--checkpoint", dest="ckpt", metavar="CKPT", help="read checkpoint from CKPT") app.add_option("--assemble", dest="assemble", metavar="CKPT", default=True, help="read checkpoint from CKPT") def main(args): values = app.get_options() if len(args) > 0: print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % ( " ".join(args)) app.help() sys.exit(1)
# ================================================================================================== import socket import threading import zookeeper from functools import wraps from twitter.common import app from twitter.common import log from twitter.common.log.options import LogOptions from twitter.common.quantity import Amount, Time app.add_option( '--zookeeper', default='zookeeper.local.twitter.com:2181', metavar='HOST:PORT[,HOST:PORT,...]', help='Comma-separated list of host:port of ZooKeeper servers', ) app.add_option( '--zookeeper_timeout', type='float', default=5.0, help='default timeout (in seconds) for ZK operations', ) app.add_option( '--enable_zookeeper_debug_logging', dest='twitter_common_zookeeper_debug', default=False, action='store_true', help='whether to enable ZK debug logging to stderr', )
import time from twitter.common import app from twitter.common.exceptions import ExceptionalThread from twitter.common.log.options import LogOptions from twitter.common.quantity import Amount, Time from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.thermos.observer.http.configure import configure_server from apache.thermos.observer.task_observer import TaskObserver app.add_option( '--mesos-root', dest='mesos_root', type='string', default=MesosPathDetector.DEFAULT_MESOS_ROOT, help= 'The mesos root directory to search for Thermos executor sandboxes [default: %default]' ) app.add_option('--port', dest='port', type='int', default=1338, help='The port on which the observer should listen.') app.add_option('--polling_interval_secs', dest='polling_interval_secs', type='int', default=int(TaskObserver.POLLING_INTERVAL.as_(Time.SECONDS)), help='The number of seconds between observer refresh attempts.')
from __future__ import print_function import pprint import sys import time from twitter.common import app from twitter.common.recordio import RecordIO, ThriftRecordReader from apache.thermos.common.ckpt import CheckpointDispatcher from gen.apache.thermos.ttypes import RunnerCkpt, RunnerState, TaskState app.add_option( "--checkpoint", dest="ckpt", metavar="CKPT", help="read checkpoint from CKPT") app.add_option( "--assemble", dest="assemble", metavar="CKPT", default=True, help="whether or not to replay the checkpoint records.") def main(args): values = app.get_options() if len(args) > 0:
from apache.aurora.executor.aurora_executor import AuroraExecutor from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider from apache.aurora.executor.common.executor_timeout import ExecutorTimeout from apache.aurora.executor.common.health_checker import HealthCheckerProvider from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir('.') app.add_option( '--announcer-enable', dest='announcer_enable', action='store_true', default=False, help= 'Enable the ServerSet announcer for this executor. Jobs must still activate using ' 'the Announcer configuration.') app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path', dest='announcer_serverset_path', type=str,
from jenkinsapi.exceptions import UnknownJob import os import sys from twitter.common import app from twitter.common import log from twitter.common import dirutil from twitter.common.log.options import LogOptions app.set_usage("jenkins --[server|outdir|config] [--job=[jobname]] [create|delete|show|enable|disable]") app.add_option( "--server", default="ci.makewhat.is:8080", dest="server", help="jenkins server" ) app.add_option( "--job", default=None, dest="job", help="job name" ) app.add_option( "--config", default=None, dest="config", help="config path" ) app.add_option(
import time from twitter.common import app from twitter.common.exceptions import ExceptionalThread from twitter.common.log.options import LogOptions from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT from apache.thermos.monitoring.detector import ChainedPathDetector, FixedPathDetector from apache.thermos.observer.http.configure import configure_server from apache.thermos.observer.task_observer import TaskObserver app.add_option( '--root', dest='root', type='string', default=DEFAULT_CHECKPOINT_ROOT, help='The thermos checkpoint root directory to search for Thermos tasks [default: %default]') app.add_option( '--mesos-root', dest='mesos_root', type='string', default=MesosPathDetector.DEFAULT_MESOS_ROOT, help='The mesos root directory to search for Thermos executor sandboxes [default: %default]') app.add_option( '--port', dest='port',
def setup(): app.add_option('--iface', default='eth0', type=str) app.add_option('--port', default=2889, type=int) app.add_option('-c', '--colors', default=False, action='store_true')
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option('--iface', default='eth0', type=str, help='The interface to sniff on') app.add_option('--port', default=2889, type=int, help='The ZAB port used by the leader') app.add_option('-c', '--colors', default=False, action='store_true', help='Color each learner/leader stream differently') app.add_option('--dump-bad-packet', default=False, action='store_true', help='Dump packets that cannot be deserialized') app.add_option('--include-pings', default=False, action='store_true', help='Whether to include pings send from learners to the leader') app.add_option('--version', default=False, action='store_true')
def setup(): app.add_option("--iface", dest="iface", metavar="IFACE", default="eth0", help="interface to capture packets from") app.add_option("--http-port", dest="http_port", metavar="HTTPPORT", type=int, default=7070, help="listen port for http endpoints") app.add_option("--http-address", dest="http_addr", metavar="HTTPADDR", type=str, default=socket.gethostname(), help="listen address for http endpoints") app.add_option("--zookeeper-port", type=int, default=2181, help="ZK's client port (from which to sniff)") app.add_option("--aggregation-depth", dest="aggregation_depth", type=int, default=0, help="aggregate paths up to a certain depth") app.add_option("--max-results", dest="max_results", type=int, default=10, help="top number of results to be exported") app.add_option("--refresh-time", dest="refresh_time", type=int, default=0, help="refresh time in the generated html") app.add_option("--niceness", dest="niceness", type=int, default=0, help="set the niceness") app.add_option( "--set-cpu-affinity", dest="cpu_affinity", metavar="CPU#[,CPU#]", type=str, default=None, help="A comma-separated list of CPU cores to pin this process to") app.add_option("--max-queued-requests", type=int, default=400000, help="max queued requests") app.add_option("--max-queued-replies", type=int, default=400000, help="max queued replies") app.add_option("--max-queued-events", type=int, default=400000, help="max queued events")
"""A Mesos-customized entry point to the thermos_observer webserver.""" import time from twitter.common import app from twitter.common.exceptions import ExceptionalThread from twitter.common.log.options import LogOptions from twitter.common.quantity import Amount, Time from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.thermos.observer.http.configure import configure_server from apache.thermos.observer.task_observer import TaskObserver app.add_option( '--mesos-root', dest='mesos_root', type='string', default=MesosPathDetector.DEFAULT_MESOS_ROOT, help='The mesos root directory to search for Thermos executor sandboxes [default: %default]') app.add_option( '--ip', dest='ip', type='string', default='0.0.0.0', help='The IP address the observer will bind to.') app.add_option( '--port', dest='port',
print(traceback.format_exc(), file=sys.stderr) MesosExecutorDriver = None CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) _LOGGER_TYPES = ', '.join(LoggerMode.VALUES) app.add_option( '--announcer-enable', dest='announcer_enable', action='store_true', default=False, help= 'Enable the ServerSet announcer for this executor. Jobs must still activate using ' 'the Announcer configuration.') app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path', dest='announcer_serverset_path', type=str,
from apache.thermos.common.ckpt import CheckpointDispatcher from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT from apache.thermos.common.options import add_binding_to, add_port_to from apache.thermos.common.path import TaskPath from apache.thermos.config.loader import ThermosConfigLoader, ThermosTaskWrapper from apache.thermos.config.schema import Process, Resources, Task from apache.thermos.core.helper import TaskRunnerHelper from apache.thermos.core.runner import TaskRunner from apache.thermos.monitoring.detector import TaskDetector from apache.thermos.monitoring.garbage import DefaultCollector, TaskGarbageCollector from apache.thermos.monitoring.monitor import TaskMonitor from gen.apache.thermos.ttypes import ProcessState, RunnerCkpt, RunnerState, TaskState app.add_option("--root", dest="root", metavar="PATH", default=DEFAULT_CHECKPOINT_ROOT, help="the thermos config root") def set_keep(option, opt_str, value, parser): setattr(parser.values, option.dest, opt_str.startswith('--keep')) def get_task_from_options(args, opts, **kw): loader = ThermosConfigLoader.load_json if opts.json else ThermosConfigLoader.load if len(args) != 1: app.error('Should specify precisely one config, instead got: %s' % args) tasks = loader(args[0], bindings=opts.bindings, **kw)
def main(args, options): from pkg_resources import WorkingSet, Requirement, find_distributions if not options.site_dir: app.error('Must supply --site') distributions = list(find_distributions(options.site_dir)) working_set = WorkingSet() for dist in distributions: working_set.add(dist) for arg in args: arg_req = Requirement.parse(arg) found_dist = working_set.find(arg_req) if not found_dist: print('Could not find %s!' % arg_req) out_zip = Distiller(found_dist).distill() print('Dumped %s => %s' % (arg_req, out_zip)) if HAS_APP: if __name__ == '__main__': app.add_option('--site', dest='site_dir', metavar='DIR', default=None, help='Directory to search for the requirement.') app.main()
app.register_commands_from(admin, help_commands, maintenance) add_verbosity_options() def main(): app.help() try: from apache.aurora.kerberos.auth_module import KerberosAuthModule register_auth_module(KerberosAuthModule()) except ImportError: # Use default auth implementation if kerberos is not available. pass LogOptions.set_stderr_log_level('INFO') LogOptions.disable_disk_logging() app.set_name('aurora-admin') app.set_usage(generate_terse_usage()) app.add_option( '--bypass-leader-redirect', action='store_true', default=False, dest='bypass_leader_redirect', help='Bypass the scheduler\'s leader redirect filter') def proxy_main(): app.main()
from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.thermos.cli.common import register_path_detector from apache.thermos.cli.main import register_commands, register_options register_commands(app) register_options(app) def register_mesos_root(_, __, value, ___): register_path_detector(MesosPathDetector(root=value)) app.add_option( '--mesos-root', dest='mesos_root', type='string', action='callback', default=MesosPathDetector.DEFAULT_MESOS_ROOT, callback=register_mesos_root, help= 'The mesos root directory to search for Thermos executor sandboxes [default: %default]' ) # register a default mesos root, since the callback will not be called if no --mesos-root specified. register_path_detector(MesosPathDetector()) LogOptions.disable_disk_logging() LogOptions.set_stderr_log_level('google:INFO') app.main()
except ImportError: print(traceback.format_exc(), file=sys.stderr) MesosExecutorDriver = None CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path', dest='announcer_serverset_path', type=str, default='/aurora', help='The root of the tree into which ServerSets should be announced. The paths will ' 'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.') app.add_option( '--announcer-allow-custom-serverset-path', dest='announcer_allow_custom_serverset_path',
return filename def main(args, options): from pkg_resources import WorkingSet, Requirement, find_distributions if not options.site_dir: app.error('Must supply --site') distributions = list(find_distributions(options.site_dir)) working_set = WorkingSet() for dist in distributions: working_set.add(dist) for arg in args: arg_req = Requirement.parse(arg) found_dist = working_set.find(arg_req) if not found_dist: print('Could not find %s!' % arg_req) out_zip = Distiller(found_dist).distill() print('Dumped %s => %s' % (arg_req, out_zip)) if HAS_APP: if __name__ == '__main__': app.add_option('--site', dest='site_dir', metavar='DIR', default=None, help='Directory to search for the requirement.') app.main()
CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) _LOGGER_DESTINATIONS = ', '.join(LoggerDestination.VALUES) _LOGGER_MODES = ', '.join(LoggerMode.VALUES) app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path', dest='announcer_serverset_path', type=str, default='/aurora', help='The root of the tree into which ServerSets should be announced. The paths will ' 'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.') app.add_option( '--announcer-allow-custom-serverset-path', dest='announcer_allow_custom_serverset_path',
import twitter.common.app as app import twitter.common.log as log import gevent from gevent.queue import Queue from flask import Flask, flash, redirect, render_template, request, Response, url_for from datetime import timedelta import json import os import platform app.set_name = "gatekeeper" app.add_option("-p", "--port", type="int", default=5000) config = HelperFunctions().read_config_from_yaml() PLATFORM = platform.system() GOOGLE_ADMIN_ACTIONS = list( k for k, v in config["actions"]["google_admin"].items() if v is True) GOOGLE_GMAIL_ACTIONS = list( k for k, v in config["actions"]["google_gmail"].items() if v is True) GOOGLE_CALENDAR_ACTIONS = list( k for k, v in config["actions"]["google_calendar"].items() if v is True) GOOGLE_DRIVE_ACTIONS = list( k for k, v in config["actions"]["google_drive"].items() if v is True) PAGERDUTY_ACTIONS = list(k for k, v in config["actions"]["pagerduty"].items() if v is True) DUO_ACTIONS = list(k for k, v in config["actions"]["duo"].items() if v is True)
from __future__ import print_function import sys from twitter.common import app from twitter.common.exceptions import ExceptionalThread from twitter.common.http import HttpServer from twitter.common.http.diagnostics import DiagnosticsEndpoints from apache.thermos.common.path import TaskPath from apache.thermos.observer.http.http_observer import BottleObserver from apache.thermos.observer.task_observer import TaskObserver app.add_option("--root", dest="root", metavar="DIR", default=TaskPath.DEFAULT_CHECKPOINT_ROOT, help="root checkpoint directory for thermos task runners") app.add_option("--port", dest="port", metavar="INT", default=1338, help="port number to listen on.") def proxy_main(): def main(args, opts): if args: print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
def setup(): app.add_option('--iface', default='eth0', type=str) app.add_option('--port', default=3888, type=int) app.add_option('-c', '--colors', default=False, action='store_true')
try: from mesos.native import MesosExecutorDriver except ImportError: print(traceback.format_exc(), file=sys.stderr) MesosExecutorDriver = None CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path', dest='announcer_serverset_path', type=str, default='/aurora', help= 'The root of the tree into which ServerSets should be announced. The paths will ' 'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.') app.add_option( '--announcer-allow-custom-serverset-path', dest='announcer_allow_custom_serverset_path',
from collections import defaultdict from rainman.codec import BEncoder from twitter.common import app, log from twitter.common.http import HttpServer from twitter.common.quantity import Amount, Time app.add_option( '--port', dest='port', default=8080, type=int, help='The port on which the torrent tracker should be run.') class TrackerRequest(object): class Error(Exception): pass class MalformedRequestError(Error): pass REQUIRED_KEYS = frozenset([ 'info_hash', 'peer_id', 'port', 'uploaded', 'downloaded', 'left', ]) OPTIONAL_KEYS = set(['ip', 'event'])
import re import sys import os.path from functools import partial from twitter.common import app, log ROOT = os.path.join(os.path.dirname(__file__), '..') sys.path.append(ROOT) from lib.modules import spout, config, models, event_emitter_2 from lib.modules import sink app.add_option("--listener", default="CarbonAsyncTcpSpout", help="Select the incoming metric connection interface") app.add_option("--writer", default="RedisSink", help="Select the sink connection interface") app.add_option("--config", help="Collector Config") # Globals EE = event_emitter_2.EventEmitter2() CONFIG = None WHITELIST = None BLACKLIST = None def setup(options): """Load the config file and add listeners for whitelisted metrics. :param options: app options """
def proxy_main(): app.add_option('--api_host', dest='api_host', help='Host for the HTTP API server') app.add_option('--api_port', dest='api_port', type='int', help='Port for the HTTP API server') app.add_option('--cluster', dest='cluster_name', help='Name of the MySQL cluster to create') app.add_option( '--password_file', dest='password_file', default=os.path.join(tempfile.gettempdir(), 'mysos', 'mysos_test_client', 'password_file'), help= "Path to the file for persisting the cluster password for testing purposes" ) @app.command @app.command_option('--num_nodes', dest='num_nodes', type='int', help='Number of nodes this cluster should have') @app.command_option('--backup_id', dest='backup_id', default=None, help="The 'backup_id' to restore from") @app.command_option('--cluster_user', dest='cluster_user', help='MySQL user name the of cluster') @app.command_option( '--size', dest='size', help= "The size of instances in the cluster as a JSON dictionary of 'cpus', 'mem', 'disk'. " "'mem' and 'disk' are specified with data size units: kb, mb, gb, etc. If given 'None'" "then app defaults are used.") @app.command_option( '--cluster_password', dest='cluster_password', help= "The password used for accessing MySQL instances in the cluster as well as deleting " "the cluster from Mysos.") def create(args, options): validate_common_options(options) if not options.num_nodes: app.error("--num_nodes is required") if not options.cluster_user: app.error("--cluster_user is required") url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name) values = dict( num_nodes=int(options.num_nodes), cluster_user=options.cluster_user, size=options.size if options.size else '', # 'urlencode()' doesn't accept None. backup_id=options.backup_id if options.backup_id else '', cluster_password=options.cluster_password if options.cluster_password else '') req = urllib2.Request(url, urllib.urlencode(values)) try: response = urllib2.urlopen(req).read() except urllib2.HTTPError as e: log.error("POST request failed: %s, %s, %s" % (e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read())) app.quit(1) try: result = json.loads(response) if not isinstance(result, dict): raise ValueError() except ValueError: log.error("Invalid response: %s" % response) app.quit(1) log.info("Cluster created. Cluster info: %s" % str(result)) with open(options.password_file, 'w') as f: f.write(result["cluster_password"]) log.info("Waiting for the master for this cluster to be elected...") master_endpoint = wait_for_master( result['cluster_url']).service_endpoint connection_str = "mysql://%s:%s@%s:%d/" % ( options.cluster_user, result["cluster_password"], master_endpoint.host, master_endpoint.port) log.info("Connecting to the MySQL cluster master: %s" % connection_str) engine = create_engine(connection_str) for i in range( 5 ): # Loop for 5 times/seconds to wait for the master to be promoted. try: # TODO(jyx): Test writing to the master and reading from the slave. result = engine.execute("SELECT 1;").scalar() assert 1 == int( result), "Expecting result to be 1 but got %s" % result break except OperationalError: if i == 4: raise log.debug("MySQL master not ready yet. Sleep for 1 second...") time.sleep(1) log.info("Cluster successfully started") @app.command def delete(args, options): validate_common_options(options) with open(options.password_file, 'r') as f: password = f.read().strip() if not password: app.error("Empty password file") url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port, options.cluster_name) values = dict(password=password) req = urllib2.Request(url, urllib.urlencode(values)) req.get_method = lambda: 'DELETE' try: response = urllib2.urlopen(req).read() except urllib2.HTTPError as e: log.error("DELETE request failed: %s, %s, %s" % (e.code, BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code], e.read())) app.quit(1) try: result = json.loads(response) if not isinstance(result, dict): raise ValueError() except ValueError: log.error("Invalid response: %s" % response) app.quit(1) log.info("Cluster deletion result: %s" % result) log.info("Waiting for the cluster to terminate...") wait_for_termination(result['cluster_url']) log.info("Cluster terminated/deleted") def validate_common_options(options): if not options.api_host: app.error("--api_host is required") if not options.api_port: app.error("--api_port is required") if not options.cluster_name: app.error("--cluster is required") if not options.password_file: app.error("--password_file is required") log.info("Using --password_file=%s" % options.password_file) safe_mkdir(os.path.dirname(options.password_file)) def main(args, options): app.help() app.main()
def setup(): LogOptions.set_stderr_log_level('NONE') app.add_option( '--packet-filter', default='tcp', type=str, help= 'pcap filter string. e.g. "tcp portrange 11221-32767" for JUnit tests') app.add_option('-c', '--colors', default=False, action='store_true') app.add_option('--dump-bad-packet', default=False, action='store_true') app.add_option('--include-pings', default=False, action='store_true', help='Whether to include ZAB/ZK pings') app.add_option('--offline', default=None, type=str, help='offline mode with a pcap file') app.add_option('--version', default=False, action='store_true')
import sys import traceback from twitter.common import app, log from apache.thermos.common.options import add_port_to from apache.thermos.common.planner import TaskPlanner from apache.thermos.common.statuses import (INTERNAL_ERROR, INVALID_TASK, TERMINAL_TASK, UNKNOWN_ERROR, UNKNOWN_USER) from apache.thermos.config.loader import ThermosConfigLoader from apache.thermos.core.process import Process from apache.thermos.core.runner import TaskRunner app.add_option("--thermos_json", dest="thermos_json", default=None, help="read a thermos Task from a serialized json blob") app.add_option("--sandbox", dest="sandbox", metavar="PATH", default=None, help="the sandbox in which this task should run") app.add_option("--checkpoint_root", dest="checkpoint_root", metavar="PATH", default=None, help="the path where we will store checkpoints") app.add_option(
from apache.aurora.executor.aurora_executor import AuroraExecutor from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider from apache.aurora.executor.common.executor_timeout import ExecutorTimeout from apache.aurora.executor.common.health_checker import HealthCheckerProvider from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir('.') app.add_option( '--announcer-enable', dest='announcer_enable', action='store_true', default=False, help='Enable the ServerSet announcer for this executor. Jobs must still activate using ' 'the Announcer configuration.') app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path',
LogOptions.set_stdout_log_level(value) except LogOptionsException, e: raise optparse.OptionValueError('Failed to parse option: %s' % e) parser.values.twitter_common_log_stdout_log_level = value _LOGGING_HELP = \ """The level at which to log to %%s [default: %%%%default]. Takes either LEVEL or scheme:LEVEL, where LEVEL is one of %s and scheme is one of %s. """ % (repr(LogOptions._LOG_LEVELS.keys()), repr(LogOptions._LOG_SCHEMES)) if HAVE_APP: app.add_option('--log_to_stdout', callback=LogOptions._stdout_options_callback, default=_DEFAULT_LOG_OPTS.twitter_common_log_stdout_log_level, type='string', action='callback', metavar='[scheme:]LEVEL', dest='twitter_common_log_stdout_log_level', help=_LOGGING_HELP % 'stdout') app.add_option('--log_to_disk', callback=LogOptions._disk_options_callback, default=_DEFAULT_LOG_OPTS.twitter_common_log_disk_log_level, type='string', action='callback', metavar='[scheme:]LEVEL', dest='twitter_common_log_disk_log_level', help=_LOGGING_HELP % 'disk') app.add_option('--log_dir', type='string',
def proxy_main(): app.add_option( '--port', dest='api_port', type='int', default=None, help='Port for the HTTP API server') app.add_option( '--mesos_master', dest='mesos_master', default=None, help='Mesos master address. It can be a ZooKeeper URL through which the master can be ' 'detected') app.add_option( '--framework_user', dest='framework_user', help='The Unix user that Mysos executor runs as') app.add_option( '--framework_role', dest='framework_role', default='*', help="The role that Mysos framework runs as. If set, Mysos only uses Mesos pool resources " "with that role. The default value '*' is what Mesos considers as the default role.\n" "NOTE: Mesos master needs to be configured to allow the specified role. See its --roles " "flag") app.add_option( '--executor_uri', dest='executor_uri', default=None, help='URI for the Mysos executor package') app.add_option( '--executor_cmd', dest='executor_cmd', default=None, help='Command to execute the executor package') app.add_option( '--executor_environ', dest='executor_environ', default=None, help="Environment variables for the executors (and the tasks) as a list of dicts keyed by " "{name, value} in JSON. Note that these variables don't affect Mesos slave components " "such as the fetcher") app.add_option( '--zk_url', dest='zk_url', default=None, help='ZooKeeper URL for various Mysos operations, in the form of ' '"zk://*****:*****@servers/path". The sub-directory <zk_url>/discover is used for ' 'communicating MySQL cluster information between Mysos scheduler and executors') # TODO(jyx): This could also be made a per-cluster configuration. app.add_option( '--election_timeout', dest='election_timeout', default='60s', help='The amount of time the scheduler waits for all slaves to respond during a MySQL master ' 'election, e.g., 60s. After the timeout the master is elected from only the slaves that ' 'have responded') app.add_option( '--admin_keypath', dest='admin_keypath', default=None, help='The path to the key file with MySQL admin credentials on Mesos slaves') app.add_option( '--work_dir', dest='work_dir', default=os.path.join(tempfile.gettempdir(), 'mysos'), help="Directory path to place Mysos work directories, e.g., web assets, state files if " "--state_storage=local. Default to a system temp directory.") app.add_option( '--state_storage', dest='state_storage', default='zk', help="Mechanism to persist scheduler state. Available options are 'zk' and 'local'. If 'zk' " "is chosen, the scheduler state is stored under <zk_url>/state; see --zk_url. Otherwise " "'local' is chosen and the state is persisted under <work_dir>/state; see --work_dir") app.add_option( '--framework_failover_timeout', dest='framework_failover_timeout', default='14d', help='Time after which Mysos framework is considered deleted. This implies losing all tasks. ' 'SHOULD BE VERY HIGH') # TODO(jyx): Flags like this are generally optional but specific executor implementations may # require them. Consider adding validators that can be plugged in so configuration errors can be # caught in the scheduler. app.add_option( '--installer_args', dest='installer_args', default=None, help='Arguments for MySQL installer directly passed along to and parsed by the installer. ' 'e.g., a serialized JSON string' ) app.add_option( '--backup_store_args', dest='backup_store_args', default=None, help="Arguments for the store for MySQL backups. Its use and format are defined by the " "backup store implementation. e.g., It can be a serialized JSON string" ) app.add_option( '--framework_authentication_file', dest='framework_authentication_file', default=None, help="Path to the key file for authenticating the framework against Mesos master. Framework " "will fail to register with Mesos if authentication is required by Mesos and this " "option is not provided" ) def main(args, options): log.info("Options in use: %s", options) if not options.api_port: app.error('Must specify --port') if not options.mesos_master: app.error('Must specify --mesos_master') if not options.framework_user: app.error('Must specify --framework_user') if not options.executor_uri: app.error('Must specify --executor_uri') if not options.executor_cmd: app.error('Must specify --executor_cmd') if not options.zk_url: app.error('Must specify --zk_url') if not options.admin_keypath: app.error('Must specify --admin_keypath') try: election_timeout = parse_time(options.election_timeout) framework_failover_timeout = parse_time(options.framework_failover_timeout) except InvalidTime as e: app.error(e.message) try: _, zk_servers, zk_root = zookeeper.parse(options.zk_url) except Exception as e: app.error("Invalid --zk_url: %s" % e.message) web_assets_dir = os.path.join(options.work_dir, "web") pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH) log.info("Extracted web assets into %s" % options.work_dir) fw_principal = None fw_secret = None if options.framework_authentication_file: try: with open(options.framework_authentication_file, "r") as f: cred = yaml.load(f) fw_principal = cred["principal"] fw_secret = cred["secret"] log.info("Loaded credential (principal=%s) for framework authentication" % fw_principal) except IOError as e: app.error("Unable to read the framework authentication key file: %s" % e) except (KeyError, yaml.YAMLError) as e: app.error("Invalid framework authentication key file format %s" % e) log.info("Starting Mysos scheduler") kazoo = KazooClient(zk_servers) kazoo.start() if options.state_storage == 'zk': log.info("Using ZooKeeper (path: %s) for state storage" % zk_root) state_provider = ZooKeeperStateProvider(kazoo, zk_root) else: log.info("Using local disk for state storage") state_provider = LocalStateProvider(options.work_dir) try: state = state_provider.load_scheduler_state() except StateProvider.Error as e: app.error(e.message) if state: log.info("Successfully restored scheduler state") framework_info = state.framework_info if framework_info.HasField('id'): log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value) else: log.info("No scheduler state to restore") framework_info = FrameworkInfo( user=options.framework_user, name=FRAMEWORK_NAME, checkpoint=True, failover_timeout=framework_failover_timeout.as_(Time.SECONDS), role=options.framework_role) if fw_principal: framework_info.principal = fw_principal state = Scheduler(framework_info) state_provider.dump_scheduler_state(state) scheduler = MysosScheduler( state, state_provider, options.framework_user, options.executor_uri, options.executor_cmd, kazoo, options.zk_url, election_timeout, options.admin_keypath, installer_args=options.installer_args, backup_store_args=options.backup_store_args, executor_environ=options.executor_environ, framework_role=options.framework_role) if fw_principal and fw_secret: cred = Credential(principal=fw_principal, secret=fw_secret) scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, options.mesos_master, cred) else: scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, options.mesos_master) scheduler_driver.start() server = HttpServer() server.mount_routes(MysosServer(scheduler, web_assets_dir)) et = ExceptionalThread( target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy')) et.daemon = True et.start() try: # Wait for the scheduler to stop. # The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the # process with SIGINT. while not scheduler.stopped.wait(timeout=0.5): pass except KeyboardInterrupt: log.info('Interrupted, exiting.') else: log.info('Scheduler exited.') app.shutdown(1) # Mysos scheduler is supposed to be long-running thus the use of exit status 1. app.main()
import os import sys import pprint import time from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState from twitter.common import app from twitter.common.recordio import RecordIO, ThriftRecordReader from twitter.thermos.common.ckpt import CheckpointDispatcher app.add_option("--checkpoint", dest = "ckpt", metavar = "CKPT", help = "read checkpoint from CKPT") app.add_option("--assemble", dest = "assemble", metavar = "CKPT", default=True, help = "read checkpoint from CKPT") def main(args): values = app.get_options() if len(args) > 0: print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (" ".join(args)) app.help() sys.exit(1) if not values.ckpt: print >> sys.stderr, "ERROR: must supply --checkpoint" app.help() sys.exit(1) fp = file(values.ckpt, "r") rr = ThriftRecordReader(fp, RunnerCkpt)
import socket import sys import time from twitter.common import app from twitter.common.exceptions import ExceptionalThread from twitter.common.http import HttpServer from twitter.common.http.diagnostics import DiagnosticsEndpoints from twitter.thermos.common.path import TaskPath from twitter.thermos.observer.task_observer import TaskObserver from twitter.thermos.observer.http.http_observer import BottleObserver app.add_option("--root", dest="root", metavar="DIR", default=TaskPath.DEFAULT_CHECKPOINT_ROOT, help="root checkpoint directory for thermos task runners") app.add_option("--port", dest="port", metavar="INT", default=1338, help="port number to listen on.") def main(args, opts): if args: print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr) app.help()
from __future__ import print_function import re from twitter.common import app from .common import Nit, PythonFile from .iterators import git_iterator, path_iterator from .plugins import list_plugins app.add_option( '-p', action='append', type='str', default=[], dest='plugins', help='Explicitly list plugins to enable.') app.add_option( '-n', action='append', type='str', default=[], dest='skip_plugins', help='Explicitly list plugins to disable.') app.add_option(
(self._host, self._port)) session = NailgunSession(sock, self._ins, self._out, self._err) try: return session.execute(self._work_dir, main_class, *args, **environment) except socket.error as e: raise NailgunError('Problem contacting nailgun server %s:%d %s' % (self._host, self._port, e)) finally: sock.close() app.add_option('--nailgun-version', dest='just_version', default=False, action='store_true', help='print product version and exit') app.add_option('--nailgun-showversion', dest='show_version', default=False, action='store_true', help='print product version and continue') app.add_option( '--nailgun-server', dest='ng_host', metavar='HOST', default=DEFAULT_NG_HOST, help='to specify the address of the nailgun server (default is %default)')
import sys import traceback from twitter.common import app, log from apache.thermos.common.options import add_port_to from apache.thermos.common.planner import TaskPlanner from apache.thermos.common.statuses import (INTERNAL_ERROR, INVALID_TASK, TERMINAL_TASK, UNKNOWN_ERROR, UNKNOWN_USER) from apache.thermos.config.loader import ThermosConfigLoader from apache.thermos.core.process import Process from apache.thermos.core.runner import TaskRunner app.add_option("--thermos_json", dest="thermos_json", default=None, help="read a thermos Task from a serialized json blob") app.add_option( "--sandbox", dest="sandbox", metavar="PATH", default=None, help= "The path on the host filesystem to the sandbox in which this task should run." ) app.add_option( '--container_sandbox', dest='container_sandbox', type=str,
import functools import os import signal import sys import traceback from twitter.common import app, log from apache.thermos.common.options import add_port_to from apache.thermos.common.planner import TaskPlanner from apache.thermos.config.loader import ThermosConfigLoader from apache.thermos.core.runner import TaskRunner app.add_option( "--thermos_json", dest="thermos_json", default=None, help="read a thermos Task from a serialized json blob") app.add_option( "--sandbox", dest="sandbox", metavar="PATH", default=None, help="the sandbox in which this task should run") app.add_option( "--checkpoint_root", dest="checkpoint_root",
def proxy_main(): app.add_option('--port', dest='api_port', type='int', default=None, help='Port for the HTTP API server') app.add_option( '--mesos_master', dest='mesos_master', default=None, help= 'Mesos master address. It can be a ZooKeeper URL through which the master can be ' 'detected') app.add_option('--framework_user', dest='framework_user', help='The Unix user that Mysos executor runs as') app.add_option( '--framework_role', dest='framework_role', default='*', help= "The role that Mysos framework runs as. If set, Mysos only uses Mesos pool resources " "with that role. The default value '*' is what Mesos considers as the default role.\n" "NOTE: Mesos master needs to be configured to allow the specified role. See its --roles " "flag") app.add_option('--executor_uri', dest='executor_uri', default=None, help='URI for the Mysos executor package') app.add_option('--executor_cmd', dest='executor_cmd', default=None, help='Command to execute the executor package') app.add_option( '--executor_environ', dest='executor_environ', default=None, help= "Environment variables for the executors (and the tasks) as a list of dicts keyed by " "{name, value} in JSON. Note that these variables don't affect Mesos slave components " "such as the fetcher") app.add_option( '--zk_url', dest='zk_url', default=None, help='ZooKeeper URL for various Mysos operations, in the form of ' '"zk://*****:*****@servers/path". The sub-directory <zk_url>/discover is used for ' 'communicating MySQL cluster information between Mysos scheduler and executors' ) # TODO(jyx): This could also be made a per-cluster configuration. app.add_option( '--election_timeout', dest='election_timeout', default='60s', help= 'The amount of time the scheduler waits for all slaves to respond during a MySQL master ' 'election, e.g., 60s. After the timeout the master is elected from only the slaves that ' 'have responded') app.add_option( '--admin_keypath', dest='admin_keypath', default=None, help= 'The path to the key file with MySQL admin credentials on Mesos slaves' ) app.add_option( '--work_dir', dest='work_dir', default=os.path.join(tempfile.gettempdir(), 'mysos'), help= "Directory path to place Mysos work directories, e.g., web assets, state files if " "--state_storage=local. Default to a system temp directory.") app.add_option( '--state_storage', dest='state_storage', default='zk', help= "Mechanism to persist scheduler state. Available options are 'zk' and 'local'. If 'zk' " "is chosen, the scheduler state is stored under <zk_url>/state; see --zk_url. Otherwise " "'local' is chosen and the state is persisted under <work_dir>/state; see --work_dir" ) app.add_option( '--framework_failover_timeout', dest='framework_failover_timeout', default='14d', help= 'Time after which Mysos framework is considered deleted. This implies losing all tasks. ' 'SHOULD BE VERY HIGH') # TODO(jyx): Flags like this are generally optional but specific executor implementations may # require them. Consider adding validators that can be plugged in so configuration errors can be # caught in the scheduler. app.add_option( '--installer_args', dest='installer_args', default=None, help= 'Arguments for MySQL installer directly passed along to and parsed by the installer. ' 'e.g., a serialized JSON string') app.add_option( '--backup_store_args', dest='backup_store_args', default=None, help= "Arguments for the store for MySQL backups. Its use and format are defined by the " "backup store implementation. e.g., It can be a serialized JSON string" ) app.add_option( '--framework_authentication_file', dest='framework_authentication_file', default=None, help= "Path to the key file for authenticating the framework against Mesos master. Framework " "will fail to register with Mesos if authentication is required by Mesos and this " "option is not provided") def main(args, options): log.info("Options in use: %s", options) if not options.api_port: app.error('Must specify --port') if not options.mesos_master: app.error('Must specify --mesos_master') if not options.framework_user: app.error('Must specify --framework_user') if not options.executor_uri: app.error('Must specify --executor_uri') if not options.executor_cmd: app.error('Must specify --executor_cmd') if not options.zk_url: app.error('Must specify --zk_url') if not options.admin_keypath: app.error('Must specify --admin_keypath') try: election_timeout = parse_time(options.election_timeout) framework_failover_timeout = parse_time( options.framework_failover_timeout) except InvalidTime as e: app.error(e.message) try: _, zk_servers, zk_root = zookeeper.parse(options.zk_url) except Exception as e: app.error("Invalid --zk_url: %s" % e.message) web_assets_dir = os.path.join(options.work_dir, "web") pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH) log.info("Extracted web assets into %s" % options.work_dir) fw_principal = None fw_secret = None if options.framework_authentication_file: try: with open(options.framework_authentication_file, "r") as f: cred = yaml.load(f) fw_principal = cred["principal"] fw_secret = cred["secret"] log.info( "Loaded credential (principal=%s) for framework authentication" % fw_principal) except IOError as e: app.error( "Unable to read the framework authentication key file: %s" % e) except (KeyError, yaml.YAMLError) as e: app.error( "Invalid framework authentication key file format %s" % e) log.info("Starting Mysos scheduler") kazoo = KazooClient(zk_servers) kazoo.start() if options.state_storage == 'zk': log.info("Using ZooKeeper (path: %s) for state storage" % zk_root) state_provider = ZooKeeperStateProvider(kazoo, zk_root) else: log.info("Using local disk for state storage") state_provider = LocalStateProvider(options.work_dir) try: state = state_provider.load_scheduler_state() except StateProvider.Error as e: app.error(e.message) if state: log.info("Successfully restored scheduler state") framework_info = state.framework_info if framework_info.HasField('id'): log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value) else: log.info("No scheduler state to restore") framework_info = FrameworkInfo( user=options.framework_user, name=FRAMEWORK_NAME, checkpoint=True, failover_timeout=framework_failover_timeout.as_(Time.SECONDS), role=options.framework_role) if fw_principal: framework_info.principal = fw_principal state = Scheduler(framework_info) state_provider.dump_scheduler_state(state) scheduler = MysosScheduler(state, state_provider, options.framework_user, options.executor_uri, options.executor_cmd, kazoo, options.zk_url, election_timeout, options.admin_keypath, installer_args=options.installer_args, backup_store_args=options.backup_store_args, executor_environ=options.executor_environ, framework_role=options.framework_role) if fw_principal and fw_secret: cred = Credential(principal=fw_principal, secret=fw_secret) scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, options.mesos_master, cred) else: scheduler_driver = mesos.native.MesosSchedulerDriver( scheduler, framework_info, options.mesos_master) scheduler_driver.start() server = HttpServer() server.mount_routes(MysosServer(scheduler, web_assets_dir)) et = ExceptionalThread(target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy')) et.daemon = True et.start() try: # Wait for the scheduler to stop. # The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the # process with SIGINT. while not scheduler.stopped.wait(timeout=0.5): pass except KeyboardInterrupt: log.info('Interrupted, exiting.') else: log.info('Scheduler exited.') app.shutdown( 1 ) # Mysos scheduler is supposed to be long-running thus the use of exit status 1. app.main()
sock = self.try_connect() if not sock: raise NailgunError('Problem connecting to nailgun server %s:%d' % (self._host, self._port)) session = NailgunSession(sock, self._ins, self._out, self._err) try: return session.execute(self._work_dir, main_class, *args, **environment) except socket.error as e: raise NailgunError('Problem contacting nailgun server %s:%d %s' % (self._host, self._port, e)) finally: sock.close() app.add_option('--nailgun-version', dest='just_version', default=False, action='store_true', help='print product version and exit') app.add_option('--nailgun-showversion', dest='show_version', default=False, action='store_true', help='print product version and continue') app.add_option('--nailgun-server', dest='ng_host', metavar='HOST', default=DEFAULT_NG_HOST, help='to specify the address of the nailgun server (default is %default)')
from twitter.common import app from twitter.common.exceptions import ExceptionalThread from twitter.common.log.options import LogOptions from twitter.common.quantity import Amount, Time from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.thermos.common.excepthook import ExceptionTerminationHandler from apache.thermos.monitoring.disk import DiskCollectorSettings from apache.thermos.monitoring.resource import TaskResourceMonitor from apache.thermos.observer.http.configure import configure_server from apache.thermos.observer.task_observer import TaskObserver app.add_option( '--mesos-root', dest='mesos_root', type='string', default=MesosPathDetector.DEFAULT_MESOS_ROOT, help= 'The mesos root directory to search for Thermos executor sandboxes [default: %default]' ) app.add_option('--ip', dest='ip', type='string', default='0.0.0.0', help='The IP address the observer will bind to.') app.add_option('--port', dest='port', type='int', default=1338, help='The port on which the observer should listen.')
import time from twitter.common import app, log from twitter.common.exceptions import ExceptionalThread from twitter.common.log.options import LogOptions from twitter.common.quantity import Amount, Time from apache.aurora.executor.common.path_detector import MesosPathDetector from apache.thermos.monitoring.resource import TaskResourceMonitor from apache.thermos.observer.http.configure import configure_server from apache.thermos.observer.task_observer import TaskObserver app.add_option( '--mesos-root', dest='mesos_root', type='string', default=MesosPathDetector.DEFAULT_MESOS_ROOT, help= 'The mesos root directory to search for Thermos executor sandboxes [default: %default]' ) app.add_option('--ip', dest='ip', type='string', default='0.0.0.0', help='The IP address the observer will bind to.') app.add_option('--port', dest='port', type='int', default=1338, help='The port on which the observer should listen.')
app.register_commands_from(admin, help_commands, maintenance) add_verbosity_options() def main(): app.help() try: from apache.aurora.kerberos.auth_module import KerberosAuthModule register_auth_module(KerberosAuthModule()) except ImportError: # Use default auth implementation if kerberos is not available. pass LogOptions.set_stderr_log_level('INFO') LogOptions.disable_disk_logging() app.set_name('aurora-admin') app.set_usage(generate_terse_usage()) app.add_option('--bypass-leader-redirect', action='store_true', default=False, dest='bypass_leader_redirect', help='Bypass the scheduler\'s leader redirect filter') def proxy_main(): app.main()
UserOverrideThermosTaskRunnerProvider ) from apache.thermos.common.path import TaskPath CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) app.add_option( '--announcer-enable', dest='announcer_enable', action='store_true', default=False, help='Enable the ServerSet announcer for this executor. Jobs must still activate using ' 'the Announcer configuration.') app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path',
from jenkinsapi.exceptions import UnknownJob import os import sys from twitter.common import app from twitter.common import log from twitter.common import dirutil from twitter.common.log.options import LogOptions app.set_usage( "jenkins --[server|outdir|config] [--job=[jobname]] [create|delete|show|enable|disable]" ) app.add_option("--server", default="ci.makewhat.is:8080", dest="server", help="jenkins server") app.add_option("--job", default=None, dest="job", help="job name") app.add_option("--config", default=None, dest="config", help="config path") app.add_option("--config_dir", default=None, dest="config_dir", help="config directory") app.add_option("--outdir", default=None, dest="outdir", help="config dump output path") app.add_option("--verbose", default=None, dest="verbose", help="print more things")
def configure_app(app): """ Register the application's options, set usage, and configure submodules. """ app.set_name('starsystem') app.set_usage("{} [opts]\nOptions marked with * are required.".format(app.name())) app.add_option('-i', '--uri', dest='subsonic_uri', help='* URI of the Subsonic server.') app.add_option('-u', '--user', dest='username', help='* Username on the specified Subsonic server.') app.add_option('-t', '--token', dest='token', help='* API token for the given username/salt combination\n' 'See: http://www.subsonic.org/pages/api.jsp') app.add_option('-s', '--salt', dest='salt', help='* Salt used to generate the API token.') app.add_option('-p', '--path', dest='download_path', help='* Path to the directory whither songs will be downloaded.') app.add_option('-S', '--since', dest='since', type='date', help='Collect all songs since the specified date.') app.add_option('-I', '--insecure', dest='insecure', default=False, action="store_true", help='Don\'t verify SSL certificates. Verification is enabled by default.') app.add_option('-g', '--gen-token-interactive', dest='gen_token', default=False, action="store_true", help='Generate an API token interactively.') app.add_option('-v', '--debug', dest='debug', default=False, action="store_true", help='Enable debug output.') app.set_option('twitter_common_log_disk_log_level', 'NONE', force=True)
try: from mesos.executor import MesosExecutorDriver except ImportError: print(traceback.format_exc(), file=sys.stderr) MesosExecutorDriver = None CWD = os.environ.get('MESOS_SANDBOX', '.') app.configure(debug=True) LogOptions.set_simple(True) LogOptions.set_disk_log_level('DEBUG') LogOptions.set_log_dir(CWD) app.add_option( '--announcer-ensemble', dest='announcer_ensemble', type=str, default=None, help='The ensemble to which the Announcer should register ServerSets.') app.add_option( '--announcer-serverset-path', dest='announcer_serverset_path', type=str, default='/aurora', help= 'The root of the tree into which ServerSets should be announced. The paths will ' 'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.') app.add_option( '--announcer-allow-custom-serverset-path', dest='announcer_allow_custom_serverset_path',