Exemplo n.º 1
0
def run():
    def main(args, opts):
        """Main"""
        server = RedirServer(opts.zk_basepath,
                             opts.subdomain,
                             opts.base_domain)
        thread = ExceptionalThread(
            target=lambda: server.run(opts.listen,
                                      opts.port,
                                      server='cherrypy'))
        thread.daemon = True
        thread.start()

        wait_forever()

    log.LogOptions.set_stderr_log_level('google:INFO')

    app.add_option('--port', help='http port', default=8080)
    app.add_option('--listen',
                   help='IP address to listen for http connections.',
                   default='0.0.0.0')
    app.add_option('--zk_basepath',
                   help='Zookeeper service path root.',
                   default='/aurora')
    app.add_option('--base_domain',
                   help='Domain name of your site.',
                   default='example.com')
    app.add_option('--subdomain',
                   help='Subdomain that roots Aurora job namespace.',
                   default='aurora')

    app.main()
Exemplo n.º 2
0
 def test_app_add_options_with_Option(self):
   # options.Option
   opt = options.Option('--option1', dest='option1')
   app.add_option(opt)
   app.init(force_args=['--option1', 'option1value', 'extraargs'])
   assert app.get_options().option1 == 'option1value'
   assert app.argv() == ['extraargs']
Exemplo n.º 3
0
def add_verbosity_options():
    from twitter.common import app
    from twitter.common.log.options import LogOptions

    def set_quiet(option, _1, _2, parser):
        setattr(parser.values, option.dest, "quiet")
        LogOptions.set_stderr_log_level("NONE")

    def set_verbose(option, _1, _2, parser):
        setattr(parser.values, option.dest, "verbose")
        LogOptions.set_stderr_log_level("DEBUG")

    app.add_option(
        "-v",
        dest="verbosity",
        default="normal",
        action="callback",
        callback=set_verbose,
        help="Verbose logging. (default: %default)",
    )

    app.add_option(
        "-q",
        dest="verbosity",
        default="normal",
        action="callback",
        callback=set_quiet,
        help="Quiet logging. (default: %default)",
    )
Exemplo n.º 4
0
def add_verbosity_options():
  from twitter.common import app
  from twitter.common.log.options import LogOptions

  def set_quiet(option, _1, _2, parser):
    setattr(parser.values, option.dest, 'quiet')
    LogOptions.set_stderr_log_level('NONE')

  def set_verbose(option, _1, _2, parser):
    setattr(parser.values, option.dest, 'verbose')
    LogOptions.set_stderr_log_level('DEBUG')

  app.add_option('-v',
                 dest='verbosity',
                 default='normal',
                 action='callback',
                 callback=set_verbose,
                 help='Verbose logging. (default: %default)')

  app.add_option('-q',
                 dest='verbosity',
                 default='normal',
                 action='callback',
                 callback=set_quiet,
                 help='Quiet logging. (default: %default)')
Exemplo n.º 5
0
def setup():
  LogOptions.set_stderr_log_level('NONE')

  app.add_option('--iface', default='eth0', type=str)
  app.add_option('--port', default=3888, type=int)
  app.add_option('-c', '--colors', default=False, action='store_true')
  app.add_option('--dump-bad-packet', default=False, action='store_true')
  app.add_option('--version', default=False, action='store_true')
from apache.aurora.executor.aurora_executor import AuroraExecutor
from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider
from apache.aurora.executor.common.executor_timeout import ExecutorTimeout
from apache.aurora.executor.common.health_checker import HealthCheckerProvider
from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir('.')


app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help='Enable the ServerSet announcer for this executor.  Jobs must still activate using '
         'the Announcer configuration.')


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
Exemplo n.º 7
0
from apache.thermos.common.ckpt import CheckpointDispatcher
from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
from apache.thermos.common.options import add_binding_to, add_port_to
from apache.thermos.common.path import TaskPath
from apache.thermos.config.loader import ThermosConfigLoader, ThermosTaskWrapper
from apache.thermos.config.schema import Process, Resources, Task
from apache.thermos.core.helper import TaskRunnerHelper
from apache.thermos.core.runner import TaskRunner
from apache.thermos.monitoring.detector import TaskDetector
from apache.thermos.monitoring.garbage import DefaultCollector, TaskGarbageCollector
from apache.thermos.monitoring.monitor import TaskMonitor

from gen.apache.thermos.ttypes import ProcessState, RunnerCkpt, RunnerState, TaskState

app.add_option("--root", dest="root", metavar="PATH",
               default=DEFAULT_CHECKPOINT_ROOT,
               help="the thermos config root")


def set_keep(option, opt_str, value, parser):
  setattr(parser.values, option.dest, opt_str.startswith('--keep'))


def get_task_from_options(args, opts, **kw):
  loader = ThermosConfigLoader.load_json if opts.json else ThermosConfigLoader.load

  if len(args) != 1:
    app.error('Should specify precisely one config, instead got: %s' % args)

  tasks = loader(args[0], bindings=opts.bindings, **kw)
Exemplo n.º 8
0
# ==================================================================================================


import socket
import threading
import zookeeper
from functools import wraps
from twitter.common import app
from twitter.common import log
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time


app.add_option(
    '--zookeeper',
    default='zookeeper.local.twitter.com:2181',
    metavar='HOST:PORT[,HOST:PORT,...]',
    help='Comma-separated list of host:port of ZooKeeper servers',
    )
app.add_option(
    '--zookeeper_timeout',
    type='float',
    default=5.0,
    help='default timeout (in seconds) for ZK operations',
    )
app.add_option(
    '--enable_zookeeper_debug_logging',
    dest='twitter_common_zookeeper_debug',
    default=False,
    action='store_true',
    help='whether to enable ZK debug logging to stderr',
    )
Exemplo n.º 9
0
CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

_LOGGER_TYPES = ', '.join(LoggerMode.VALUES)


app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help='Enable the ServerSet announcer for this executor.  Jobs must still activate using '
         'the Announcer configuration.')


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
Exemplo n.º 10
0
import functools
import os
import signal
import sys
import traceback

from twitter.common import app, log
from apache.thermos.common.options import add_port_to
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.config.loader import ThermosConfigLoader
from apache.thermos.core.runner import TaskRunner


app.add_option(
    "--thermos_json",
    dest="thermos_json",
    default=None,
    help="read a thermos Task from a serialized json blob")


app.add_option(
    "--sandbox",
    dest="sandbox",
    metavar="PATH",
    default=None,
    help="the sandbox in which this task should run")


app.add_option(
     "--checkpoint_root",
     dest="checkpoint_root",
Exemplo n.º 11
0
    sock = self.try_connect()
    if not sock:
      raise NailgunError('Problem connecting to nailgun server %s:%d' % (self._host, self._port))

    session = NailgunSession(sock, self._ins, self._out, self._err)
    try:
      return session.execute(self._work_dir, main_class, *args, **environment)
    except socket.error as e:
      raise NailgunError('Problem contacting nailgun server %s:%d %s' % (self._host, self._port, e))
    finally:
      sock.close()


app.add_option('--nailgun-version',
               dest='just_version',
               default=False,
               action='store_true',
               help='print product version and exit')

app.add_option('--nailgun-showversion',
               dest='show_version',
               default=False,
               action='store_true',
               help='print product version and continue')

app.add_option('--nailgun-server',
               dest='ng_host',
               metavar='HOST',
               default=DEFAULT_NG_HOST,
               help='to specify the address of the nailgun server (default is %default)')
Exemplo n.º 12
0
import os
import sys
import pprint
import time

from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState

from twitter.common import app
from twitter.common.recordio import RecordIO, ThriftRecordReader
from twitter.thermos.common.ckpt import CheckpointDispatcher

app.add_option("--checkpoint", dest = "ckpt", metavar = "CKPT",
               help = "read checkpoint from CKPT")
app.add_option("--assemble", dest = "assemble", metavar = "CKPT", default=True,
               help = "read checkpoint from CKPT")

def main(args):
  values = app.get_options()

  if len(args) > 0:
    print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (" ".join(args))
    app.help()
    sys.exit(1)

  if not values.ckpt:
    print >> sys.stderr, "ERROR: must supply --checkpoint"
    app.help()
    sys.exit(1)

  fp = file(values.ckpt, "r")
  rr = ThriftRecordReader(fp, RunnerCkpt)
Exemplo n.º 13
0
app.register_commands_from(admin, help_commands, maintenance)
add_verbosity_options()


def main():
  app.help()

try:
  from apache.aurora.kerberos.auth_module import KerberosAuthModule
  register_auth_module(KerberosAuthModule())
except ImportError:
  # Use default auth implementation if kerberos is not available.
  pass

LogOptions.set_stderr_log_level('INFO')
LogOptions.disable_disk_logging()
app.set_name('aurora-admin')
app.set_usage(generate_terse_usage())

app.add_option(
    '--bypass-leader-redirect',
    action='store_true',
    default=False,
    dest='bypass_leader_redirect',
    help='Bypass the scheduler\'s leader redirect filter')


def proxy_main():
  app.main()
Exemplo n.º 14
0
from collections import defaultdict

from rainman.codec import BEncoder

from twitter.common import app, log
from twitter.common.http import HttpServer
from twitter.common.quantity import Amount, Time


app.add_option(
    '--port',
    dest='port',
    default=8080,
    type=int,
    help='The port on which the torrent tracker should be run.')


class TrackerRequest(object):
  class Error(Exception): pass
  class MalformedRequestError(Error): pass

  REQUIRED_KEYS = frozenset([
    'info_hash',
    'peer_id',
    'port',
    'uploaded',
    'downloaded',
    'left',
  ])

  OPTIONAL_KEYS = set(['ip', 'event'])
Exemplo n.º 15
0
      LogOptions.set_stdout_log_level(value)
    except LogOptionsException, e:
      raise optparse.OptionValueError('Failed to parse option: %s' % e)
    parser.values.twitter_common_log_stdout_log_level = value

_LOGGING_HELP = \
"""The level at which to log to %%s [default: %%%%default].
Takes either LEVEL or scheme:LEVEL, where LEVEL is one
of %s and scheme is one of %s.
""" % (repr(LogOptions._LOG_LEVELS.keys()), repr(LogOptions._LOG_SCHEMES))

if HAVE_APP:
  app.add_option('--log_to_stdout',
              callback=LogOptions._stdout_options_callback,
              default=_DEFAULT_LOG_OPTS.twitter_common_log_stdout_log_level,
              type='string',
              action='callback',
              metavar='[scheme:]LEVEL',
              dest='twitter_common_log_stdout_log_level',
              help=_LOGGING_HELP % 'stdout')

  app.add_option('--log_to_disk',
              callback=LogOptions._disk_options_callback,
              default=_DEFAULT_LOG_OPTS.twitter_common_log_disk_log_level,
              type='string',
              action='callback',
              metavar='[scheme:]LEVEL',
              dest='twitter_common_log_disk_log_level',
              help=_LOGGING_HELP % 'disk')

  app.add_option('--log_dir',
              type='string',
Exemplo n.º 16
0
def setup():
  app.add_option('--iface', default='eth0', type=str)
  app.add_option('--port', default=3888, type=int)
  app.add_option('-c', '--colors', default=False, action='store_true')
Exemplo n.º 17
0
from jenkinsapi.exceptions import UnknownJob

import os
import sys

from twitter.common import app
from twitter.common import log
from twitter.common import dirutil

from twitter.common.log.options import LogOptions


app.set_usage("jenkins --[server|outdir|config]  [--job=[jobname]] [create|delete|show|enable|disable]")
app.add_option(
  "--server",
  default="ci.makewhat.is:8080",
  dest="server",
  help="jenkins server"
)
app.add_option(
  "--job",
  default=None,
  dest="job",
  help="job name"
)
app.add_option(
  "--config",
  default=None,
  dest="config",
  help="config path"
)
app.add_option(
Exemplo n.º 18
0
import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
from apache.thermos.monitoring.detector import ChainedPathDetector, FixedPathDetector
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--root',
    dest='root',
    type='string',
    default=DEFAULT_CHECKPOINT_ROOT,
    help='The thermos checkpoint root directory to search for Thermos tasks [default: %default]')


app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help='The mesos root directory to search for Thermos executor sandboxes [default: %default]')


app.add_option(
    '--port',
    dest='port',
Exemplo n.º 19
0
def proxy_main():
  app.add_option(
      '--port',
      dest='api_port',
      type='int',
      default=None,
      help='Port for the HTTP API server')

  app.add_option(
      '--mesos_master',
      dest='mesos_master',
      default=None,
      help='Mesos master address. It can be a ZooKeeper URL through which the master can be '
           'detected')

  app.add_option(
      '--framework_user',
      dest='framework_user',
      help='The Unix user that Mysos executor runs as')

  app.add_option(
      '--framework_role',
      dest='framework_role',
      default='*',
      help="The role that Mysos framework runs as. If set, Mysos only uses Mesos pool resources "
           "with that role. The default value '*' is what Mesos considers as the default role.\n"
           "NOTE: Mesos master needs to be configured to allow the specified role. See its --roles "
           "flag")

  app.add_option(
      '--executor_uri',
      dest='executor_uri',
      default=None,
      help='URI for the Mysos executor package')

  app.add_option(
      '--executor_cmd',
      dest='executor_cmd',
      default=None,
      help='Command to execute the executor package')

  app.add_option(
      '--executor_environ',
      dest='executor_environ',
      default=None,
      help="Environment variables for the executors (and the tasks) as a list of dicts keyed by "
           "{name, value} in JSON. Note that these variables don't affect Mesos slave components "
           "such as the fetcher")

  app.add_option(
      '--zk_url',
      dest='zk_url',
      default=None,
      help='ZooKeeper URL for various Mysos operations, in the form of '
           '"zk://username:[email protected]/path". The sub-directory <zk_url>/discover is used for '
           'communicating MySQL cluster information between Mysos scheduler and executors')

  # TODO(jyx): This could also be made a per-cluster configuration.
  app.add_option(
      '--election_timeout',
      dest='election_timeout',
      default='60s',
      help='The amount of time the scheduler waits for all slaves to respond during a MySQL master '
           'election, e.g., 60s. After the timeout the master is elected from only the slaves that '
           'have responded')

  app.add_option(
      '--admin_keypath',
      dest='admin_keypath',
      default=None,
      help='The path to the key file with MySQL admin credentials on Mesos slaves')

  app.add_option(
      '--work_dir',
      dest='work_dir',
      default=os.path.join(tempfile.gettempdir(), 'mysos'),
      help="Directory path to place Mysos work directories, e.g., web assets, state files if "
           "--state_storage=local. Default to a system temp directory.")

  app.add_option(
      '--state_storage',
      dest='state_storage',
      default='zk',
      help="Mechanism to persist scheduler state. Available options are 'zk' and 'local'. If 'zk' "
           "is chosen, the scheduler state is stored under <zk_url>/state; see --zk_url. Otherwise "
           "'local' is chosen and the state is persisted under <work_dir>/state; see --work_dir")

  app.add_option(
      '--framework_failover_timeout',
      dest='framework_failover_timeout',
      default='14d',
      help='Time after which Mysos framework is considered deleted. This implies losing all tasks. '
           'SHOULD BE VERY HIGH')

  # TODO(jyx): Flags like this are generally optional but specific executor implementations may
  # require them. Consider adding validators that can be plugged in so configuration errors can be
  # caught in the scheduler.
  app.add_option(
      '--installer_args',
      dest='installer_args',
      default=None,
      help='Arguments for MySQL installer directly passed along to and parsed by the installer. '
           'e.g., a serialized JSON string'
  )

  app.add_option(
      '--backup_store_args',
      dest='backup_store_args',
      default=None,
      help="Arguments for the store for MySQL backups. Its use and format are defined by the "
           "backup store implementation. e.g., It can be a serialized JSON string"
  )

  app.add_option(
      '--framework_authentication_file',
      dest='framework_authentication_file',
      default=None,
      help="Path to the key file for authenticating the framework against Mesos master. Framework "
           "will fail to register with Mesos if authentication is required by Mesos and this "
           "option is not provided"
  )

  def main(args, options):
    log.info("Options in use: %s", options)

    if not options.api_port:
      app.error('Must specify --port')

    if not options.mesos_master:
      app.error('Must specify --mesos_master')

    if not options.framework_user:
      app.error('Must specify --framework_user')

    if not options.executor_uri:
      app.error('Must specify --executor_uri')

    if not options.executor_cmd:
      app.error('Must specify --executor_cmd')

    if not options.zk_url:
      app.error('Must specify --zk_url')

    if not options.admin_keypath:
      app.error('Must specify --admin_keypath')

    try:
      election_timeout = parse_time(options.election_timeout)
      framework_failover_timeout = parse_time(options.framework_failover_timeout)
    except InvalidTime as e:
      app.error(e.message)

    try:
      _, zk_servers, zk_root = zookeeper.parse(options.zk_url)
    except Exception as e:
      app.error("Invalid --zk_url: %s" % e.message)

    web_assets_dir = os.path.join(options.work_dir, "web")
    pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH)
    log.info("Extracted web assets into %s" % options.work_dir)

    fw_principal = None
    fw_secret = None
    if options.framework_authentication_file:
      try:
        with open(options.framework_authentication_file, "r") as f:
          cred = yaml.load(f)
        fw_principal = cred["principal"]
        fw_secret = cred["secret"]
        log.info("Loaded credential (principal=%s) for framework authentication" % fw_principal)
      except IOError as e:
        app.error("Unable to read the framework authentication key file: %s" % e)
      except (KeyError, yaml.YAMLError) as e:
        app.error("Invalid framework authentication key file format %s" % e)

    log.info("Starting Mysos scheduler")

    kazoo = KazooClient(zk_servers)
    kazoo.start()

    if options.state_storage == 'zk':
      log.info("Using ZooKeeper (path: %s) for state storage" % zk_root)
      state_provider = ZooKeeperStateProvider(kazoo, zk_root)
    else:
      log.info("Using local disk for state storage")
      state_provider = LocalStateProvider(options.work_dir)

    try:
      state = state_provider.load_scheduler_state()
    except StateProvider.Error as e:
      app.error(e.message)

    if state:
      log.info("Successfully restored scheduler state")
      framework_info = state.framework_info
      if framework_info.HasField('id'):
        log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value)
    else:
      log.info("No scheduler state to restore")
      framework_info = FrameworkInfo(
          user=options.framework_user,
          name=FRAMEWORK_NAME,
          checkpoint=True,
          failover_timeout=framework_failover_timeout.as_(Time.SECONDS),
          role=options.framework_role)
      if fw_principal:
        framework_info.principal = fw_principal
      state = Scheduler(framework_info)
      state_provider.dump_scheduler_state(state)

    scheduler = MysosScheduler(
        state,
        state_provider,
        options.framework_user,
        options.executor_uri,
        options.executor_cmd,
        kazoo,
        options.zk_url,
        election_timeout,
        options.admin_keypath,
        installer_args=options.installer_args,
        backup_store_args=options.backup_store_args,
        executor_environ=options.executor_environ,
        framework_role=options.framework_role)

    if fw_principal and fw_secret:
      cred = Credential(principal=fw_principal, secret=fw_secret)
      scheduler_driver = mesos.native.MesosSchedulerDriver(
          scheduler,
          framework_info,
          options.mesos_master,
          cred)
    else:
      scheduler_driver = mesos.native.MesosSchedulerDriver(
          scheduler,
          framework_info,
          options.mesos_master)

    scheduler_driver.start()

    server = HttpServer()
    server.mount_routes(MysosServer(scheduler, web_assets_dir))

    et = ExceptionalThread(
        target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy'))
    et.daemon = True
    et.start()

    try:
      # Wait for the scheduler to stop.
      # The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the
      # process with SIGINT.
      while not scheduler.stopped.wait(timeout=0.5):
        pass
    except KeyboardInterrupt:
      log.info('Interrupted, exiting.')
    else:
      log.info('Scheduler exited.')

    app.shutdown(1)  # Mysos scheduler is supposed to be long-running thus the use of exit status 1.

  app.main()
Exemplo n.º 20
0
from __future__ import print_function

import re

from twitter.common import app

from .common import Nit, PythonFile
from .iterators import git_iterator, path_iterator
from .plugins import list_plugins


app.add_option(
  '-p',
  action='append',
  type='str',
  default=[],
  dest='plugins',
  help='Explicitly list plugins to enable.')


app.add_option(
  '-n',
  action='append',
  type='str',
  default=[],
  dest='skip_plugins',
  help='Explicitly list plugins to disable.')


app.add_option(
Exemplo n.º 21
0
import socket
import sys
import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.http import HttpServer
from twitter.common.http.diagnostics import DiagnosticsEndpoints
from twitter.thermos.common.path import TaskPath
from twitter.thermos.observer.task_observer import TaskObserver
from twitter.thermos.observer.http.http_observer import BottleObserver


app.add_option("--root",
               dest="root",
               metavar="DIR",
               default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
               help="root checkpoint directory for thermos task runners")


app.add_option("--port",
               dest="port",
               metavar="INT",
               default=1338,
               help="port number to listen on.")


def main(args, opts):
  if args:
    print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
    app.help()
    UserOverrideThermosTaskRunnerProvider
)
from apache.thermos.common.path import TaskPath

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)


app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help='Enable the ServerSet announcer for this executor.  Jobs must still activate using '
         'the Announcer configuration.')


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
Exemplo n.º 23
0
    return filename


def main(args, options):
  from pkg_resources import WorkingSet, Requirement, find_distributions

  if not options.site_dir:
    app.error('Must supply --site')

  distributions = list(find_distributions(options.site_dir))
  working_set = WorkingSet()
  for dist in distributions:
    working_set.add(dist)

  for arg in args:
    arg_req = Requirement.parse(arg)
    found_dist = working_set.find(arg_req)
    if not found_dist:
      print('Could not find %s!' % arg_req)
    out_zip = Distiller(found_dist).distill()
    print('Dumped %s => %s' % (arg_req, out_zip))


if HAS_APP:
  if __name__ == '__main__':
    app.add_option('--site', dest='site_dir', metavar='DIR', default=None,
                   help='Directory to search for the requirement.')

  app.main()
Exemplo n.º 24
0
def setup():
  LogOptions.set_stderr_log_level('NONE')

  app.add_option('--iface', default='eth0', type=str,
                 help='The interface to sniff on')
  app.add_option('--port', default=2889, type=int,
                 help='The ZAB port used by the leader')
  app.add_option('-c', '--colors', default=False, action='store_true',
                 help='Color each learner/leader stream differently')
  app.add_option('--dump-bad-packet', default=False, action='store_true',
                 help='Dump packets that cannot be deserialized')
  app.add_option('--include-pings', default=False, action='store_true',
                 help='Whether to include pings send from learners to the leader')
  app.add_option('--version', default=False, action='store_true')
Exemplo n.º 25
0
except ImportError:
  print(traceback.format_exc(), file=sys.stderr)
  MesosExecutorDriver = None


CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
    default='/aurora',
    help='The root of the tree into which ServerSets should be announced.  The paths will '
         'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.')

app.add_option(
    '--announcer-allow-custom-serverset-path',
    dest='announcer_allow_custom_serverset_path',
Exemplo n.º 26
0
import re
import sys
import os.path
from functools import partial

from twitter.common import app, log

ROOT = os.path.join(os.path.dirname(__file__), '..')
sys.path.append(ROOT)

from lib.modules import spout, config, models, event_emitter_2
from lib.modules import sink


app.add_option("--listener", default="CarbonAsyncTcpSpout",
               help="Select the incoming metric connection interface")
app.add_option("--writer", default="RedisSink",
               help="Select the sink connection interface")
app.add_option("--config", help="Collector Config")

# Globals
EE = event_emitter_2.EventEmitter2()
CONFIG = None
WHITELIST = None
BLACKLIST = None

def setup(options):
    """Load the config file and add listeners for whitelisted metrics.

       :param options: app options
    """
Exemplo n.º 27
0
CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

_LOGGER_DESTINATIONS = ', '.join(LoggerDestination.VALUES)
_LOGGER_MODES = ', '.join(LoggerMode.VALUES)


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
    default='/aurora',
    help='The root of the tree into which ServerSets should be announced.  The paths will '
         'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.')

app.add_option(
    '--announcer-allow-custom-serverset-path',
    dest='announcer_allow_custom_serverset_path',
Exemplo n.º 28
0
"""A Mesos-customized entry point to the thermos_observer webserver."""

import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help='The mesos root directory to search for Thermos executor sandboxes [default: %default]')


app.add_option(
    '--ip',
    dest='ip',
    type='string',
    default='0.0.0.0',
    help='The IP address the observer will bind to.')


app.add_option(
    '--port',
    dest='port',
Exemplo n.º 29
0
def configure_app(app):
    """ Register the application's options, set usage, and configure submodules. """

    app.set_name('starsystem')

    app.set_usage("{} [opts]\nOptions marked with * are required.".format(app.name()))

    app.add_option('-i', '--uri', dest='subsonic_uri',
                   help='* URI of the Subsonic server.')
    app.add_option('-u', '--user', dest='username',
                   help='* Username on the specified Subsonic server.')
    app.add_option('-t', '--token', dest='token',
                   help='* API token for the given username/salt combination\n'
                        'See: http://www.subsonic.org/pages/api.jsp')
    app.add_option('-s', '--salt', dest='salt',
                   help='* Salt used to generate the API token.')
    app.add_option('-p', '--path', dest='download_path',
                   help='* Path to the directory whither songs will be downloaded.')
    app.add_option('-S', '--since', dest='since', type='date',
                   help='Collect all songs since the specified date.')
    app.add_option('-I', '--insecure', dest='insecure', default=False, action="store_true",
                   help='Don\'t verify SSL certificates. Verification is enabled by default.')
    app.add_option('-g', '--gen-token-interactive', dest='gen_token', default=False,
                   action="store_true", help='Generate an API token interactively.')
    app.add_option('-v', '--debug', dest='debug', default=False,
                   action="store_true", help='Enable debug output.')

    app.set_option('twitter_common_log_disk_log_level', 'NONE', force=True)
Exemplo n.º 30
0
import atexit
import errno
import socket
import subprocess
import time

from twitter.common.quantity import Amount, Time

try:
  from twitter.common import app
  HAS_APP=True

  app.add_option(
    '--tunnel_host',
    type='string',
    dest='tunnel_host',
    default='nest1.corp.twitter.com',
    help='Host to tunnel commands through (default: %default)')

except ImportError:
  HAS_APP=False


__all__ = (
  'TunnelHelper',
)


def safe_kill(po):
  """
    Given a Popen object, safely kill it without an unexpected exception.