Example #1
0
def add_verbosity_options():
  from twitter.common import app
  from twitter.common.log.options import LogOptions

  def set_quiet(option, _1, _2, parser):
    setattr(parser.values, option.dest, 'quiet')
    LogOptions.set_stderr_log_level('NONE')

  def set_verbose(option, _1, _2, parser):
    setattr(parser.values, option.dest, 'verbose')
    LogOptions.set_stderr_log_level('DEBUG')

  app.add_option('-v',
                 dest='verbosity',
                 default='normal',
                 action='callback',
                 callback=set_verbose,
                 help='Verbose logging. (default: %default)')

  app.add_option('-q',
                 dest='verbosity',
                 default='normal',
                 action='callback',
                 callback=set_quiet,
                 help='Quiet logging. (default: %default)')
Example #2
0
def add_verbosity_options():
  from twitter.common import app
  from twitter.common.log.options import LogOptions

  def set_quiet(option, _1, _2, parser):
    setattr(parser.values, option.dest, 'quiet')
    LogOptions.set_stderr_log_level('NONE')

  def set_verbose(option, _1, _2, parser):
    setattr(parser.values, option.dest, 'verbose')
    LogOptions.set_stderr_log_level('DEBUG')

  app.add_option('-v',
                 dest='verbosity',
                 default='normal',
                 action='callback',
                 callback=set_verbose,
                 help='Verbose logging. (default: %default)')

  app.add_option('-q',
                 dest='verbosity',
                 default='normal',
                 action='callback',
                 callback=set_quiet,
                 help='Quiet logging. (default: %default)')
Example #3
0
def add_verbosity_options():
    from twitter.common import app
    from twitter.common.log.options import LogOptions

    def set_quiet(option, _1, _2, parser):
        setattr(parser.values, option.dest, "quiet")
        LogOptions.set_stderr_log_level("NONE")

    def set_verbose(option, _1, _2, parser):
        setattr(parser.values, option.dest, "verbose")
        LogOptions.set_stderr_log_level("DEBUG")

    app.add_option(
        "-v",
        dest="verbosity",
        default="normal",
        action="callback",
        callback=set_verbose,
        help="Verbose logging. (default: %default)",
    )

    app.add_option(
        "-q",
        dest="verbosity",
        default="normal",
        action="callback",
        callback=set_quiet,
        help="Quiet logging. (default: %default)",
    )
Example #4
0
def run():
    def main(args, opts):
        """Main"""
        server = RedirServer(opts.zk_basepath,
                             opts.subdomain,
                             opts.base_domain)
        thread = ExceptionalThread(
            target=lambda: server.run(opts.listen,
                                      opts.port,
                                      server='cherrypy'))
        thread.daemon = True
        thread.start()

        wait_forever()

    log.LogOptions.set_stderr_log_level('google:INFO')

    app.add_option('--port', help='http port', default=8080)
    app.add_option('--listen',
                   help='IP address to listen for http connections.',
                   default='0.0.0.0')
    app.add_option('--zk_basepath',
                   help='Zookeeper service path root.',
                   default='/aurora')
    app.add_option('--base_domain',
                   help='Domain name of your site.',
                   default='example.com')
    app.add_option('--subdomain',
                   help='Subdomain that roots Aurora job namespace.',
                   default='aurora')

    app.main()
Example #5
0
 def test_app_add_options_with_Option(self):
   # options.Option
   opt = options.Option('--option1', dest='option1')
   app.add_option(opt)
   app.init(force_args=['--option1', 'option1value', 'extraargs'])
   assert app.get_options().option1 == 'option1value'
   assert app.argv() == ['extraargs']
Example #6
0
 def test_app_add_options_with_Option(self):
     # options.Option
     opt = options.Option('--option1', dest='option1')
     app.add_option(opt)
     app.init(force_args=['--option1', 'option1value', 'extraargs'])
     assert app.get_options().option1 == 'option1value'
     assert app.argv() == ['extraargs']
Example #7
0
def setup():
  LogOptions.set_stderr_log_level('NONE')

  app.add_option('--iface', default='eth0', type=str)
  app.add_option('--port', default=3888, type=int)
  app.add_option('-c', '--colors', default=False, action='store_true')
  app.add_option('--dump-bad-packet', default=False, action='store_true')
  app.add_option('--version', default=False, action='store_true')
Example #8
0
def setup():
    LogOptions.set_stderr_log_level('NONE')

    app.add_option('--iface', default='eth0', type=str)
    app.add_option('--port', default=3888, type=int)
    app.add_option('-c', '--colors', default=False, action='store_true')
    app.add_option('--dump-bad-packet', default=False, action='store_true')
    app.add_option('--version', default=False, action='store_true')

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

_LOGGER_TYPES = ', '.join(LoggerMode.VALUES)


app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help='Enable the ServerSet announcer for this executor.  Jobs must still activate using '
         'the Announcer configuration.')


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
Example #10
0
import atexit
import errno
import socket
import subprocess
import time

from twitter.common.quantity import Amount, Time

try:
  from twitter.common import app
  HAS_APP=True

  app.add_option(
    '--tunnel_host',
    type='string',
    dest='tunnel_host',
    default='nest1.corp.twitter.com',
    help='Host to tunnel commands through (default: %default)')

except ImportError:
  HAS_APP=False


__all__ = (
  'TunnelHelper',
)


def safe_kill(po):
  """
    Given a Popen object, safely kill it without an unexpected exception.
Example #11
0
import os
import sys
import pprint
import time

from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState

from twitter.common import app
from twitter.common.recordio import RecordIO, ThriftRecordReader
from twitter.thermos.common.ckpt import CheckpointDispatcher

app.add_option("--checkpoint",
               dest="ckpt",
               metavar="CKPT",
               help="read checkpoint from CKPT")
app.add_option("--assemble",
               dest="assemble",
               metavar="CKPT",
               default=True,
               help="read checkpoint from CKPT")


def main(args):
    values = app.get_options()

    if len(args) > 0:
        print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (
            " ".join(args))
        app.help()
        sys.exit(1)
Example #12
0
# ==================================================================================================


import socket
import threading
import zookeeper
from functools import wraps
from twitter.common import app
from twitter.common import log
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time


app.add_option(
    '--zookeeper',
    default='zookeeper.local.twitter.com:2181',
    metavar='HOST:PORT[,HOST:PORT,...]',
    help='Comma-separated list of host:port of ZooKeeper servers',
    )
app.add_option(
    '--zookeeper_timeout',
    type='float',
    default=5.0,
    help='default timeout (in seconds) for ZK operations',
    )
app.add_option(
    '--enable_zookeeper_debug_logging',
    dest='twitter_common_zookeeper_debug',
    default=False,
    action='store_true',
    help='whether to enable ZK debug logging to stderr',
    )
Example #13
0
import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help=
    'The mesos root directory to search for Thermos executor sandboxes [default: %default]'
)

app.add_option('--port',
               dest='port',
               type='int',
               default=1338,
               help='The port on which the observer should listen.')

app.add_option('--polling_interval_secs',
               dest='polling_interval_secs',
               type='int',
               default=int(TaskObserver.POLLING_INTERVAL.as_(Time.SECONDS)),
               help='The number of seconds between observer refresh attempts.')
from __future__ import print_function

import pprint
import sys
import time

from twitter.common import app
from twitter.common.recordio import RecordIO, ThriftRecordReader

from apache.thermos.common.ckpt import CheckpointDispatcher

from gen.apache.thermos.ttypes import RunnerCkpt, RunnerState, TaskState

app.add_option(
    "--checkpoint",
    dest="ckpt",
    metavar="CKPT",
    help="read checkpoint from CKPT")

app.add_option(
    "--assemble",
    dest="assemble",
    metavar="CKPT",
    default=True,
    help="whether or not to replay the checkpoint records.")


def main(args):
  values = app.get_options()

  if len(args) > 0:
from apache.aurora.executor.aurora_executor import AuroraExecutor
from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider
from apache.aurora.executor.common.executor_timeout import ExecutorTimeout
from apache.aurora.executor.common.health_checker import HealthCheckerProvider
from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir('.')

app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help=
    'Enable the ServerSet announcer for this executor.  Jobs must still activate using '
    'the Announcer configuration.')

app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')

app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
Example #16
0
from jenkinsapi.exceptions import UnknownJob

import os
import sys

from twitter.common import app
from twitter.common import log
from twitter.common import dirutil

from twitter.common.log.options import LogOptions


app.set_usage("jenkins --[server|outdir|config]  [--job=[jobname]] [create|delete|show|enable|disable]")
app.add_option(
  "--server",
  default="ci.makewhat.is:8080",
  dest="server",
  help="jenkins server"
)
app.add_option(
  "--job",
  default=None,
  dest="job",
  help="job name"
)
app.add_option(
  "--config",
  default=None,
  dest="config",
  help="config path"
)
app.add_option(
Example #17
0
import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
from apache.thermos.monitoring.detector import ChainedPathDetector, FixedPathDetector
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--root',
    dest='root',
    type='string',
    default=DEFAULT_CHECKPOINT_ROOT,
    help='The thermos checkpoint root directory to search for Thermos tasks [default: %default]')


app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help='The mesos root directory to search for Thermos executor sandboxes [default: %default]')


app.add_option(
    '--port',
    dest='port',
Example #18
0
def setup():
    app.add_option('--iface', default='eth0', type=str)
    app.add_option('--port', default=2889, type=int)
    app.add_option('-c', '--colors', default=False, action='store_true')
Example #19
0
def setup():
  LogOptions.set_stderr_log_level('NONE')

  app.add_option('--iface', default='eth0', type=str,
                 help='The interface to sniff on')
  app.add_option('--port', default=2889, type=int,
                 help='The ZAB port used by the leader')
  app.add_option('-c', '--colors', default=False, action='store_true',
                 help='Color each learner/leader stream differently')
  app.add_option('--dump-bad-packet', default=False, action='store_true',
                 help='Dump packets that cannot be deserialized')
  app.add_option('--include-pings', default=False, action='store_true',
                 help='Whether to include pings send from learners to the leader')
  app.add_option('--version', default=False, action='store_true')
Example #20
0
def setup():
    app.add_option("--iface",
                   dest="iface",
                   metavar="IFACE",
                   default="eth0",
                   help="interface to capture packets from")
    app.add_option("--http-port",
                   dest="http_port",
                   metavar="HTTPPORT",
                   type=int,
                   default=7070,
                   help="listen port for http endpoints")
    app.add_option("--http-address",
                   dest="http_addr",
                   metavar="HTTPADDR",
                   type=str,
                   default=socket.gethostname(),
                   help="listen address for http endpoints")
    app.add_option("--zookeeper-port",
                   type=int,
                   default=2181,
                   help="ZK's client port (from which to sniff)")
    app.add_option("--aggregation-depth",
                   dest="aggregation_depth",
                   type=int,
                   default=0,
                   help="aggregate paths up to a certain depth")
    app.add_option("--max-results",
                   dest="max_results",
                   type=int,
                   default=10,
                   help="top number of results to be exported")
    app.add_option("--refresh-time",
                   dest="refresh_time",
                   type=int,
                   default=0,
                   help="refresh time in the generated html")
    app.add_option("--niceness",
                   dest="niceness",
                   type=int,
                   default=0,
                   help="set the niceness")
    app.add_option(
        "--set-cpu-affinity",
        dest="cpu_affinity",
        metavar="CPU#[,CPU#]",
        type=str,
        default=None,
        help="A comma-separated list of CPU cores to pin this process to")
    app.add_option("--max-queued-requests",
                   type=int,
                   default=400000,
                   help="max queued requests")
    app.add_option("--max-queued-replies",
                   type=int,
                   default=400000,
                   help="max queued replies")
    app.add_option("--max-queued-events",
                   type=int,
                   default=400000,
                   help="max queued events")
Example #21
0
"""A Mesos-customized entry point to the thermos_observer webserver."""

import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help='The mesos root directory to search for Thermos executor sandboxes [default: %default]')


app.add_option(
    '--ip',
    dest='ip',
    type='string',
    default='0.0.0.0',
    help='The IP address the observer will bind to.')


app.add_option(
    '--port',
    dest='port',
Example #22
0
    print(traceback.format_exc(), file=sys.stderr)
    MesosExecutorDriver = None

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

_LOGGER_TYPES = ', '.join(LoggerMode.VALUES)

app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help=
    'Enable the ServerSet announcer for this executor.  Jobs must still activate using '
    'the Announcer configuration.')

app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')

app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
Example #23
0
from apache.thermos.common.ckpt import CheckpointDispatcher
from apache.thermos.common.constants import DEFAULT_CHECKPOINT_ROOT
from apache.thermos.common.options import add_binding_to, add_port_to
from apache.thermos.common.path import TaskPath
from apache.thermos.config.loader import ThermosConfigLoader, ThermosTaskWrapper
from apache.thermos.config.schema import Process, Resources, Task
from apache.thermos.core.helper import TaskRunnerHelper
from apache.thermos.core.runner import TaskRunner
from apache.thermos.monitoring.detector import TaskDetector
from apache.thermos.monitoring.garbage import DefaultCollector, TaskGarbageCollector
from apache.thermos.monitoring.monitor import TaskMonitor

from gen.apache.thermos.ttypes import ProcessState, RunnerCkpt, RunnerState, TaskState

app.add_option("--root", dest="root", metavar="PATH",
               default=DEFAULT_CHECKPOINT_ROOT,
               help="the thermos config root")


def set_keep(option, opt_str, value, parser):
  setattr(parser.values, option.dest, opt_str.startswith('--keep'))


def get_task_from_options(args, opts, **kw):
  loader = ThermosConfigLoader.load_json if opts.json else ThermosConfigLoader.load

  if len(args) != 1:
    app.error('Should specify precisely one config, instead got: %s' % args)

  tasks = loader(args[0], bindings=opts.bindings, **kw)
Example #24
0
def main(args, options):
    from pkg_resources import WorkingSet, Requirement, find_distributions

    if not options.site_dir:
        app.error('Must supply --site')

    distributions = list(find_distributions(options.site_dir))
    working_set = WorkingSet()
    for dist in distributions:
        working_set.add(dist)

    for arg in args:
        arg_req = Requirement.parse(arg)
        found_dist = working_set.find(arg_req)
        if not found_dist:
            print('Could not find %s!' % arg_req)
        out_zip = Distiller(found_dist).distill()
        print('Dumped %s => %s' % (arg_req, out_zip))


if HAS_APP:
    if __name__ == '__main__':
        app.add_option('--site',
                       dest='site_dir',
                       metavar='DIR',
                       default=None,
                       help='Directory to search for the requirement.')

    app.main()
Example #25
0
app.register_commands_from(admin, help_commands, maintenance)
add_verbosity_options()


def main():
  app.help()

try:
  from apache.aurora.kerberos.auth_module import KerberosAuthModule
  register_auth_module(KerberosAuthModule())
except ImportError:
  # Use default auth implementation if kerberos is not available.
  pass

LogOptions.set_stderr_log_level('INFO')
LogOptions.disable_disk_logging()
app.set_name('aurora-admin')
app.set_usage(generate_terse_usage())

app.add_option(
    '--bypass-leader-redirect',
    action='store_true',
    default=False,
    dest='bypass_leader_redirect',
    help='Bypass the scheduler\'s leader redirect filter')


def proxy_main():
  app.main()
Example #26
0
from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.cli.common import register_path_detector
from apache.thermos.cli.main import register_commands, register_options

register_commands(app)
register_options(app)


def register_mesos_root(_, __, value, ___):
    register_path_detector(MesosPathDetector(root=value))


app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    action='callback',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    callback=register_mesos_root,
    help=
    'The mesos root directory to search for Thermos executor sandboxes [default: %default]'
)

# register a default mesos root, since the callback will not be called if no --mesos-root specified.
register_path_detector(MesosPathDetector())

LogOptions.disable_disk_logging()
LogOptions.set_stderr_log_level('google:INFO')

app.main()
except ImportError:
  print(traceback.format_exc(), file=sys.stderr)
  MesosExecutorDriver = None


CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
    default='/aurora',
    help='The root of the tree into which ServerSets should be announced.  The paths will '
         'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.')

app.add_option(
    '--announcer-allow-custom-serverset-path',
    dest='announcer_allow_custom_serverset_path',
Example #28
0
    return filename


def main(args, options):
  from pkg_resources import WorkingSet, Requirement, find_distributions

  if not options.site_dir:
    app.error('Must supply --site')

  distributions = list(find_distributions(options.site_dir))
  working_set = WorkingSet()
  for dist in distributions:
    working_set.add(dist)

  for arg in args:
    arg_req = Requirement.parse(arg)
    found_dist = working_set.find(arg_req)
    if not found_dist:
      print('Could not find %s!' % arg_req)
    out_zip = Distiller(found_dist).distill()
    print('Dumped %s => %s' % (arg_req, out_zip))


if HAS_APP:
  if __name__ == '__main__':
    app.add_option('--site', dest='site_dir', metavar='DIR', default=None,
                   help='Directory to search for the requirement.')

  app.main()
Example #29
0

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

_LOGGER_DESTINATIONS = ', '.join(LoggerDestination.VALUES)
_LOGGER_MODES = ', '.join(LoggerMode.VALUES)


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
    default='/aurora',
    help='The root of the tree into which ServerSets should be announced.  The paths will '
         'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.')

app.add_option(
    '--announcer-allow-custom-serverset-path',
    dest='announcer_allow_custom_serverset_path',
Example #30
0
import twitter.common.app as app
import twitter.common.log as log

import gevent
from gevent.queue import Queue

from flask import Flask, flash, redirect, render_template, request, Response, url_for

from datetime import timedelta
import json
import os
import platform

app.set_name = "gatekeeper"
app.add_option("-p", "--port", type="int", default=5000)
config = HelperFunctions().read_config_from_yaml()

PLATFORM = platform.system()

GOOGLE_ADMIN_ACTIONS = list(
    k for k, v in config["actions"]["google_admin"].items() if v is True)
GOOGLE_GMAIL_ACTIONS = list(
    k for k, v in config["actions"]["google_gmail"].items() if v is True)
GOOGLE_CALENDAR_ACTIONS = list(
    k for k, v in config["actions"]["google_calendar"].items() if v is True)
GOOGLE_DRIVE_ACTIONS = list(
    k for k, v in config["actions"]["google_drive"].items() if v is True)
PAGERDUTY_ACTIONS = list(k for k, v in config["actions"]["pagerduty"].items()
                         if v is True)
DUO_ACTIONS = list(k for k, v in config["actions"]["duo"].items() if v is True)
from __future__ import print_function

import sys

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.http import HttpServer
from twitter.common.http.diagnostics import DiagnosticsEndpoints

from apache.thermos.common.path import TaskPath
from apache.thermos.observer.http.http_observer import BottleObserver
from apache.thermos.observer.task_observer import TaskObserver

app.add_option("--root",
               dest="root",
               metavar="DIR",
               default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
               help="root checkpoint directory for thermos task runners")

app.add_option("--port",
               dest="port",
               metavar="INT",
               default=1338,
               help="port number to listen on.")


def proxy_main():
    def main(args, opts):
        if args:
            print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)),
                  file=sys.stderr)
Example #32
0
def setup():
  app.add_option('--iface', default='eth0', type=str)
  app.add_option('--port', default=3888, type=int)
  app.add_option('-c', '--colors', default=False, action='store_true')
Example #33
0
try:
    from mesos.native import MesosExecutorDriver
except ImportError:
    print(traceback.format_exc(), file=sys.stderr)
    MesosExecutorDriver = None

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')

app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
    default='/aurora',
    help=
    'The root of the tree into which ServerSets should be announced.  The paths will '
    'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.')

app.add_option(
    '--announcer-allow-custom-serverset-path',
    dest='announcer_allow_custom_serverset_path',
Example #34
0
from collections import defaultdict

from rainman.codec import BEncoder

from twitter.common import app, log
from twitter.common.http import HttpServer
from twitter.common.quantity import Amount, Time


app.add_option(
    '--port',
    dest='port',
    default=8080,
    type=int,
    help='The port on which the torrent tracker should be run.')


class TrackerRequest(object):
  class Error(Exception): pass
  class MalformedRequestError(Error): pass

  REQUIRED_KEYS = frozenset([
    'info_hash',
    'peer_id',
    'port',
    'uploaded',
    'downloaded',
    'left',
  ])

  OPTIONAL_KEYS = set(['ip', 'event'])
Example #35
0
import re
import sys
import os.path
from functools import partial

from twitter.common import app, log

ROOT = os.path.join(os.path.dirname(__file__), '..')
sys.path.append(ROOT)

from lib.modules import spout, config, models, event_emitter_2
from lib.modules import sink


app.add_option("--listener", default="CarbonAsyncTcpSpout",
               help="Select the incoming metric connection interface")
app.add_option("--writer", default="RedisSink",
               help="Select the sink connection interface")
app.add_option("--config", help="Collector Config")

# Globals
EE = event_emitter_2.EventEmitter2()
CONFIG = None
WHITELIST = None
BLACKLIST = None

def setup(options):
    """Load the config file and add listeners for whitelisted metrics.

       :param options: app options
    """
def proxy_main():
    app.add_option('--api_host',
                   dest='api_host',
                   help='Host for the HTTP API server')

    app.add_option('--api_port',
                   dest='api_port',
                   type='int',
                   help='Port for the HTTP API server')

    app.add_option('--cluster',
                   dest='cluster_name',
                   help='Name of the MySQL cluster to create')

    app.add_option(
        '--password_file',
        dest='password_file',
        default=os.path.join(tempfile.gettempdir(), 'mysos',
                             'mysos_test_client', 'password_file'),
        help=
        "Path to the file for persisting the cluster password for testing purposes"
    )

    @app.command
    @app.command_option('--num_nodes',
                        dest='num_nodes',
                        type='int',
                        help='Number of nodes this cluster should have')
    @app.command_option('--backup_id',
                        dest='backup_id',
                        default=None,
                        help="The 'backup_id' to restore from")
    @app.command_option('--cluster_user',
                        dest='cluster_user',
                        help='MySQL user name the of cluster')
    @app.command_option(
        '--size',
        dest='size',
        help=
        "The size of instances in the cluster as a JSON dictionary of 'cpus', 'mem', 'disk'. "
        "'mem' and 'disk' are specified with data size units: kb, mb, gb, etc. If given 'None'"
        "then app defaults are used.")
    @app.command_option(
        '--cluster_password',
        dest='cluster_password',
        help=
        "The password used for accessing MySQL instances in the cluster as well as deleting "
        "the cluster from Mysos.")
    def create(args, options):
        validate_common_options(options)

        if not options.num_nodes:
            app.error("--num_nodes is required")

        if not options.cluster_user:
            app.error("--cluster_user is required")

        url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port,
                                            options.cluster_name)
        values = dict(
            num_nodes=int(options.num_nodes),
            cluster_user=options.cluster_user,
            size=options.size
            if options.size else '',  # 'urlencode()' doesn't accept None.
            backup_id=options.backup_id if options.backup_id else '',
            cluster_password=options.cluster_password
            if options.cluster_password else '')

        req = urllib2.Request(url, urllib.urlencode(values))
        try:
            response = urllib2.urlopen(req).read()
        except urllib2.HTTPError as e:
            log.error("POST request failed: %s, %s, %s" %
                      (e.code,
                       BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code],
                       e.read()))
            app.quit(1)

        try:
            result = json.loads(response)
            if not isinstance(result, dict):
                raise ValueError()
        except ValueError:
            log.error("Invalid response: %s" % response)
            app.quit(1)

        log.info("Cluster created. Cluster info: %s" % str(result))
        with open(options.password_file, 'w') as f:
            f.write(result["cluster_password"])

        log.info("Waiting for the master for this cluster to be elected...")
        master_endpoint = wait_for_master(
            result['cluster_url']).service_endpoint

        connection_str = "mysql://%s:%s@%s:%d/" % (
            options.cluster_user, result["cluster_password"],
            master_endpoint.host, master_endpoint.port)
        log.info("Connecting to the MySQL cluster master: %s" % connection_str)
        engine = create_engine(connection_str)

        for i in range(
                5
        ):  # Loop for 5 times/seconds to wait for the master to be promoted.
            try:
                # TODO(jyx): Test writing to the master and reading from the slave.
                result = engine.execute("SELECT 1;").scalar()
                assert 1 == int(
                    result), "Expecting result to be 1 but got %s" % result
                break
            except OperationalError:
                if i == 4:
                    raise
                log.debug("MySQL master not ready yet. Sleep for 1 second...")
                time.sleep(1)

        log.info("Cluster successfully started")

    @app.command
    def delete(args, options):
        validate_common_options(options)

        with open(options.password_file, 'r') as f:
            password = f.read().strip()
            if not password:
                app.error("Empty password file")

        url = 'http://%s:%s/clusters/%s' % (options.api_host, options.api_port,
                                            options.cluster_name)
        values = dict(password=password)

        req = urllib2.Request(url, urllib.urlencode(values))
        req.get_method = lambda: 'DELETE'

        try:
            response = urllib2.urlopen(req).read()
        except urllib2.HTTPError as e:
            log.error("DELETE request failed: %s, %s, %s" %
                      (e.code,
                       BaseHTTPServer.BaseHTTPRequestHandler.responses[e.code],
                       e.read()))
            app.quit(1)

        try:
            result = json.loads(response)
            if not isinstance(result, dict):
                raise ValueError()
        except ValueError:
            log.error("Invalid response: %s" % response)
            app.quit(1)

        log.info("Cluster deletion result: %s" % result)

        log.info("Waiting for the cluster to terminate...")
        wait_for_termination(result['cluster_url'])

        log.info("Cluster terminated/deleted")

    def validate_common_options(options):
        if not options.api_host:
            app.error("--api_host is required")

        if not options.api_port:
            app.error("--api_port is required")

        if not options.cluster_name:
            app.error("--cluster is required")

        if not options.password_file:
            app.error("--password_file is required")
        log.info("Using --password_file=%s" % options.password_file)
        safe_mkdir(os.path.dirname(options.password_file))

    def main(args, options):
        app.help()

    app.main()
Example #37
0
def setup():
    LogOptions.set_stderr_log_level('NONE')

    app.add_option(
        '--packet-filter',
        default='tcp',
        type=str,
        help=
        'pcap filter string. e.g. "tcp portrange 11221-32767" for JUnit tests')
    app.add_option('-c', '--colors', default=False, action='store_true')
    app.add_option('--dump-bad-packet', default=False, action='store_true')
    app.add_option('--include-pings',
                   default=False,
                   action='store_true',
                   help='Whether to include ZAB/ZK pings')
    app.add_option('--offline',
                   default=None,
                   type=str,
                   help='offline mode with a pcap file')
    app.add_option('--version', default=False, action='store_true')
Example #38
0
import sys
import traceback

from twitter.common import app, log

from apache.thermos.common.options import add_port_to
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.common.statuses import (INTERNAL_ERROR, INVALID_TASK,
                                            TERMINAL_TASK, UNKNOWN_ERROR,
                                            UNKNOWN_USER)
from apache.thermos.config.loader import ThermosConfigLoader
from apache.thermos.core.process import Process
from apache.thermos.core.runner import TaskRunner

app.add_option("--thermos_json",
               dest="thermos_json",
               default=None,
               help="read a thermos Task from a serialized json blob")

app.add_option("--sandbox",
               dest="sandbox",
               metavar="PATH",
               default=None,
               help="the sandbox in which this task should run")

app.add_option("--checkpoint_root",
               dest="checkpoint_root",
               metavar="PATH",
               default=None,
               help="the path where we will store checkpoints")

app.add_option(
from apache.aurora.executor.aurora_executor import AuroraExecutor
from apache.aurora.executor.common.announcer import DefaultAnnouncerCheckerProvider
from apache.aurora.executor.common.executor_timeout import ExecutorTimeout
from apache.aurora.executor.common.health_checker import HealthCheckerProvider
from apache.aurora.executor.thermos_task_runner import DefaultThermosTaskRunnerProvider

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir('.')


app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help='Enable the ServerSet announcer for this executor.  Jobs must still activate using '
         'the Announcer configuration.')


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
Example #40
0
      LogOptions.set_stdout_log_level(value)
    except LogOptionsException, e:
      raise optparse.OptionValueError('Failed to parse option: %s' % e)
    parser.values.twitter_common_log_stdout_log_level = value

_LOGGING_HELP = \
"""The level at which to log to %%s [default: %%%%default].
Takes either LEVEL or scheme:LEVEL, where LEVEL is one
of %s and scheme is one of %s.
""" % (repr(LogOptions._LOG_LEVELS.keys()), repr(LogOptions._LOG_SCHEMES))

if HAVE_APP:
  app.add_option('--log_to_stdout',
              callback=LogOptions._stdout_options_callback,
              default=_DEFAULT_LOG_OPTS.twitter_common_log_stdout_log_level,
              type='string',
              action='callback',
              metavar='[scheme:]LEVEL',
              dest='twitter_common_log_stdout_log_level',
              help=_LOGGING_HELP % 'stdout')

  app.add_option('--log_to_disk',
              callback=LogOptions._disk_options_callback,
              default=_DEFAULT_LOG_OPTS.twitter_common_log_disk_log_level,
              type='string',
              action='callback',
              metavar='[scheme:]LEVEL',
              dest='twitter_common_log_disk_log_level',
              help=_LOGGING_HELP % 'disk')

  app.add_option('--log_dir',
              type='string',
Example #41
0
def proxy_main():
  app.add_option(
      '--port',
      dest='api_port',
      type='int',
      default=None,
      help='Port for the HTTP API server')

  app.add_option(
      '--mesos_master',
      dest='mesos_master',
      default=None,
      help='Mesos master address. It can be a ZooKeeper URL through which the master can be '
           'detected')

  app.add_option(
      '--framework_user',
      dest='framework_user',
      help='The Unix user that Mysos executor runs as')

  app.add_option(
      '--framework_role',
      dest='framework_role',
      default='*',
      help="The role that Mysos framework runs as. If set, Mysos only uses Mesos pool resources "
           "with that role. The default value '*' is what Mesos considers as the default role.\n"
           "NOTE: Mesos master needs to be configured to allow the specified role. See its --roles "
           "flag")

  app.add_option(
      '--executor_uri',
      dest='executor_uri',
      default=None,
      help='URI for the Mysos executor package')

  app.add_option(
      '--executor_cmd',
      dest='executor_cmd',
      default=None,
      help='Command to execute the executor package')

  app.add_option(
      '--executor_environ',
      dest='executor_environ',
      default=None,
      help="Environment variables for the executors (and the tasks) as a list of dicts keyed by "
           "{name, value} in JSON. Note that these variables don't affect Mesos slave components "
           "such as the fetcher")

  app.add_option(
      '--zk_url',
      dest='zk_url',
      default=None,
      help='ZooKeeper URL for various Mysos operations, in the form of '
           '"zk://*****:*****@servers/path". The sub-directory <zk_url>/discover is used for '
           'communicating MySQL cluster information between Mysos scheduler and executors')

  # TODO(jyx): This could also be made a per-cluster configuration.
  app.add_option(
      '--election_timeout',
      dest='election_timeout',
      default='60s',
      help='The amount of time the scheduler waits for all slaves to respond during a MySQL master '
           'election, e.g., 60s. After the timeout the master is elected from only the slaves that '
           'have responded')

  app.add_option(
      '--admin_keypath',
      dest='admin_keypath',
      default=None,
      help='The path to the key file with MySQL admin credentials on Mesos slaves')

  app.add_option(
      '--work_dir',
      dest='work_dir',
      default=os.path.join(tempfile.gettempdir(), 'mysos'),
      help="Directory path to place Mysos work directories, e.g., web assets, state files if "
           "--state_storage=local. Default to a system temp directory.")

  app.add_option(
      '--state_storage',
      dest='state_storage',
      default='zk',
      help="Mechanism to persist scheduler state. Available options are 'zk' and 'local'. If 'zk' "
           "is chosen, the scheduler state is stored under <zk_url>/state; see --zk_url. Otherwise "
           "'local' is chosen and the state is persisted under <work_dir>/state; see --work_dir")

  app.add_option(
      '--framework_failover_timeout',
      dest='framework_failover_timeout',
      default='14d',
      help='Time after which Mysos framework is considered deleted. This implies losing all tasks. '
           'SHOULD BE VERY HIGH')

  # TODO(jyx): Flags like this are generally optional but specific executor implementations may
  # require them. Consider adding validators that can be plugged in so configuration errors can be
  # caught in the scheduler.
  app.add_option(
      '--installer_args',
      dest='installer_args',
      default=None,
      help='Arguments for MySQL installer directly passed along to and parsed by the installer. '
           'e.g., a serialized JSON string'
  )

  app.add_option(
      '--backup_store_args',
      dest='backup_store_args',
      default=None,
      help="Arguments for the store for MySQL backups. Its use and format are defined by the "
           "backup store implementation. e.g., It can be a serialized JSON string"
  )

  app.add_option(
      '--framework_authentication_file',
      dest='framework_authentication_file',
      default=None,
      help="Path to the key file for authenticating the framework against Mesos master. Framework "
           "will fail to register with Mesos if authentication is required by Mesos and this "
           "option is not provided"
  )

  def main(args, options):
    log.info("Options in use: %s", options)

    if not options.api_port:
      app.error('Must specify --port')

    if not options.mesos_master:
      app.error('Must specify --mesos_master')

    if not options.framework_user:
      app.error('Must specify --framework_user')

    if not options.executor_uri:
      app.error('Must specify --executor_uri')

    if not options.executor_cmd:
      app.error('Must specify --executor_cmd')

    if not options.zk_url:
      app.error('Must specify --zk_url')

    if not options.admin_keypath:
      app.error('Must specify --admin_keypath')

    try:
      election_timeout = parse_time(options.election_timeout)
      framework_failover_timeout = parse_time(options.framework_failover_timeout)
    except InvalidTime as e:
      app.error(e.message)

    try:
      _, zk_servers, zk_root = zookeeper.parse(options.zk_url)
    except Exception as e:
      app.error("Invalid --zk_url: %s" % e.message)

    web_assets_dir = os.path.join(options.work_dir, "web")
    pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH)
    log.info("Extracted web assets into %s" % options.work_dir)

    fw_principal = None
    fw_secret = None
    if options.framework_authentication_file:
      try:
        with open(options.framework_authentication_file, "r") as f:
          cred = yaml.load(f)
        fw_principal = cred["principal"]
        fw_secret = cred["secret"]
        log.info("Loaded credential (principal=%s) for framework authentication" % fw_principal)
      except IOError as e:
        app.error("Unable to read the framework authentication key file: %s" % e)
      except (KeyError, yaml.YAMLError) as e:
        app.error("Invalid framework authentication key file format %s" % e)

    log.info("Starting Mysos scheduler")

    kazoo = KazooClient(zk_servers)
    kazoo.start()

    if options.state_storage == 'zk':
      log.info("Using ZooKeeper (path: %s) for state storage" % zk_root)
      state_provider = ZooKeeperStateProvider(kazoo, zk_root)
    else:
      log.info("Using local disk for state storage")
      state_provider = LocalStateProvider(options.work_dir)

    try:
      state = state_provider.load_scheduler_state()
    except StateProvider.Error as e:
      app.error(e.message)

    if state:
      log.info("Successfully restored scheduler state")
      framework_info = state.framework_info
      if framework_info.HasField('id'):
        log.info("Recovered scheduler's FrameworkID is %s" % framework_info.id.value)
    else:
      log.info("No scheduler state to restore")
      framework_info = FrameworkInfo(
          user=options.framework_user,
          name=FRAMEWORK_NAME,
          checkpoint=True,
          failover_timeout=framework_failover_timeout.as_(Time.SECONDS),
          role=options.framework_role)
      if fw_principal:
        framework_info.principal = fw_principal
      state = Scheduler(framework_info)
      state_provider.dump_scheduler_state(state)

    scheduler = MysosScheduler(
        state,
        state_provider,
        options.framework_user,
        options.executor_uri,
        options.executor_cmd,
        kazoo,
        options.zk_url,
        election_timeout,
        options.admin_keypath,
        installer_args=options.installer_args,
        backup_store_args=options.backup_store_args,
        executor_environ=options.executor_environ,
        framework_role=options.framework_role)

    if fw_principal and fw_secret:
      cred = Credential(principal=fw_principal, secret=fw_secret)
      scheduler_driver = mesos.native.MesosSchedulerDriver(
          scheduler,
          framework_info,
          options.mesos_master,
          cred)
    else:
      scheduler_driver = mesos.native.MesosSchedulerDriver(
          scheduler,
          framework_info,
          options.mesos_master)

    scheduler_driver.start()

    server = HttpServer()
    server.mount_routes(MysosServer(scheduler, web_assets_dir))

    et = ExceptionalThread(
        target=server.run, args=('0.0.0.0', options.api_port, 'cherrypy'))
    et.daemon = True
    et.start()

    try:
      # Wait for the scheduler to stop.
      # The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the
      # process with SIGINT.
      while not scheduler.stopped.wait(timeout=0.5):
        pass
    except KeyboardInterrupt:
      log.info('Interrupted, exiting.')
    else:
      log.info('Scheduler exited.')

    app.shutdown(1)  # Mysos scheduler is supposed to be long-running thus the use of exit status 1.

  app.main()
Example #42
0
import os
import sys
import pprint
import time

from gen.twitter.thermos.ttypes import RunnerState, RunnerCkpt, TaskState

from twitter.common import app
from twitter.common.recordio import RecordIO, ThriftRecordReader
from twitter.thermos.common.ckpt import CheckpointDispatcher

app.add_option("--checkpoint", dest = "ckpt", metavar = "CKPT",
               help = "read checkpoint from CKPT")
app.add_option("--assemble", dest = "assemble", metavar = "CKPT", default=True,
               help = "read checkpoint from CKPT")

def main(args):
  values = app.get_options()

  if len(args) > 0:
    print >> sys.stderr, "ERROR: unrecognized arguments: %s\n" % (" ".join(args))
    app.help()
    sys.exit(1)

  if not values.ckpt:
    print >> sys.stderr, "ERROR: must supply --checkpoint"
    app.help()
    sys.exit(1)

  fp = file(values.ckpt, "r")
  rr = ThriftRecordReader(fp, RunnerCkpt)
Example #43
0
import socket
import sys
import time

from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.http import HttpServer
from twitter.common.http.diagnostics import DiagnosticsEndpoints
from twitter.thermos.common.path import TaskPath
from twitter.thermos.observer.task_observer import TaskObserver
from twitter.thermos.observer.http.http_observer import BottleObserver


app.add_option("--root",
               dest="root",
               metavar="DIR",
               default=TaskPath.DEFAULT_CHECKPOINT_ROOT,
               help="root checkpoint directory for thermos task runners")


app.add_option("--port",
               dest="port",
               metavar="INT",
               default=1338,
               help="port number to listen on.")


def main(args, opts):
  if args:
    print("ERROR: unrecognized arguments: %s\n" % (" ".join(args)), file=sys.stderr)
    app.help()
from __future__ import print_function

import re

from twitter.common import app

from .common import Nit, PythonFile
from .iterators import git_iterator, path_iterator
from .plugins import list_plugins


app.add_option(
  '-p',
  action='append',
  type='str',
  default=[],
  dest='plugins',
  help='Explicitly list plugins to enable.')


app.add_option(
  '-n',
  action='append',
  type='str',
  default=[],
  dest='skip_plugins',
  help='Explicitly list plugins to disable.')


app.add_option(
Example #45
0
                               (self._host, self._port))

        session = NailgunSession(sock, self._ins, self._out, self._err)
        try:
            return session.execute(self._work_dir, main_class, *args,
                                   **environment)
        except socket.error as e:
            raise NailgunError('Problem contacting nailgun server %s:%d %s' %
                               (self._host, self._port, e))
        finally:
            sock.close()


app.add_option('--nailgun-version',
               dest='just_version',
               default=False,
               action='store_true',
               help='print product version and exit')

app.add_option('--nailgun-showversion',
               dest='show_version',
               default=False,
               action='store_true',
               help='print product version and continue')

app.add_option(
    '--nailgun-server',
    dest='ng_host',
    metavar='HOST',
    default=DEFAULT_NG_HOST,
    help='to specify the address of the nailgun server (default is %default)')
Example #46
0
import sys
import traceback

from twitter.common import app, log

from apache.thermos.common.options import add_port_to
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.common.statuses import (INTERNAL_ERROR, INVALID_TASK,
                                            TERMINAL_TASK, UNKNOWN_ERROR,
                                            UNKNOWN_USER)
from apache.thermos.config.loader import ThermosConfigLoader
from apache.thermos.core.process import Process
from apache.thermos.core.runner import TaskRunner

app.add_option("--thermos_json",
               dest="thermos_json",
               default=None,
               help="read a thermos Task from a serialized json blob")

app.add_option(
    "--sandbox",
    dest="sandbox",
    metavar="PATH",
    default=None,
    help=
    "The path on the host filesystem to the sandbox in which this task should run."
)

app.add_option(
    '--container_sandbox',
    dest='container_sandbox',
    type=str,
import functools
import os
import signal
import sys
import traceback

from twitter.common import app, log
from apache.thermos.common.options import add_port_to
from apache.thermos.common.planner import TaskPlanner
from apache.thermos.config.loader import ThermosConfigLoader
from apache.thermos.core.runner import TaskRunner


app.add_option(
    "--thermos_json",
    dest="thermos_json",
    default=None,
    help="read a thermos Task from a serialized json blob")


app.add_option(
    "--sandbox",
    dest="sandbox",
    metavar="PATH",
    default=None,
    help="the sandbox in which this task should run")


app.add_option(
     "--checkpoint_root",
     dest="checkpoint_root",
Example #48
0
def proxy_main():
    app.add_option('--port',
                   dest='api_port',
                   type='int',
                   default=None,
                   help='Port for the HTTP API server')

    app.add_option(
        '--mesos_master',
        dest='mesos_master',
        default=None,
        help=
        'Mesos master address. It can be a ZooKeeper URL through which the master can be '
        'detected')

    app.add_option('--framework_user',
                   dest='framework_user',
                   help='The Unix user that Mysos executor runs as')

    app.add_option(
        '--framework_role',
        dest='framework_role',
        default='*',
        help=
        "The role that Mysos framework runs as. If set, Mysos only uses Mesos pool resources "
        "with that role. The default value '*' is what Mesos considers as the default role.\n"
        "NOTE: Mesos master needs to be configured to allow the specified role. See its --roles "
        "flag")

    app.add_option('--executor_uri',
                   dest='executor_uri',
                   default=None,
                   help='URI for the Mysos executor package')

    app.add_option('--executor_cmd',
                   dest='executor_cmd',
                   default=None,
                   help='Command to execute the executor package')

    app.add_option(
        '--executor_environ',
        dest='executor_environ',
        default=None,
        help=
        "Environment variables for the executors (and the tasks) as a list of dicts keyed by "
        "{name, value} in JSON. Note that these variables don't affect Mesos slave components "
        "such as the fetcher")

    app.add_option(
        '--zk_url',
        dest='zk_url',
        default=None,
        help='ZooKeeper URL for various Mysos operations, in the form of '
        '"zk://*****:*****@servers/path". The sub-directory <zk_url>/discover is used for '
        'communicating MySQL cluster information between Mysos scheduler and executors'
    )

    # TODO(jyx): This could also be made a per-cluster configuration.
    app.add_option(
        '--election_timeout',
        dest='election_timeout',
        default='60s',
        help=
        'The amount of time the scheduler waits for all slaves to respond during a MySQL master '
        'election, e.g., 60s. After the timeout the master is elected from only the slaves that '
        'have responded')

    app.add_option(
        '--admin_keypath',
        dest='admin_keypath',
        default=None,
        help=
        'The path to the key file with MySQL admin credentials on Mesos slaves'
    )

    app.add_option(
        '--work_dir',
        dest='work_dir',
        default=os.path.join(tempfile.gettempdir(), 'mysos'),
        help=
        "Directory path to place Mysos work directories, e.g., web assets, state files if "
        "--state_storage=local. Default to a system temp directory.")

    app.add_option(
        '--state_storage',
        dest='state_storage',
        default='zk',
        help=
        "Mechanism to persist scheduler state. Available options are 'zk' and 'local'. If 'zk' "
        "is chosen, the scheduler state is stored under <zk_url>/state; see --zk_url. Otherwise "
        "'local' is chosen and the state is persisted under <work_dir>/state; see --work_dir"
    )

    app.add_option(
        '--framework_failover_timeout',
        dest='framework_failover_timeout',
        default='14d',
        help=
        'Time after which Mysos framework is considered deleted. This implies losing all tasks. '
        'SHOULD BE VERY HIGH')

    # TODO(jyx): Flags like this are generally optional but specific executor implementations may
    # require them. Consider adding validators that can be plugged in so configuration errors can be
    # caught in the scheduler.
    app.add_option(
        '--installer_args',
        dest='installer_args',
        default=None,
        help=
        'Arguments for MySQL installer directly passed along to and parsed by the installer. '
        'e.g., a serialized JSON string')

    app.add_option(
        '--backup_store_args',
        dest='backup_store_args',
        default=None,
        help=
        "Arguments for the store for MySQL backups. Its use and format are defined by the "
        "backup store implementation. e.g., It can be a serialized JSON string"
    )

    app.add_option(
        '--framework_authentication_file',
        dest='framework_authentication_file',
        default=None,
        help=
        "Path to the key file for authenticating the framework against Mesos master. Framework "
        "will fail to register with Mesos if authentication is required by Mesos and this "
        "option is not provided")

    def main(args, options):
        log.info("Options in use: %s", options)

        if not options.api_port:
            app.error('Must specify --port')

        if not options.mesos_master:
            app.error('Must specify --mesos_master')

        if not options.framework_user:
            app.error('Must specify --framework_user')

        if not options.executor_uri:
            app.error('Must specify --executor_uri')

        if not options.executor_cmd:
            app.error('Must specify --executor_cmd')

        if not options.zk_url:
            app.error('Must specify --zk_url')

        if not options.admin_keypath:
            app.error('Must specify --admin_keypath')

        try:
            election_timeout = parse_time(options.election_timeout)
            framework_failover_timeout = parse_time(
                options.framework_failover_timeout)
        except InvalidTime as e:
            app.error(e.message)

        try:
            _, zk_servers, zk_root = zookeeper.parse(options.zk_url)
        except Exception as e:
            app.error("Invalid --zk_url: %s" % e.message)

        web_assets_dir = os.path.join(options.work_dir, "web")
        pkgutil.unpack_assets(web_assets_dir, MYSOS_MODULE, ASSET_RELPATH)
        log.info("Extracted web assets into %s" % options.work_dir)

        fw_principal = None
        fw_secret = None
        if options.framework_authentication_file:
            try:
                with open(options.framework_authentication_file, "r") as f:
                    cred = yaml.load(f)
                fw_principal = cred["principal"]
                fw_secret = cred["secret"]
                log.info(
                    "Loaded credential (principal=%s) for framework authentication"
                    % fw_principal)
            except IOError as e:
                app.error(
                    "Unable to read the framework authentication key file: %s"
                    % e)
            except (KeyError, yaml.YAMLError) as e:
                app.error(
                    "Invalid framework authentication key file format %s" % e)

        log.info("Starting Mysos scheduler")

        kazoo = KazooClient(zk_servers)
        kazoo.start()

        if options.state_storage == 'zk':
            log.info("Using ZooKeeper (path: %s) for state storage" % zk_root)
            state_provider = ZooKeeperStateProvider(kazoo, zk_root)
        else:
            log.info("Using local disk for state storage")
            state_provider = LocalStateProvider(options.work_dir)

        try:
            state = state_provider.load_scheduler_state()
        except StateProvider.Error as e:
            app.error(e.message)

        if state:
            log.info("Successfully restored scheduler state")
            framework_info = state.framework_info
            if framework_info.HasField('id'):
                log.info("Recovered scheduler's FrameworkID is %s" %
                         framework_info.id.value)
        else:
            log.info("No scheduler state to restore")
            framework_info = FrameworkInfo(
                user=options.framework_user,
                name=FRAMEWORK_NAME,
                checkpoint=True,
                failover_timeout=framework_failover_timeout.as_(Time.SECONDS),
                role=options.framework_role)
            if fw_principal:
                framework_info.principal = fw_principal
            state = Scheduler(framework_info)
            state_provider.dump_scheduler_state(state)

        scheduler = MysosScheduler(state,
                                   state_provider,
                                   options.framework_user,
                                   options.executor_uri,
                                   options.executor_cmd,
                                   kazoo,
                                   options.zk_url,
                                   election_timeout,
                                   options.admin_keypath,
                                   installer_args=options.installer_args,
                                   backup_store_args=options.backup_store_args,
                                   executor_environ=options.executor_environ,
                                   framework_role=options.framework_role)

        if fw_principal and fw_secret:
            cred = Credential(principal=fw_principal, secret=fw_secret)
            scheduler_driver = mesos.native.MesosSchedulerDriver(
                scheduler, framework_info, options.mesos_master, cred)
        else:
            scheduler_driver = mesos.native.MesosSchedulerDriver(
                scheduler, framework_info, options.mesos_master)

        scheduler_driver.start()

        server = HttpServer()
        server.mount_routes(MysosServer(scheduler, web_assets_dir))

        et = ExceptionalThread(target=server.run,
                               args=('0.0.0.0', options.api_port, 'cherrypy'))
        et.daemon = True
        et.start()

        try:
            # Wait for the scheduler to stop.
            # The use of 'stopped' event instead of scheduler_driver.join() is necessary to stop the
            # process with SIGINT.
            while not scheduler.stopped.wait(timeout=0.5):
                pass
        except KeyboardInterrupt:
            log.info('Interrupted, exiting.')
        else:
            log.info('Scheduler exited.')

        app.shutdown(
            1
        )  # Mysos scheduler is supposed to be long-running thus the use of exit status 1.

    app.main()
Example #49
0
    sock = self.try_connect()
    if not sock:
      raise NailgunError('Problem connecting to nailgun server %s:%d' % (self._host, self._port))

    session = NailgunSession(sock, self._ins, self._out, self._err)
    try:
      return session.execute(self._work_dir, main_class, *args, **environment)
    except socket.error as e:
      raise NailgunError('Problem contacting nailgun server %s:%d %s' % (self._host, self._port, e))
    finally:
      sock.close()


app.add_option('--nailgun-version',
               dest='just_version',
               default=False,
               action='store_true',
               help='print product version and exit')

app.add_option('--nailgun-showversion',
               dest='show_version',
               default=False,
               action='store_true',
               help='print product version and continue')

app.add_option('--nailgun-server',
               dest='ng_host',
               metavar='HOST',
               default=DEFAULT_NG_HOST,
               help='to specify the address of the nailgun server (default is %default)')
Example #50
0
from twitter.common import app
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.common.excepthook import ExceptionTerminationHandler
from apache.thermos.monitoring.disk import DiskCollectorSettings
from apache.thermos.monitoring.resource import TaskResourceMonitor
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help=
    'The mesos root directory to search for Thermos executor sandboxes [default: %default]'
)

app.add_option('--ip',
               dest='ip',
               type='string',
               default='0.0.0.0',
               help='The IP address the observer will bind to.')

app.add_option('--port',
               dest='port',
               type='int',
               default=1338,
               help='The port on which the observer should listen.')
Example #51
0
import time

from twitter.common import app, log
from twitter.common.exceptions import ExceptionalThread
from twitter.common.log.options import LogOptions
from twitter.common.quantity import Amount, Time

from apache.aurora.executor.common.path_detector import MesosPathDetector
from apache.thermos.monitoring.resource import TaskResourceMonitor
from apache.thermos.observer.http.configure import configure_server
from apache.thermos.observer.task_observer import TaskObserver

app.add_option(
    '--mesos-root',
    dest='mesos_root',
    type='string',
    default=MesosPathDetector.DEFAULT_MESOS_ROOT,
    help=
    'The mesos root directory to search for Thermos executor sandboxes [default: %default]'
)

app.add_option('--ip',
               dest='ip',
               type='string',
               default='0.0.0.0',
               help='The IP address the observer will bind to.')

app.add_option('--port',
               dest='port',
               type='int',
               default=1338,
               help='The port on which the observer should listen.')
Example #52
0
app.register_commands_from(admin, help_commands, maintenance)
add_verbosity_options()


def main():
    app.help()


try:
    from apache.aurora.kerberos.auth_module import KerberosAuthModule
    register_auth_module(KerberosAuthModule())
except ImportError:
    # Use default auth implementation if kerberos is not available.
    pass

LogOptions.set_stderr_log_level('INFO')
LogOptions.disable_disk_logging()
app.set_name('aurora-admin')
app.set_usage(generate_terse_usage())

app.add_option('--bypass-leader-redirect',
               action='store_true',
               default=False,
               dest='bypass_leader_redirect',
               help='Bypass the scheduler\'s leader redirect filter')


def proxy_main():
    app.main()
    UserOverrideThermosTaskRunnerProvider
)
from apache.thermos.common.path import TaskPath

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)


app.add_option(
    '--announcer-enable',
    dest='announcer_enable',
    action='store_true',
    default=False,
    help='Enable the ServerSet announcer for this executor.  Jobs must still activate using '
         'the Announcer configuration.')


app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')


app.add_option(
    '--announcer-serverset-path',
Example #54
0
from jenkinsapi.exceptions import UnknownJob

import os
import sys

from twitter.common import app
from twitter.common import log
from twitter.common import dirutil

from twitter.common.log.options import LogOptions

app.set_usage(
    "jenkins --[server|outdir|config]  [--job=[jobname]] [create|delete|show|enable|disable]"
)
app.add_option("--server",
               default="ci.makewhat.is:8080",
               dest="server",
               help="jenkins server")
app.add_option("--job", default=None, dest="job", help="job name")
app.add_option("--config", default=None, dest="config", help="config path")
app.add_option("--config_dir",
               default=None,
               dest="config_dir",
               help="config directory")
app.add_option("--outdir",
               default=None,
               dest="outdir",
               help="config dump output path")
app.add_option("--verbose",
               default=None,
               dest="verbose",
               help="print more things")
Example #55
0
def configure_app(app):
    """ Register the application's options, set usage, and configure submodules. """

    app.set_name('starsystem')

    app.set_usage("{} [opts]\nOptions marked with * are required.".format(app.name()))

    app.add_option('-i', '--uri', dest='subsonic_uri',
                   help='* URI of the Subsonic server.')
    app.add_option('-u', '--user', dest='username',
                   help='* Username on the specified Subsonic server.')
    app.add_option('-t', '--token', dest='token',
                   help='* API token for the given username/salt combination\n'
                        'See: http://www.subsonic.org/pages/api.jsp')
    app.add_option('-s', '--salt', dest='salt',
                   help='* Salt used to generate the API token.')
    app.add_option('-p', '--path', dest='download_path',
                   help='* Path to the directory whither songs will be downloaded.')
    app.add_option('-S', '--since', dest='since', type='date',
                   help='Collect all songs since the specified date.')
    app.add_option('-I', '--insecure', dest='insecure', default=False, action="store_true",
                   help='Don\'t verify SSL certificates. Verification is enabled by default.')
    app.add_option('-g', '--gen-token-interactive', dest='gen_token', default=False,
                   action="store_true", help='Generate an API token interactively.')
    app.add_option('-v', '--debug', dest='debug', default=False,
                   action="store_true", help='Enable debug output.')

    app.set_option('twitter_common_log_disk_log_level', 'NONE', force=True)
try:
    from mesos.executor import MesosExecutorDriver
except ImportError:
    print(traceback.format_exc(), file=sys.stderr)
    MesosExecutorDriver = None

CWD = os.environ.get('MESOS_SANDBOX', '.')

app.configure(debug=True)
LogOptions.set_simple(True)
LogOptions.set_disk_log_level('DEBUG')
LogOptions.set_log_dir(CWD)

app.add_option(
    '--announcer-ensemble',
    dest='announcer_ensemble',
    type=str,
    default=None,
    help='The ensemble to which the Announcer should register ServerSets.')

app.add_option(
    '--announcer-serverset-path',
    dest='announcer_serverset_path',
    type=str,
    default='/aurora',
    help=
    'The root of the tree into which ServerSets should be announced.  The paths will '
    'be of the form $ROOT/$ROLE/$ENVIRONMENT/$JOBNAME.')

app.add_option(
    '--announcer-allow-custom-serverset-path',
    dest='announcer_allow_custom_serverset_path',