Exemple #1
0
 def test_load_cli_plugin(self):
   cli_kwargs = ['a=1', 'b=2']
   instance = load_cli_plugin('__builtin__.dict', cli_kwargs)
   self.assertEquals(instance['a'], '1')
   self.assertEquals(instance['b'], '2')
   extra_kwargs = {'c': '3'}
   instance = load_cli_plugin('__builtin__.dict', cli_kwargs, extra_kwargs)
   self.assertEquals(instance['a'], '1')
   self.assertEquals(instance['b'], '2')
   self.assertEquals(instance['c'], '3')
Exemple #2
0
def _setup_metrics(metric_publisher_class, metric_publisher_args):
  """
  Sets up a metrics publisher based on command line args.

  Args:
    metric_publisher_class
    metric_publisher_args
  """
  try:
    publisher = load_cli_plugin(metric_publisher_class,
                                metric_publisher_args)
    add_publisher(publisher)
  except Exception:
    logger.exception('Metrics failure.')
    raise
Exemple #3
0
def _setup_metrics(metric_publisher_class, metric_publisher_args):
    """
  Sets up a metrics publisher based on command line args.

  Args:
    metric_publisher_class
    metric_publisher_args
  """
    try:
        publisher = load_cli_plugin(metric_publisher_class,
                                    metric_publisher_args)
        add_publisher(publisher)
    except Exception:
        logger.exception('Metrics failure.')
        raise
Exemple #4
0
def synchronize(registration_source,
                registration_class,
                registration_arg=_DEFAULT_REGISTRATION_KWARGS,
                write=False):
  """Add and remove Aurproxy task instances from upstream services.

  Intended to be run by administrators or to be called automatically as a cron
  as a supplement to / safety net for the pluggable in-task registration and
  deregistration Aurproxy events that are driven by Aurora lifecycle events.

  Args:
    registration_source - JSON string - Source configuration.
      Format:
        {'registration_class': 'python.class.path',
         'arg1': 'val1',
         'arg2': 'val2'}
      Example:
        '{"source_class": "aurproxy.sources.AuroraSource"
          "announcer_serverset_path": "/aurora/",
          "zk_servers": "0.zk.mycluster.mydomain.com:2181,
                         1.zk.mycluster.mydomain.com:2181,
                         2.zk.mycluster.mydomain.com:2181,
          "environment": "devel",
          "job": "proxytest",
          "role": "proxy",
          "endpoint": "http"}
      See source class definition for valid kwargs.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    write - bool - Whether to apply the changes.
  """
  try:
    source_dict = json.loads(registration_source)
  except (TypeError, ValueError):
    raise commandr.CommandrUsageError(
      'Invalid JSON configuration specified via --registration_source',
    )

  source = load_klass_plugin(source_dict,
                             klass_field_name='source_class')
  extra_kwargs = {'source': source}
  registerer = load_cli_plugin(registration_class,
                               registration_arg,
                               extra_kwargs=extra_kwargs)
  registerer.synchronize(write)
Exemple #5
0
def synchronize(registration_source,
                registration_class,
                registration_arg=_DEFAULT_REGISTRATION_KWARGS,
                write=False):
    """Add and remove Aurproxy task instances from upstream services.

  Intended to be run by administrators or to be called automatically as a cron
  as a supplement to / safety net for the pluggable in-task registration and
  deregistration Aurproxy events that are driven by Aurora lifecycle events.

  Args:
    registration_source - JSON string - Source configuration.
      Format:
        {'registration_class': 'python.class.path',
         'arg1': 'val1',
         'arg2': 'val2'}
      Example:
        '{"source_class": "aurproxy.sources.AuroraSource"
          "announcer_serverset_path": "/aurora/",
          "zk_servers": "0.zk.mycluster.mydomain.com:2181,
                         1.zk.mycluster.mydomain.com:2181,
                         2.zk.mycluster.mydomain.com:2181,
          "environment": "devel",
          "job": "proxytest",
          "role": "proxy",
          "endpoint": "http"}
      See source class definition for valid kwargs.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    write - bool - Whether to apply the changes.
  """
    try:
        source_dict = json.loads(registration_source)
    except (TypeError, ValueError):
        raise commandr.CommandrUsageError(
            'Invalid JSON configuration specified via --registration_source', )

    source = load_klass_plugin(source_dict, klass_field_name='source_class')
    extra_kwargs = {'source': source}
    registerer = load_cli_plugin(registration_class,
                                 registration_arg,
                                 extra_kwargs=extra_kwargs)
    registerer.synchronize(write)
Exemple #6
0
def run(management_port,
        config,
        backend=_DEFAULT_BACKEND,
        update_period=_DEFAULT_UPDATE_PERIOD,
        max_update_frequency=_DEFAULT_MAX_UPDATE_FREQUENCY,
        weight_adjustment_delay_seconds=_DEFAULT_WEIGHT_ADJUSTMENT_DELAY_SEC,
        registration_class=_DEFAULT_REGISTRATION_CLASS,
        registration_arg=_DEFAULT_REGISTRATION_KWARGS,
        metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
        metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
        mirror_source=_DEFAULT_MIRROR_SOURCE,
        mirror_ports=_DEFAULT_MIRROR_PORTS,
        mirror_max_qps=_DEFAULT_MIRROR_MAX_QPS,
        mirror_max_update_frequency=_DEFAULT_MIRROR_MAX_UPDATE_FREQUENCY,
        mirror_pid_path=_MIRROR_PID_PATH,
        sentry_dsn=None,
        setup=False):
  """Run the Aurproxy load balancer manager.

  Args:
    management_port - int - port for the manager application to listen on for
      Aurora lifecycle queries and events (/health, /quitquit, etc.).
    config - JSON String - Load balancer configuration. See README.md for
      detailed documentation.
    backend - Load balancer manager backend to use. EG: "nginx".
    update_period - int - frequency with which the need to update is checked.
    max_update_frequency - int - minimum number of seconds between updates.
    weight_adjustment_delay_seconds - int - number of seconds to wait before
      starting configured share adjusters. May not want them running
      immediately after aurproxy deploys.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    metric_publisher_class - str - Python class path for metrics publisher
      class.
    metric_publisher_arg - list(str) - List of equal-sign-delimited string
      kwarg pairs.
      Example:
        ["source=cluster.role.environment.job"]
    mirror_source - JSON String - Source configuration for gor repeater to
      which http traffic should be mirrored.
    mirror_ports - Comma separated integer string - Local ports to mirror.
      Example: "8080,8081"
    mirror_max_qps - Max QPS to mirror to gor repeater.
    mirror_max_update_frequency - integer - number of seconds between updates
      of mirror configuration.
    sentry_dsn - str - Sentry DSN for error logging.
    setup - bool - When run in setup mode, aurproxy will render a configuration
     for the managed load balancer once and then exit. Run aurproxy once in
     setup mode to set up the load balancer, then start aurproxy and the load
     balancer together.
  """
  # Set up sentry error logging
  if sentry_dsn:
    setup_sentry(sentry_dsn)

  # Load config
  try:
    proxy_config = json.loads(config)
  except (TypeError, ValueError):
    raise commandr.CommandrUsageError('Invalid JSON configuration specified via --config')

  # Set up updater
  try:
    proxy_updater = ProxyUpdater(backend, proxy_config, update_period,
                               max_update_frequency)
  except AurProxyConfigException as exc:
    raise commandr.CommandrUsageError(
      'Invalid configuration: {}'.format(str(exc)),
    )

  # Set up mirroring
  mirror_updater = None
  if mirror_source:
    mirror_updater = load_mirror_updater(mirror_source,
                                         mirror_ports,
                                         mirror_max_qps,
                                         mirror_max_update_frequency,
                                         _MIRROR_COMMAND_TEMPLATE_PATH,
                                         mirror_pid_path)

  if setup:
    proxy_updater.set_up()
    if mirror_updater:
      mirror_updater.set_up()
  else:
    # Set up metrics
    set_root_prefix('aurproxy')
    if metric_publisher_class:
      _setup_metrics(metric_publisher_class, metric_publisher_arg)

    # Set up registration
    if registration_class:
      try:
        registerer = load_cli_plugin(registration_class, registration_arg)
        registerer.add()
        register_shutdown_handler(registerer.remove)
      except Exception:
        logger.exception('Registration failure.')
        raise

    # Start the updaters and extract blueprints
    proxy_updater.start(weight_adjustment_delay_seconds)
    blueprints = proxy_updater.blueprints
    if mirror_updater:
      mirror_updater.start()
      blueprints += mirror_updater.blueprints

    _start_web(management_port, sentry_dsn, blueprints)
Exemple #7
0
def run(management_port,
        config,
        backend=_DEFAULT_BACKEND,
        update_period=_DEFAULT_UPDATE_PERIOD,
        max_update_frequency=_DEFAULT_MAX_UPDATE_FREQUENCY,
        weight_adjustment_delay_seconds=_DEFAULT_WEIGHT_ADJUSTMENT_DELAY_SEC,
        registration_class=_DEFAULT_REGISTRATION_CLASS,
        registration_arg=_DEFAULT_REGISTRATION_KWARGS,
        metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
        metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
        mirror_source=_DEFAULT_MIRROR_SOURCE,
        mirror_ports=_DEFAULT_MIRROR_PORTS,
        mirror_max_qps=_DEFAULT_MIRROR_MAX_QPS,
        mirror_max_update_frequency=_DEFAULT_MIRROR_MAX_UPDATE_FREQUENCY,
        mirror_pid_path=_MIRROR_PID_PATH,
        sentry_dsn=None,
        setup=False):
    """Run the Aurproxy load balancer manager.

  Args:
    management_port - int - port for the manager application to listen on for
      Aurora lifecycle queries and events (/health, /quitquit, etc.).
    config - JSON String or file:// location of a JSON document - Load balancer
      configuration. See README.md for detailed documentation.
    backend - Load balancer manager backend to use. EG: "nginx".
    update_period - int - frequency with which the need to update is checked.
    max_update_frequency - int - minimum number of seconds between updates.
    weight_adjustment_delay_seconds - int - number of seconds to wait before
      starting configured share adjusters. May not want them running
      immediately after aurproxy deploys.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    metric_publisher_class - str - Python class path for metrics publisher
      class.
    metric_publisher_arg - list(str) - List of equal-sign-delimited string
      kwarg pairs.
      Example:
        ["source=cluster.role.environment.job"]
    mirror_source - JSON String - Source configuration for gor repeater to
      which http traffic should be mirrored.
    mirror_ports - Comma separated integer string - Local ports to mirror.
      Example: "8080,8081"
    mirror_max_qps - Max QPS to mirror to gor repeater.
    mirror_max_update_frequency - integer - number of seconds between updates
      of mirror configuration.
    sentry_dsn - str - Sentry DSN for error logging.
    setup - bool - When run in setup mode, aurproxy will render a configuration
     for the managed load balancer once and then exit. Run aurproxy once in
     setup mode to set up the load balancer, then start aurproxy and the load
     balancer together.
  """
    # Set up sentry error logging
    if sentry_dsn:
        setup_sentry(sentry_dsn)

    # Load config
    try:
        if config.startswith('file://'):
            with open(config.split('file://', 1)[1]) as config_fh:
                proxy_config = json.load(config_fh)
        else:
            proxy_config = json.loads(config)
    except (TypeError, ValueError):
        raise commandr.CommandrUsageError(
            'Invalid JSON configuration specified via --config')
    except IOError as err:
        raise commandr.CommandrUsageError('Failed to read --config file: %s' %
                                          err)

    # Set up updater
    try:
        proxy_updater = ProxyUpdater(backend, proxy_config, update_period,
                                     max_update_frequency)
    except AurProxyConfigException as exc:
        raise commandr.CommandrUsageError(
            'Invalid configuration: {}'.format(str(exc)), )

    # Set up mirroring
    mirror_updater = None
    if mirror_source:
        mirror_updater = load_mirror_updater(mirror_source, mirror_ports,
                                             mirror_max_qps,
                                             mirror_max_update_frequency,
                                             _MIRROR_COMMAND_TEMPLATE_PATH,
                                             mirror_pid_path)

    if setup:
        proxy_updater.set_up()
        if mirror_updater:
            mirror_updater.set_up()
    else:
        # Set up metrics
        set_root_prefix('aurproxy')
        if metric_publisher_class:
            _setup_metrics(metric_publisher_class, metric_publisher_arg)

        # Set up registration
        if registration_class:
            try:
                registerer = load_cli_plugin(registration_class,
                                             registration_arg)
                registerer.add()
                register_shutdown_handler(registerer.remove)
            except Exception:
                logger.exception('Registration failure.')
                raise

        # Start the updaters and extract blueprints
        proxy_updater.start(weight_adjustment_delay_seconds)
        blueprints = proxy_updater.blueprints
        if mirror_updater:
            mirror_updater.start()
            blueprints += mirror_updater.blueprints

        _start_web(management_port, sentry_dsn, blueprints)
Exemple #8
0
def run(management_port,
        config,
        backend=_DEFAULT_BACKEND,
        update_period=_DEFAULT_UPDATE_PERIOD,
        max_update_frequency=_DEFAULT_MAX_UPDATE_FREQUENCY,
        weight_adjustment_delay_seconds=_DEFAULT_WEIGHT_ADJUSTMENT_DELAY_SEC,
        registration_class=_DEFAULT_REGISTRATION_CLASS,
        registration_arg=_DEFAULT_REGISTRATION_KWARGS,
        metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
        metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
        sentry_dsn=None,
        setup=False):
  """Run the Aurproxy load balancer manager.

  Args:
    management_port - int - port for the manager application to listen on for
      Aurora lifecycle queries and events (/health, /quitquit, etc.).
    config - JSON String - Load balancer configuration. See README.md for
      detailed documentation.
    backend - Load balancer manager backend to use. EG: "nginx".
    update_period - int - frequency with which the need to update is checked.
    max_update_frequency - int - minimum number of seconds between updates.
    weight_adjustment_delay_seconds - int - number of seconds to wait before
      starting configured share adjusters. May not want them running
      immediately after aurproxy deploys.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    metric_publisher_class - str - Python class path for metrics publisher
      class.
    metric_publisher_arg - list(str) - List of equal-sign-delimited string
      kwarg pairs.
      Example:
        ["source=cluster.role.environment.job"]
    sentry_dsn - str - Sentry DSN for error logging.
    setup - bool - When run in setup mode, aurproxy will render a configuration
     for the managed load balancer once and then exit. Run aurproxy once in
     setup mode to set up the load balancer, then start aurproxy and the load
     balancer together.
  """
  # Set up sentry error logging
  if sentry_dsn:
    setup_sentry(sentry_dsn)

  # Load config
  proxy_config = json.loads(config)

  # Set up updater
  proxy_updater = ProxyUpdater(backend, proxy_config, update_period,
                               max_update_frequency)

  if setup:
    proxy_updater.set_up()
  else:
    # Set up metrics
    set_root_prefix('aurproxy')
    if metric_publisher_class:
      try:
        publisher = load_cli_plugin(metric_publisher_class,
                                    metric_publisher_arg)
        add_publisher(publisher)
      except Exception:
        logger.exception('Metrics failure.')
        raise

    # Set up registration
    if registration_class:
      try:
        registerer = load_cli_plugin(registration_class, registration_arg)
        registerer.add()
        register_shutdown_handler(registerer.remove)
      except Exception:
        logger.exception('Registration failure.')
        raise

    # Start the proxy updater.
    proxy_updater.start(weight_adjustment_delay_seconds)

    # Set up management application
    app = Flask(__name__)
    app.register_blueprint(lifecycle_blueprint)
    if sentry_dsn:
      app = setup_sentry_wsgi(app, sentry_dsn)
    http_server = WSGIServer(('0.0.0.0', int(management_port)), app)
    http_server.serve_forever()