Example #1
0
 def test_load_klass_plugin(self):
   def build_klass_dict(klass_field_name='klass'):
     return {
       klass_field_name: '__builtin__.dict',
       'a': '1',
       'b': '2',
     }
   klass_dict = build_klass_dict()
   instance = load_klass_plugin(klass_dict)
   self.assertEquals(instance['a'], '1')
   self.assertEquals(instance['b'], '2')
   extra_kwargs = {'c':'3'}
   instance = load_klass_plugin(klass_dict, **extra_kwargs)
   self.assertEquals(instance['a'], '1')
   self.assertEquals(instance['b'], '2')
   self.assertEquals(instance['c'], '3')
   klass_dict = build_klass_dict('klass2')
   instance = load_klass_plugin(klass_dict, klass_field_name='klass2')
   self.assertEquals(instance['a'], '1')
   self.assertEquals(instance['b'], '2')
Example #2
0
def run_replay(management_port,
               replay_port,
               replay_source,
               replay_max_qps,
               replay_max_update_frequency=_DEFAULT_REPLAY_MAX_UPDATE_FREQUENCY,
               replay_pid_path=_MIRROR_PID_PATH,
               metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
               metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
               sentry_dsn=None,
               setup=False):
  """Run the Aurproxy traffic replay server manager.

  Args:
    management_port - int - port for the manager application to listen on for
      Aurora lifecycle queries and events (/health, /quitquit, etc.).
    replay_port - int - port on which the replay server to listen on for a
      mirrored traffic stream.
    replay_source - JSON string - Source configuration to which gor replay
      server will send traffic.
    replay_max_qps - maximum QPS to replay to listeners.
    replay_max_update_frequency - Max QPS to replay to source endpoints.
    metric_publisher_class - str - Python class path for metrics publisher
      class.
    metric_publisher_arg - list(str) - List of equal-sign-delimited string
      kwarg pairs.
      Example:
        ["source=cluster.role.environment.job"]
    sentry_dsn - str - Sentry DSN for error logging.
    setup - bool - When run in setup mode, aurproxy will render a configuration
      for the replay server once and then exit. Run aurproxy once in setup mode
      to set up the replay server, then start aurproxy and the replay server
      together.
  """
  try:
    source_dict = json.loads(replay_source)
  except (TypeError, ValueError):
    raise commandr.CommandrUsageError('Invalid JSON configuration specified via --replay_source')

  source = load_klass_plugin(source_dict, klass_field_name='source_class')
  mirror_updater = load_mirror_updater(source,
                                       replay_port,
                                       replay_max_qps,
                                       replay_max_update_frequency,
                                       _REPLAY_COMMAND_TEMPLATE_PATH,
                                       replay_pid_path)
  if setup:
    mirror_updater.update(kill_running=False)
  else:
    if metric_publisher_class:
      _setup_metrics(metric_publisher_class, metric_publisher_arg)
    mirror_updater.start()
    _start_web(management_port, sentry_dsn, mirror_updater.blueprints)
Example #3
0
def run_replay(
        management_port,
        replay_port,
        replay_source,
        replay_max_qps,
        replay_max_update_frequency=_DEFAULT_REPLAY_MAX_UPDATE_FREQUENCY,
        replay_pid_path=_MIRROR_PID_PATH,
        metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
        metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
        sentry_dsn=None,
        setup=False):
    """Run the Aurproxy traffic replay server manager.

  Args:
    management_port - int - port for the manager application to listen on for
      Aurora lifecycle queries and events (/health, /quitquit, etc.).
    replay_port - int - port on which the replay server to listen on for a
      mirrored traffic stream.
    replay_source - JSON string - Source configuration to which gor replay
      server will send traffic.
    replay_max_qps - maximum QPS to replay to listeners.
    replay_max_update_frequency - Max QPS to replay to source endpoints.
    metric_publisher_class - str - Python class path for metrics publisher
      class.
    metric_publisher_arg - list(str) - List of equal-sign-delimited string
      kwarg pairs.
      Example:
        ["source=cluster.role.environment.job"]
    sentry_dsn - str - Sentry DSN for error logging.
    setup - bool - When run in setup mode, aurproxy will render a configuration
      for the replay server once and then exit. Run aurproxy once in setup mode
      to set up the replay server, then start aurproxy and the replay server
      together.
  """
    try:
        source_dict = json.loads(replay_source)
    except (TypeError, ValueError):
        raise commandr.CommandrUsageError(
            'Invalid JSON configuration specified via --replay_source')

    source = load_klass_plugin(source_dict, klass_field_name='source_class')
    mirror_updater = load_mirror_updater(source, replay_port, replay_max_qps,
                                         replay_max_update_frequency,
                                         _REPLAY_COMMAND_TEMPLATE_PATH,
                                         replay_pid_path)
    if setup:
        mirror_updater.update(kill_running=False)
    else:
        if metric_publisher_class:
            _setup_metrics(metric_publisher_class, metric_publisher_arg)
        mirror_updater.start()
        _start_web(management_port, sentry_dsn, mirror_updater.blueprints)
Example #4
0
    def _load_proxy_source(self, source):
        source_copy = copy.deepcopy(source)
        share_adjusters = source_copy.pop('share_adjusters', None)
        sa_factories = []
        if share_adjusters:
            sa_factories = self._load_share_adjuster_factories(share_adjusters)

        extra_kwargs = {}
        extra_kwargs['signal_update_fn'] = self.signal_update
        extra_kwargs['share_adjuster_factories'] = sa_factories
        proxy_source = load_klass_plugin(source_copy,
                                         klass_field_name='source_class',
                                         **extra_kwargs)
        return proxy_source
Example #5
0
  def _load_proxy_source(self, source):
    source_copy = copy.deepcopy(source)
    share_adjusters = source_copy.pop('share_adjusters', None)
    sa_factories = []
    if share_adjusters:
      sa_factories = self._load_share_adjuster_factories(share_adjusters)

    extra_kwargs = {}
    extra_kwargs['signal_update_fn'] = self.signal_update
    extra_kwargs['share_adjuster_factories'] = sa_factories
    proxy_source = load_klass_plugin(source_copy,
                                     klass_field_name='source_class',
                                     **extra_kwargs)
    return proxy_source
Example #6
0
def synchronize(registration_source,
                registration_class,
                registration_arg=_DEFAULT_REGISTRATION_KWARGS,
                write=False):
  """Add and remove Aurproxy task instances from upstream services.

  Intended to be run by administrators or to be called automatically as a cron
  as a supplement to / safety net for the pluggable in-task registration and
  deregistration Aurproxy events that are driven by Aurora lifecycle events.

  Args:
    registration_source - JSON string - Source configuration.
      Format:
        {'registration_class': 'python.class.path',
         'arg1': 'val1',
         'arg2': 'val2'}
      Example:
        '{"source_class": "aurproxy.sources.AuroraSource"
          "announcer_serverset_path": "/aurora/",
          "zk_servers": "0.zk.mycluster.mydomain.com:2181,
                         1.zk.mycluster.mydomain.com:2181,
                         2.zk.mycluster.mydomain.com:2181,
          "environment": "devel",
          "job": "proxytest",
          "role": "proxy",
          "endpoint": "http"}
      See source class definition for valid kwargs.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    write - bool - Whether to apply the changes.
  """
  try:
    source_dict = json.loads(registration_source)
  except (TypeError, ValueError):
    raise commandr.CommandrUsageError(
      'Invalid JSON configuration specified via --registration_source',
    )

  source = load_klass_plugin(source_dict,
                             klass_field_name='source_class')
  extra_kwargs = {'source': source}
  registerer = load_cli_plugin(registration_class,
                               registration_arg,
                               extra_kwargs=extra_kwargs)
  registerer.synchronize(write)
Example #7
0
def synchronize(registration_source,
                registration_class,
                registration_arg=_DEFAULT_REGISTRATION_KWARGS,
                write=False):
    """Add and remove Aurproxy task instances from upstream services.

  Intended to be run by administrators or to be called automatically as a cron
  as a supplement to / safety net for the pluggable in-task registration and
  deregistration Aurproxy events that are driven by Aurora lifecycle events.

  Args:
    registration_source - JSON string - Source configuration.
      Format:
        {'registration_class': 'python.class.path',
         'arg1': 'val1',
         'arg2': 'val2'}
      Example:
        '{"source_class": "aurproxy.sources.AuroraSource"
          "announcer_serverset_path": "/aurora/",
          "zk_servers": "0.zk.mycluster.mydomain.com:2181,
                         1.zk.mycluster.mydomain.com:2181,
                         2.zk.mycluster.mydomain.com:2181,
          "environment": "devel",
          "job": "proxytest",
          "role": "proxy",
          "endpoint": "http"}
      See source class definition for valid kwargs.
    registration_class - str - Python class path for registration class.
    registration_arg - list(str) - List of equal-sign-delimited string kwarg
      pairs.
      Example:
        ["domain=myapp.mydomain.com", "type=A"]
    write - bool - Whether to apply the changes.
  """
    try:
        source_dict = json.loads(registration_source)
    except (TypeError, ValueError):
        raise commandr.CommandrUsageError(
            'Invalid JSON configuration specified via --registration_source', )

    source = load_klass_plugin(source_dict, klass_field_name='source_class')
    extra_kwargs = {'source': source}
    registerer = load_cli_plugin(registration_class,
                                 registration_arg,
                                 extra_kwargs=extra_kwargs)
    registerer.synchronize(write)
Example #8
0
def load_mirror_updater(source,
                        ports,
                        max_qps,
                        max_update_frequency,
                        command_template_path,
                        pid_path):
  """
  Load a MirrorUpdater.

  Args:
    source - JSON string or ProxySource - Source whose endpoints describe gor
      repeaters.
    ports - string of comma seperated integers- Local ports to mirror.
      Example: "8080,8081"
    max_qps - integer - Max QPS to mirror to gor repeater.
    max_update_frequency - integer - number of seconds between updates of
      mirror configuration.
    command_template_path - str - path to command template to be rendered.

  Returns:
    A MirrorUpdater instance.
  """
  if not source:
    raise AurProxyConfigException('source_config required!')
  if not ports:
    raise AurProxyConfigException('ports required!')
  if not max_qps:
    raise AurProxyConfigException('max_qps required!')
  if not os.path.isfile(command_template_path):
    msg = '"{0}" doesn\'t exist!'.format(command_template_path)
    raise AurProxyConfigException(msg)
  ports = [ int(p) for p in ports.split(',') ]

  if not isinstance(source, ProxySource):
    source_dict = json.loads(source)
    source = load_klass_plugin(source_dict,
                               klass_field_name='source_class')

  return MirrorUpdater(source,
                       ports,
                       max_qps,
                       max_update_frequency,
                       command_template_path,
                       pid_path)
Example #9
0
    def _load_source(self, source_config):
        """
    Loads a source from a source configuration string.

    Args:
      source_config - JSON string source configuration.

    Returns:
      tellapart.aurproxy.source.ProxySource instance.
    """
        source_class = source_config.get('source_class', None)
        if not source_class or source_class not in self._source_whitelist:
            message = 'Invalid source_class \'{0}\''.format(source_class)
            raise abort(403, message=message)

        source = load_klass_plugin(source_config,
                                   klass_field_name='source_class')
        source.register_on_add(self.__on_source_add)
        source.register_on_remove(self.__on_source_remove)
        return source
Example #10
0
  def _load_source(self, source_config):
    """
    Loads a source from a source configuration string.

    Args:
      source_config - JSON string source configuration.

    Returns:
      tellapart.aurproxy.source.ProxySource instance.
    """
    source_class = source_config.get('source_class', None)
    if not source_class or source_class not in self._source_whitelist:
      message = 'Invalid source_class \'{0}\''.format(source_class)
      raise abort(403, message=message)

    source = load_klass_plugin(source_config,
                               klass_field_name='source_class')
    source.register_on_add(self.__on_source_add)
    source.register_on_remove(self.__on_source_remove)
    return source
Example #11
0
def load_mirror_updater(source, ports, max_qps, max_update_frequency,
                        command_template_path, pid_path):
    """
  Load a MirrorUpdater.

  Args:
    source - JSON string or ProxySource - Source whose endpoints describe gor
      repeaters.
    ports - string of comma seperated integers- Local ports to mirror.
      Example: "8080,8081"
    max_qps - integer - Max QPS to mirror to gor repeater.
    max_update_frequency - integer - number of seconds between updates of
      mirror configuration.
    command_template_path - str - path to command template to be rendered.

  Returns:
    A MirrorUpdater instance.
  """
    if not source:
        raise AurProxyConfigException('source_config required!')
    if not ports:
        raise AurProxyConfigException('ports required!')
    if not max_qps:
        raise AurProxyConfigException('max_qps required!')
    if not os.path.isfile(command_template_path):
        msg = '"{0}" doesn\'t exist!'.format(command_template_path)
        raise AurProxyConfigException(msg)
    ports = [int(p) for p in ports.split(',')]

    if not isinstance(source, ProxySource):
        source_dict = json.loads(source)
        source = load_klass_plugin(source_dict,
                                   klass_field_name='source_class')

    return MirrorUpdater(source, ports, max_qps, max_update_frequency,
                         command_template_path, pid_path)