def test_load_klass_plugin(self):
def build_klass_dict(klass_field_name='klass'):
return {
klass_field_name: '__builtin__.dict',
'a': '1',
'b': '2',
}
klass_dict = build_klass_dict()
instance = load_klass_plugin(klass_dict)
self.assertEquals(instance['a'], '1')
self.assertEquals(instance['b'], '2')
extra_kwargs = {'c':'3'}
instance = load_klass_plugin(klass_dict, **extra_kwargs)
self.assertEquals(instance['a'], '1')
self.assertEquals(instance['b'], '2')
self.assertEquals(instance['c'], '3')
klass_dict = build_klass_dict('klass2')
instance = load_klass_plugin(klass_dict, klass_field_name='klass2')
self.assertEquals(instance['a'], '1')
self.assertEquals(instance['b'], '2')
def run_replay(management_port,
replay_port,
replay_source,
replay_max_qps,
replay_max_update_frequency=_DEFAULT_REPLAY_MAX_UPDATE_FREQUENCY,
replay_pid_path=_MIRROR_PID_PATH,
metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
sentry_dsn=None,
setup=False):
"""Run the Aurproxy traffic replay server manager.
Args:
management_port - int - port for the manager application to listen on for
Aurora lifecycle queries and events (/health, /quitquit, etc.).
replay_port - int - port on which the replay server to listen on for a
mirrored traffic stream.
replay_source - JSON string - Source configuration to which gor replay
server will send traffic.
replay_max_qps - maximum QPS to replay to listeners.
replay_max_update_frequency - Max QPS to replay to source endpoints.
metric_publisher_class - str - Python class path for metrics publisher
class.
metric_publisher_arg - list(str) - List of equal-sign-delimited string
kwarg pairs.
Example:
["source=cluster.role.environment.job"]
sentry_dsn - str - Sentry DSN for error logging.
setup - bool - When run in setup mode, aurproxy will render a configuration
for the replay server once and then exit. Run aurproxy once in setup mode
to set up the replay server, then start aurproxy and the replay server
together.
"""
try:
source_dict = json.loads(replay_source)
except (TypeError, ValueError):
raise commandr.CommandrUsageError('Invalid JSON configuration specified via --replay_source')
source = load_klass_plugin(source_dict, klass_field_name='source_class')
mirror_updater = load_mirror_updater(source,
replay_port,
replay_max_qps,
replay_max_update_frequency,
_REPLAY_COMMAND_TEMPLATE_PATH,
replay_pid_path)
if setup:
mirror_updater.update(kill_running=False)
else:
if metric_publisher_class:
_setup_metrics(metric_publisher_class, metric_publisher_arg)
mirror_updater.start()
_start_web(management_port, sentry_dsn, mirror_updater.blueprints)
def run_replay(
management_port,
replay_port,
replay_source,
replay_max_qps,
replay_max_update_frequency=_DEFAULT_REPLAY_MAX_UPDATE_FREQUENCY,
replay_pid_path=_MIRROR_PID_PATH,
metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
sentry_dsn=None,
setup=False):
"""Run the Aurproxy traffic replay server manager.
Args:
management_port - int - port for the manager application to listen on for
Aurora lifecycle queries and events (/health, /quitquit, etc.).
replay_port - int - port on which the replay server to listen on for a
mirrored traffic stream.
replay_source - JSON string - Source configuration to which gor replay
server will send traffic.
replay_max_qps - maximum QPS to replay to listeners.
replay_max_update_frequency - Max QPS to replay to source endpoints.
metric_publisher_class - str - Python class path for metrics publisher
class.
metric_publisher_arg - list(str) - List of equal-sign-delimited string
kwarg pairs.
Example:
["source=cluster.role.environment.job"]
sentry_dsn - str - Sentry DSN for error logging.
setup - bool - When run in setup mode, aurproxy will render a configuration
for the replay server once and then exit. Run aurproxy once in setup mode
to set up the replay server, then start aurproxy and the replay server
together.
"""
try:
source_dict = json.loads(replay_source)
except (TypeError, ValueError):
raise commandr.CommandrUsageError(
'Invalid JSON configuration specified via --replay_source')
source = load_klass_plugin(source_dict, klass_field_name='source_class')
mirror_updater = load_mirror_updater(source, replay_port, replay_max_qps,
replay_max_update_frequency,
_REPLAY_COMMAND_TEMPLATE_PATH,
replay_pid_path)
if setup:
mirror_updater.update(kill_running=False)
else:
if metric_publisher_class:
_setup_metrics(metric_publisher_class, metric_publisher_arg)
mirror_updater.start()
_start_web(management_port, sentry_dsn, mirror_updater.blueprints)
def _load_proxy_source(self, source):
source_copy = copy.deepcopy(source)
share_adjusters = source_copy.pop('share_adjusters', None)
sa_factories = []
if share_adjusters:
sa_factories = self._load_share_adjuster_factories(share_adjusters)
extra_kwargs = {}
extra_kwargs['signal_update_fn'] = self.signal_update
extra_kwargs['share_adjuster_factories'] = sa_factories
proxy_source = load_klass_plugin(source_copy,
klass_field_name='source_class',
**extra_kwargs)
return proxy_source
def _load_proxy_source(self, source):
source_copy = copy.deepcopy(source)
share_adjusters = source_copy.pop('share_adjusters', None)
sa_factories = []
if share_adjusters:
sa_factories = self._load_share_adjuster_factories(share_adjusters)
extra_kwargs = {}
extra_kwargs['signal_update_fn'] = self.signal_update
extra_kwargs['share_adjuster_factories'] = sa_factories
proxy_source = load_klass_plugin(source_copy,
klass_field_name='source_class',
**extra_kwargs)
return proxy_source
def synchronize(registration_source,
registration_class,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
write=False):
"""Add and remove Aurproxy task instances from upstream services.
Intended to be run by administrators or to be called automatically as a cron
as a supplement to / safety net for the pluggable in-task registration and
deregistration Aurproxy events that are driven by Aurora lifecycle events.
Args:
registration_source - JSON string - Source configuration.
Format:
{'registration_class': 'python.class.path',
'arg1': 'val1',
'arg2': 'val2'}
Example:
'{"source_class": "aurproxy.sources.AuroraSource"
"announcer_serverset_path": "/aurora/",
"zk_servers": "0.zk.mycluster.mydomain.com:2181,
1.zk.mycluster.mydomain.com:2181,
2.zk.mycluster.mydomain.com:2181,
"environment": "devel",
"job": "proxytest",
"role": "proxy",
"endpoint": "http"}
See source class definition for valid kwargs.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
write - bool - Whether to apply the changes.
"""
try:
source_dict = json.loads(registration_source)
except (TypeError, ValueError):
raise commandr.CommandrUsageError(
'Invalid JSON configuration specified via --registration_source',
)
source = load_klass_plugin(source_dict,
klass_field_name='source_class')
extra_kwargs = {'source': source}
registerer = load_cli_plugin(registration_class,
registration_arg,
extra_kwargs=extra_kwargs)
registerer.synchronize(write)
def synchronize(registration_source,
registration_class,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
write=False):
"""Add and remove Aurproxy task instances from upstream services.
Intended to be run by administrators or to be called automatically as a cron
as a supplement to / safety net for the pluggable in-task registration and
deregistration Aurproxy events that are driven by Aurora lifecycle events.
Args:
registration_source - JSON string - Source configuration.
Format:
{'registration_class': 'python.class.path',
'arg1': 'val1',
'arg2': 'val2'}
Example:
'{"source_class": "aurproxy.sources.AuroraSource"
"announcer_serverset_path": "/aurora/",
"zk_servers": "0.zk.mycluster.mydomain.com:2181,
1.zk.mycluster.mydomain.com:2181,
2.zk.mycluster.mydomain.com:2181,
"environment": "devel",
"job": "proxytest",
"role": "proxy",
"endpoint": "http"}
See source class definition for valid kwargs.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
write - bool - Whether to apply the changes.
"""
try:
source_dict = json.loads(registration_source)
except (TypeError, ValueError):
raise commandr.CommandrUsageError(
'Invalid JSON configuration specified via --registration_source', )
source = load_klass_plugin(source_dict, klass_field_name='source_class')
extra_kwargs = {'source': source}
registerer = load_cli_plugin(registration_class,
registration_arg,
extra_kwargs=extra_kwargs)
registerer.synchronize(write)
def load_mirror_updater(source,
ports,
max_qps,
max_update_frequency,
command_template_path,
pid_path):
"""
Load a MirrorUpdater.
Args:
source - JSON string or ProxySource - Source whose endpoints describe gor
repeaters.
ports - string of comma seperated integers- Local ports to mirror.
Example: "8080,8081"
max_qps - integer - Max QPS to mirror to gor repeater.
max_update_frequency - integer - number of seconds between updates of
mirror configuration.
command_template_path - str - path to command template to be rendered.
Returns:
A MirrorUpdater instance.
"""
if not source:
raise AurProxyConfigException('source_config required!')
if not ports:
raise AurProxyConfigException('ports required!')
if not max_qps:
raise AurProxyConfigException('max_qps required!')
if not os.path.isfile(command_template_path):
msg = '"{0}" doesn\'t exist!'.format(command_template_path)
raise AurProxyConfigException(msg)
ports = [ int(p) for p in ports.split(',') ]
if not isinstance(source, ProxySource):
source_dict = json.loads(source)
source = load_klass_plugin(source_dict,
klass_field_name='source_class')
return MirrorUpdater(source,
ports,
max_qps,
max_update_frequency,
command_template_path,
pid_path)
def _load_source(self, source_config):
"""
Loads a source from a source configuration string.
Args:
source_config - JSON string source configuration.
Returns:
tellapart.aurproxy.source.ProxySource instance.
"""
source_class = source_config.get('source_class', None)
if not source_class or source_class not in self._source_whitelist:
message = 'Invalid source_class \'{0}\''.format(source_class)
raise abort(403, message=message)
source = load_klass_plugin(source_config,
klass_field_name='source_class')
source.register_on_add(self.__on_source_add)
source.register_on_remove(self.__on_source_remove)
return source
def _load_source(self, source_config):
"""
Loads a source from a source configuration string.
Args:
source_config - JSON string source configuration.
Returns:
tellapart.aurproxy.source.ProxySource instance.
"""
source_class = source_config.get('source_class', None)
if not source_class or source_class not in self._source_whitelist:
message = 'Invalid source_class \'{0}\''.format(source_class)
raise abort(403, message=message)
source = load_klass_plugin(source_config,
klass_field_name='source_class')
source.register_on_add(self.__on_source_add)
source.register_on_remove(self.__on_source_remove)
return source
def load_mirror_updater(source, ports, max_qps, max_update_frequency,
command_template_path, pid_path):
"""
Load a MirrorUpdater.
Args:
source - JSON string or ProxySource - Source whose endpoints describe gor
repeaters.
ports - string of comma seperated integers- Local ports to mirror.
Example: "8080,8081"
max_qps - integer - Max QPS to mirror to gor repeater.
max_update_frequency - integer - number of seconds between updates of
mirror configuration.
command_template_path - str - path to command template to be rendered.
Returns:
A MirrorUpdater instance.
"""
if not source:
raise AurProxyConfigException('source_config required!')
if not ports:
raise AurProxyConfigException('ports required!')
if not max_qps:
raise AurProxyConfigException('max_qps required!')
if not os.path.isfile(command_template_path):
msg = '"{0}" doesn\'t exist!'.format(command_template_path)
raise AurProxyConfigException(msg)
ports = [int(p) for p in ports.split(',')]
if not isinstance(source, ProxySource):
source_dict = json.loads(source)
source = load_klass_plugin(source_dict,
klass_field_name='source_class')
return MirrorUpdater(source, ports, max_qps, max_update_frequency,
command_template_path, pid_path)
