def test_load_cli_plugin(self):
cli_kwargs = ['a=1', 'b=2']
instance = load_cli_plugin('__builtin__.dict', cli_kwargs)
self.assertEquals(instance['a'], '1')
self.assertEquals(instance['b'], '2')
extra_kwargs = {'c': '3'}
instance = load_cli_plugin('__builtin__.dict', cli_kwargs, extra_kwargs)
self.assertEquals(instance['a'], '1')
self.assertEquals(instance['b'], '2')
self.assertEquals(instance['c'], '3')
def _setup_metrics(metric_publisher_class, metric_publisher_args):
"""
Sets up a metrics publisher based on command line args.
Args:
metric_publisher_class
metric_publisher_args
"""
try:
publisher = load_cli_plugin(metric_publisher_class,
metric_publisher_args)
add_publisher(publisher)
except Exception:
logger.exception('Metrics failure.')
raise
def _setup_metrics(metric_publisher_class, metric_publisher_args):
"""
Sets up a metrics publisher based on command line args.
Args:
metric_publisher_class
metric_publisher_args
"""
try:
publisher = load_cli_plugin(metric_publisher_class,
metric_publisher_args)
add_publisher(publisher)
except Exception:
logger.exception('Metrics failure.')
raise
def synchronize(registration_source,
registration_class,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
write=False):
"""Add and remove Aurproxy task instances from upstream services.
Intended to be run by administrators or to be called automatically as a cron
as a supplement to / safety net for the pluggable in-task registration and
deregistration Aurproxy events that are driven by Aurora lifecycle events.
Args:
registration_source - JSON string - Source configuration.
Format:
{'registration_class': 'python.class.path',
'arg1': 'val1',
'arg2': 'val2'}
Example:
'{"source_class": "aurproxy.sources.AuroraSource"
"announcer_serverset_path": "/aurora/",
"zk_servers": "0.zk.mycluster.mydomain.com:2181,
1.zk.mycluster.mydomain.com:2181,
2.zk.mycluster.mydomain.com:2181,
"environment": "devel",
"job": "proxytest",
"role": "proxy",
"endpoint": "http"}
See source class definition for valid kwargs.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
write - bool - Whether to apply the changes.
"""
try:
source_dict = json.loads(registration_source)
except (TypeError, ValueError):
raise commandr.CommandrUsageError(
'Invalid JSON configuration specified via --registration_source',
)
source = load_klass_plugin(source_dict,
klass_field_name='source_class')
extra_kwargs = {'source': source}
registerer = load_cli_plugin(registration_class,
registration_arg,
extra_kwargs=extra_kwargs)
registerer.synchronize(write)
def synchronize(registration_source,
registration_class,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
write=False):
"""Add and remove Aurproxy task instances from upstream services.
Intended to be run by administrators or to be called automatically as a cron
as a supplement to / safety net for the pluggable in-task registration and
deregistration Aurproxy events that are driven by Aurora lifecycle events.
Args:
registration_source - JSON string - Source configuration.
Format:
{'registration_class': 'python.class.path',
'arg1': 'val1',
'arg2': 'val2'}
Example:
'{"source_class": "aurproxy.sources.AuroraSource"
"announcer_serverset_path": "/aurora/",
"zk_servers": "0.zk.mycluster.mydomain.com:2181,
1.zk.mycluster.mydomain.com:2181,
2.zk.mycluster.mydomain.com:2181,
"environment": "devel",
"job": "proxytest",
"role": "proxy",
"endpoint": "http"}
See source class definition for valid kwargs.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
write - bool - Whether to apply the changes.
"""
try:
source_dict = json.loads(registration_source)
except (TypeError, ValueError):
raise commandr.CommandrUsageError(
'Invalid JSON configuration specified via --registration_source', )
source = load_klass_plugin(source_dict, klass_field_name='source_class')
extra_kwargs = {'source': source}
registerer = load_cli_plugin(registration_class,
registration_arg,
extra_kwargs=extra_kwargs)
registerer.synchronize(write)
def run(management_port,
config,
backend=_DEFAULT_BACKEND,
update_period=_DEFAULT_UPDATE_PERIOD,
max_update_frequency=_DEFAULT_MAX_UPDATE_FREQUENCY,
weight_adjustment_delay_seconds=_DEFAULT_WEIGHT_ADJUSTMENT_DELAY_SEC,
registration_class=_DEFAULT_REGISTRATION_CLASS,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
mirror_source=_DEFAULT_MIRROR_SOURCE,
mirror_ports=_DEFAULT_MIRROR_PORTS,
mirror_max_qps=_DEFAULT_MIRROR_MAX_QPS,
mirror_max_update_frequency=_DEFAULT_MIRROR_MAX_UPDATE_FREQUENCY,
mirror_pid_path=_MIRROR_PID_PATH,
sentry_dsn=None,
setup=False):
"""Run the Aurproxy load balancer manager.
Args:
management_port - int - port for the manager application to listen on for
Aurora lifecycle queries and events (/health, /quitquit, etc.).
config - JSON String - Load balancer configuration. See README.md for
detailed documentation.
backend - Load balancer manager backend to use. EG: "nginx".
update_period - int - frequency with which the need to update is checked.
max_update_frequency - int - minimum number of seconds between updates.
weight_adjustment_delay_seconds - int - number of seconds to wait before
starting configured share adjusters. May not want them running
immediately after aurproxy deploys.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
metric_publisher_class - str - Python class path for metrics publisher
class.
metric_publisher_arg - list(str) - List of equal-sign-delimited string
kwarg pairs.
Example:
["source=cluster.role.environment.job"]
mirror_source - JSON String - Source configuration for gor repeater to
which http traffic should be mirrored.
mirror_ports - Comma separated integer string - Local ports to mirror.
Example: "8080,8081"
mirror_max_qps - Max QPS to mirror to gor repeater.
mirror_max_update_frequency - integer - number of seconds between updates
of mirror configuration.
sentry_dsn - str - Sentry DSN for error logging.
setup - bool - When run in setup mode, aurproxy will render a configuration
for the managed load balancer once and then exit. Run aurproxy once in
setup mode to set up the load balancer, then start aurproxy and the load
balancer together.
"""
# Set up sentry error logging
if sentry_dsn:
setup_sentry(sentry_dsn)
# Load config
try:
proxy_config = json.loads(config)
except (TypeError, ValueError):
raise commandr.CommandrUsageError('Invalid JSON configuration specified via --config')
# Set up updater
try:
proxy_updater = ProxyUpdater(backend, proxy_config, update_period,
max_update_frequency)
except AurProxyConfigException as exc:
raise commandr.CommandrUsageError(
'Invalid configuration: {}'.format(str(exc)),
)
# Set up mirroring
mirror_updater = None
if mirror_source:
mirror_updater = load_mirror_updater(mirror_source,
mirror_ports,
mirror_max_qps,
mirror_max_update_frequency,
_MIRROR_COMMAND_TEMPLATE_PATH,
mirror_pid_path)
if setup:
proxy_updater.set_up()
if mirror_updater:
mirror_updater.set_up()
else:
# Set up metrics
set_root_prefix('aurproxy')
if metric_publisher_class:
_setup_metrics(metric_publisher_class, metric_publisher_arg)
# Set up registration
if registration_class:
try:
registerer = load_cli_plugin(registration_class, registration_arg)
registerer.add()
register_shutdown_handler(registerer.remove)
except Exception:
logger.exception('Registration failure.')
raise
# Start the updaters and extract blueprints
proxy_updater.start(weight_adjustment_delay_seconds)
blueprints = proxy_updater.blueprints
if mirror_updater:
mirror_updater.start()
blueprints += mirror_updater.blueprints
_start_web(management_port, sentry_dsn, blueprints)
def run(management_port,
config,
backend=_DEFAULT_BACKEND,
update_period=_DEFAULT_UPDATE_PERIOD,
max_update_frequency=_DEFAULT_MAX_UPDATE_FREQUENCY,
weight_adjustment_delay_seconds=_DEFAULT_WEIGHT_ADJUSTMENT_DELAY_SEC,
registration_class=_DEFAULT_REGISTRATION_CLASS,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
mirror_source=_DEFAULT_MIRROR_SOURCE,
mirror_ports=_DEFAULT_MIRROR_PORTS,
mirror_max_qps=_DEFAULT_MIRROR_MAX_QPS,
mirror_max_update_frequency=_DEFAULT_MIRROR_MAX_UPDATE_FREQUENCY,
mirror_pid_path=_MIRROR_PID_PATH,
sentry_dsn=None,
setup=False):
"""Run the Aurproxy load balancer manager.
Args:
management_port - int - port for the manager application to listen on for
Aurora lifecycle queries and events (/health, /quitquit, etc.).
config - JSON String or file:// location of a JSON document - Load balancer
configuration. See README.md for detailed documentation.
backend - Load balancer manager backend to use. EG: "nginx".
update_period - int - frequency with which the need to update is checked.
max_update_frequency - int - minimum number of seconds between updates.
weight_adjustment_delay_seconds - int - number of seconds to wait before
starting configured share adjusters. May not want them running
immediately after aurproxy deploys.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
metric_publisher_class - str - Python class path for metrics publisher
class.
metric_publisher_arg - list(str) - List of equal-sign-delimited string
kwarg pairs.
Example:
["source=cluster.role.environment.job"]
mirror_source - JSON String - Source configuration for gor repeater to
which http traffic should be mirrored.
mirror_ports - Comma separated integer string - Local ports to mirror.
Example: "8080,8081"
mirror_max_qps - Max QPS to mirror to gor repeater.
mirror_max_update_frequency - integer - number of seconds between updates
of mirror configuration.
sentry_dsn - str - Sentry DSN for error logging.
setup - bool - When run in setup mode, aurproxy will render a configuration
for the managed load balancer once and then exit. Run aurproxy once in
setup mode to set up the load balancer, then start aurproxy and the load
balancer together.
"""
# Set up sentry error logging
if sentry_dsn:
setup_sentry(sentry_dsn)
# Load config
try:
if config.startswith('file://'):
with open(config.split('file://', 1)[1]) as config_fh:
proxy_config = json.load(config_fh)
else:
proxy_config = json.loads(config)
except (TypeError, ValueError):
raise commandr.CommandrUsageError(
'Invalid JSON configuration specified via --config')
except IOError as err:
raise commandr.CommandrUsageError('Failed to read --config file: %s' %
err)
# Set up updater
try:
proxy_updater = ProxyUpdater(backend, proxy_config, update_period,
max_update_frequency)
except AurProxyConfigException as exc:
raise commandr.CommandrUsageError(
'Invalid configuration: {}'.format(str(exc)), )
# Set up mirroring
mirror_updater = None
if mirror_source:
mirror_updater = load_mirror_updater(mirror_source, mirror_ports,
mirror_max_qps,
mirror_max_update_frequency,
_MIRROR_COMMAND_TEMPLATE_PATH,
mirror_pid_path)
if setup:
proxy_updater.set_up()
if mirror_updater:
mirror_updater.set_up()
else:
# Set up metrics
set_root_prefix('aurproxy')
if metric_publisher_class:
_setup_metrics(metric_publisher_class, metric_publisher_arg)
# Set up registration
if registration_class:
try:
registerer = load_cli_plugin(registration_class,
registration_arg)
registerer.add()
register_shutdown_handler(registerer.remove)
except Exception:
logger.exception('Registration failure.')
raise
# Start the updaters and extract blueprints
proxy_updater.start(weight_adjustment_delay_seconds)
blueprints = proxy_updater.blueprints
if mirror_updater:
mirror_updater.start()
blueprints += mirror_updater.blueprints
_start_web(management_port, sentry_dsn, blueprints)
def run(management_port,
config,
backend=_DEFAULT_BACKEND,
update_period=_DEFAULT_UPDATE_PERIOD,
max_update_frequency=_DEFAULT_MAX_UPDATE_FREQUENCY,
weight_adjustment_delay_seconds=_DEFAULT_WEIGHT_ADJUSTMENT_DELAY_SEC,
registration_class=_DEFAULT_REGISTRATION_CLASS,
registration_arg=_DEFAULT_REGISTRATION_KWARGS,
metric_publisher_class=_DEFAULT_METRIC_PUBLISHER_CLASS,
metric_publisher_arg=_DEFAULT_METRIC_PUBLISHER_KWARGS,
sentry_dsn=None,
setup=False):
"""Run the Aurproxy load balancer manager.
Args:
management_port - int - port for the manager application to listen on for
Aurora lifecycle queries and events (/health, /quitquit, etc.).
config - JSON String - Load balancer configuration. See README.md for
detailed documentation.
backend - Load balancer manager backend to use. EG: "nginx".
update_period - int - frequency with which the need to update is checked.
max_update_frequency - int - minimum number of seconds between updates.
weight_adjustment_delay_seconds - int - number of seconds to wait before
starting configured share adjusters. May not want them running
immediately after aurproxy deploys.
registration_class - str - Python class path for registration class.
registration_arg - list(str) - List of equal-sign-delimited string kwarg
pairs.
Example:
["domain=myapp.mydomain.com", "type=A"]
metric_publisher_class - str - Python class path for metrics publisher
class.
metric_publisher_arg - list(str) - List of equal-sign-delimited string
kwarg pairs.
Example:
["source=cluster.role.environment.job"]
sentry_dsn - str - Sentry DSN for error logging.
setup - bool - When run in setup mode, aurproxy will render a configuration
for the managed load balancer once and then exit. Run aurproxy once in
setup mode to set up the load balancer, then start aurproxy and the load
balancer together.
"""
# Set up sentry error logging
if sentry_dsn:
setup_sentry(sentry_dsn)
# Load config
proxy_config = json.loads(config)
# Set up updater
proxy_updater = ProxyUpdater(backend, proxy_config, update_period,
max_update_frequency)
if setup:
proxy_updater.set_up()
else:
# Set up metrics
set_root_prefix('aurproxy')
if metric_publisher_class:
try:
publisher = load_cli_plugin(metric_publisher_class,
metric_publisher_arg)
add_publisher(publisher)
except Exception:
logger.exception('Metrics failure.')
raise
# Set up registration
if registration_class:
try:
registerer = load_cli_plugin(registration_class, registration_arg)
registerer.add()
register_shutdown_handler(registerer.remove)
except Exception:
logger.exception('Registration failure.')
raise
# Start the proxy updater.
proxy_updater.start(weight_adjustment_delay_seconds)
# Set up management application
app = Flask(__name__)
app.register_blueprint(lifecycle_blueprint)
if sentry_dsn:
app = setup_sentry_wsgi(app, sentry_dsn)
http_server = WSGIServer(('0.0.0.0', int(management_port)), app)
http_server.serve_forever()