def test_available_location_types(self, mock_data):
location_types = available_location_types()
assert isinstance(location_types, list)
assert len(location_types) > 0
def generate_configuration(synapse_tools_config, zookeeper_topology, services):
synapse_config = generate_base_config(synapse_tools_config)
available_locations = available_location_types()
location_depth_mapping = {
loc: depth
for depth, loc in enumerate(available_locations)
}
available_locations = set(available_locations)
for (service_name, service_info) in services:
proxy_port = service_info.get('proxy_port')
if proxy_port is None:
continue
discover_type = service_info.get('discover', 'region')
advertise_types = sorted(
[
advertise_typ
for advertise_typ in service_info.get('advertise', ['region'])
# don't consider invalid advertise types
if advertise_typ in available_locations
],
key=lambda typ: location_depth_mapping[typ],
reverse=True, # consider the most specific types first
)
if discover_type not in advertise_types:
return {}
base_haproxy_cfg = base_haproxy_cfg_for_service(
service_name=service_name,
service_info=service_info,
zookeeper_topology=zookeeper_topology,
synapse_tools_config=synapse_tools_config,
)
for advertise_type in advertise_types:
config = copy.deepcopy(base_haproxy_cfg)
backend_identifier = get_backend_name(service_name, discover_type, advertise_type)
if advertise_type == discover_type:
# Specify a proxy port to create a frontend for this service
config['haproxy']['port'] = str(proxy_port)
config['discovery']['label_filters'] = [
{
'label': '%s:%s' % (advertise_type, get_current_location(advertise_type)),
'value': '',
'condition': 'equals',
},
]
config['haproxy']['backend_name'] = backend_identifier
synapse_config['services'][backend_identifier] = config
# populate the ACLs to route to the service backends
synapse_config['services'][service_name]['haproxy']['frontend'].extend(
generate_acls_for_service(
service_name=service_name,
discover_type=discover_type,
advertise_types=advertise_types,
)
)
return synapse_config
def generate_configuration(
synapse_tools_config: SynapseToolsConfig,
zookeeper_topology: Iterable[str],
services: Iterable[Tuple[str, ServiceNamespaceConfig]],
) -> BaseConfig:
synapse_config = generate_base_config(synapse_tools_config)
available_locations = available_location_types()
location_depth_mapping = {
loc: depth
for depth, loc in enumerate(available_locations)
}
available_locations = set(available_locations)
for (service_name, service_info) in services:
proxy_port = service_info.get('proxy_port', -1)
# If we end up with the default value or a negative number in general,
# then we know that the service does not want to be in SmartStack
if proxy_port is not None and proxy_port < 0:
continue
# Note that at this point proxy_port can be:
# * valid number: Wants Load balancing (HAProxy/Nginx)
# * None: Wants discovery, but no load balancing (files)
discover_type = service_info.get('discover', 'region')
advertise_types = sorted(
[
advertise_typ
for advertise_typ in service_info.get('advertise', ['region'])
# don't consider invalid advertise types
if advertise_typ in available_locations
],
key=lambda typ: location_depth_mapping[typ],
reverse=True, # consider the most specific types first
)
if discover_type not in advertise_types:
return {}
base_watcher_cfg = base_watcher_cfg_for_service(
service_name=service_name,
service_info=cast(ServiceInfo, service_info),
zookeeper_topology=zookeeper_topology,
synapse_tools_config=synapse_tools_config,
)
socket_path = _get_socket_path(synapse_tools_config, service_name)
socket_proxy_path = _get_socket_path(synapse_tools_config,
service_name,
proxy_proto=True)
endpoint_timeouts = service_info.get('endpoint_timeouts', {})
for (advertise_type, endpoint_name) in _get_backends_for_service(
advertise_types,
endpoint_timeouts,
):
backend_identifier = get_backend_name(service_name, discover_type,
advertise_type,
endpoint_name)
config = copy.deepcopy(base_watcher_cfg)
config['discovery']['label_filters'] = [
{
'label':
'%s:%s' %
(advertise_type, get_current_location(advertise_type)),
'value':
'',
'condition':
'equals',
},
]
if endpoint_name != HAPROXY_DEFAULT_SECTION:
endpoint_timeout = endpoint_timeouts[endpoint_name]
# Override the 'timeout server' value
timeout_index_list = [
i for i, v in enumerate(config['haproxy']['backend'])
if v.startswith("timeout server ")
]
if len(timeout_index_list) > 0:
timeout_index = timeout_index_list[0]
config['haproxy']['backend'][
timeout_index] = 'timeout server %dms' % endpoint_timeout
else:
config['haproxy']['backend'].append('timeout server %dms' %
endpoint_timeout)
if proxy_port is None:
config['haproxy'] = {'disabled': True}
if synapse_tools_config['listen_with_nginx']:
config['nginx'] = {'disabled': True}
else:
if advertise_type == discover_type and endpoint_name == HAPROXY_DEFAULT_SECTION:
# Specify a proxy port to create a frontend for this service
if synapse_tools_config['listen_with_haproxy']:
config['haproxy']['port'] = str(proxy_port)
config['haproxy']['frontend'].extend([
'bind {0}'.format(socket_path),
'bind {0} accept-proxy'.format(socket_proxy_path),
])
# If listen_with_haproxy is False, then have
# HAProxy bind only to the socket. Nginx may or may not
# be listening on ports based on listen_with_nginx values
# at this stage.
else:
config['haproxy']['port'] = None
config['haproxy']['bind_address'] = socket_path
config['haproxy']['frontend'].append(
'bind {0} accept-proxy'.format(socket_proxy_path))
else:
# The backend only watchers don't need frontend
# because they have no listen port, so Synapse doens't
# generate a frontend section for them at all
del config['haproxy']['frontend'] # type: ignore
config['haproxy']['backend_name'] = backend_identifier
synapse_config['services'][backend_identifier] = config
if proxy_port is not None:
# If nginx is supported, include a single additional static
# service watcher per service that listens on the right port and
# proxies back to the unix socket exposed by HAProxy
if synapse_tools_config['listen_with_nginx']:
listener_name = '{0}.nginx_listener'.format(service_name)
synapse_config['services'][listener_name] = (
_generate_nginx_for_watcher(
service_name=service_name,
service_info=cast(ServiceInfo, service_info),
synapse_tools_config=synapse_tools_config,
))
# Add HAProxy options for plugins
for plugin_name in PLUGIN_REGISTRY:
plugin_instance = PLUGIN_REGISTRY[plugin_name](
service_name=service_name,
service_info=cast(ServiceInfo, service_info),
synapse_tools_config=synapse_tools_config,
)
config_to_opts = [
(synapse_config['services'][service_name]['haproxy']
['frontend'], plugin_instance.frontend_options(),
plugin_instance.prepend_options('frontend')),
(synapse_config['services'][service_name]['haproxy']
['backend'], plugin_instance.backend_options(),
plugin_instance.prepend_options('backend')),
(synapse_config['haproxy']['global'],
plugin_instance.global_options(),
plugin_instance.prepend_options('global')),
(synapse_config['haproxy']['defaults'],
plugin_instance.defaults_options(),
plugin_instance.prepend_options('defaults'))
]
for (cfg, opts, prepend_options) in config_to_opts:
options = [x for x in opts if x not in cfg]
if prepend_options:
cfg[0:0] += options
else:
cfg.extend(options)
# TODO(jlynch|2017-08-15): move this to a plugin!
# populate the ACLs to route to the service backends, this must
# happen last because ordering of use_backend ACLs matters.
synapse_config['services'][service_name]['haproxy'][
'frontend'].extend(
generate_acls_for_service(
service_name=service_name,
discover_type=discover_type,
advertise_types=advertise_types,
endpoint_timeouts=endpoint_timeouts,
))
return synapse_config
def generate_configuration(synapse_tools_config, zookeeper_topology, services):
synapse_config = generate_base_config(synapse_tools_config)
available_locations = available_location_types()
location_depth_mapping = {
loc: depth
for depth, loc in enumerate(available_locations)
}
available_locations = set(available_locations)
proxies = [
service_info['proxied_through']
for _, service_info in services
if service_info.get('proxied_through') is not None
]
for (service_name, service_info) in services:
proxy_port = service_info.get('proxy_port', -1)
# If we end up with the default value or a negative number in general,
# then we know that the service does not want to be in SmartStack
if proxy_port is not None and proxy_port < 0:
continue
# Note that at this point proxy_port can be:
# * valid number: Wants Load balancing (HAProxy/Nginx)
# * None: Wants discovery, but no load balancing (files)
discover_type = service_info.get('discover', 'region')
advertise_types = sorted(
[
advertise_typ
for advertise_typ in service_info.get('advertise', ['region'])
# don't consider invalid advertise types
if advertise_typ in available_locations
],
key=lambda typ: location_depth_mapping[typ],
reverse=True, # consider the most specific types first
)
if discover_type not in advertise_types:
return {}
base_watcher_cfg = base_watcher_cfg_for_service(
service_name=service_name,
service_info=service_info,
zookeeper_topology=zookeeper_topology,
synapse_tools_config=synapse_tools_config,
is_proxy=service_name in proxies,
)
socket_path = _get_socket_path(
synapse_tools_config, service_name
)
for advertise_type in advertise_types:
backend_identifier = get_backend_name(
service_name, discover_type, advertise_type
)
config = copy.deepcopy(base_watcher_cfg)
config['discovery']['label_filters'] = [
{
'label': '%s:%s' % (advertise_type, get_current_location(advertise_type)),
'value': '',
'condition': 'equals',
},
]
if proxy_port is None:
config['haproxy'] = {'disabled': True}
if synapse_tools_config['listen_with_nginx']:
config['nginx'] = {'disabled': True}
else:
if advertise_type == discover_type:
# Specify a proxy port to create a frontend for this service
if synapse_tools_config['listen_with_haproxy']:
config['haproxy']['port'] = str(proxy_port)
config['haproxy']['frontend'].append(
'bind {0}'.format(socket_path)
)
# If listen_with_haproxy is False, then have
# HAProxy bind only to the socket. Nginx may or may not
# be listening on ports based on listen_with_nginx values
# at this stage.
else:
config['haproxy']['port'] = None
config['haproxy']['bind_address'] = _get_socket_path(
synapse_tools_config, service_name
)
else:
# The backend only watchers don't need frontend
# because they have no listen port, so Synapse doens't
# generate a frontend section for them at all
del config['haproxy']['frontend']
config['haproxy']['backend_name'] = backend_identifier
synapse_config['services'][backend_identifier] = config
if proxy_port is not None:
proxied_through = service_info.get('proxied_through')
healthcheck_uri = service_info.get('healthcheck_uri', '/status')
# populate the ACLs to route to the service backends
synapse_config['services'][service_name]['haproxy']['frontend'].extend(
generate_acls_for_service(
service_name=service_name,
discover_type=discover_type,
advertise_types=advertise_types,
proxied_through=proxied_through,
healthcheck_uri=healthcheck_uri,
)
)
# If nginx is supported, include a single additional static
# service watcher per service that listens on the right port and
# proxies back to the unix socket exposed by HAProxy
if synapse_tools_config['listen_with_nginx']:
listener_name = '{0}.nginx_listener'.format(service_name)
synapse_config['services'][listener_name] = (
_generate_nginx_for_watcher(
service_name, service_info, synapse_tools_config
)
)
return synapse_config
