def test_available_location_types(self, mock_data): location_types = available_location_types() assert isinstance(location_types, list) assert len(location_types) > 0
def generate_configuration(synapse_tools_config, zookeeper_topology, services): synapse_config = generate_base_config(synapse_tools_config) available_locations = available_location_types() location_depth_mapping = { loc: depth for depth, loc in enumerate(available_locations) } available_locations = set(available_locations) for (service_name, service_info) in services: proxy_port = service_info.get('proxy_port') if proxy_port is None: continue discover_type = service_info.get('discover', 'region') advertise_types = sorted( [ advertise_typ for advertise_typ in service_info.get('advertise', ['region']) # don't consider invalid advertise types if advertise_typ in available_locations ], key=lambda typ: location_depth_mapping[typ], reverse=True, # consider the most specific types first ) if discover_type not in advertise_types: return {} base_haproxy_cfg = base_haproxy_cfg_for_service( service_name=service_name, service_info=service_info, zookeeper_topology=zookeeper_topology, synapse_tools_config=synapse_tools_config, ) for advertise_type in advertise_types: config = copy.deepcopy(base_haproxy_cfg) backend_identifier = get_backend_name(service_name, discover_type, advertise_type) if advertise_type == discover_type: # Specify a proxy port to create a frontend for this service config['haproxy']['port'] = str(proxy_port) config['discovery']['label_filters'] = [ { 'label': '%s:%s' % (advertise_type, get_current_location(advertise_type)), 'value': '', 'condition': 'equals', }, ] config['haproxy']['backend_name'] = backend_identifier synapse_config['services'][backend_identifier] = config # populate the ACLs to route to the service backends synapse_config['services'][service_name]['haproxy']['frontend'].extend( generate_acls_for_service( service_name=service_name, discover_type=discover_type, advertise_types=advertise_types, ) ) return synapse_config
def generate_configuration( synapse_tools_config: SynapseToolsConfig, zookeeper_topology: Iterable[str], services: Iterable[Tuple[str, ServiceNamespaceConfig]], ) -> BaseConfig: synapse_config = generate_base_config(synapse_tools_config) available_locations = available_location_types() location_depth_mapping = { loc: depth for depth, loc in enumerate(available_locations) } available_locations = set(available_locations) for (service_name, service_info) in services: proxy_port = service_info.get('proxy_port', -1) # If we end up with the default value or a negative number in general, # then we know that the service does not want to be in SmartStack if proxy_port is not None and proxy_port < 0: continue # Note that at this point proxy_port can be: # * valid number: Wants Load balancing (HAProxy/Nginx) # * None: Wants discovery, but no load balancing (files) discover_type = service_info.get('discover', 'region') advertise_types = sorted( [ advertise_typ for advertise_typ in service_info.get('advertise', ['region']) # don't consider invalid advertise types if advertise_typ in available_locations ], key=lambda typ: location_depth_mapping[typ], reverse=True, # consider the most specific types first ) if discover_type not in advertise_types: return {} base_watcher_cfg = base_watcher_cfg_for_service( service_name=service_name, service_info=cast(ServiceInfo, service_info), zookeeper_topology=zookeeper_topology, synapse_tools_config=synapse_tools_config, ) socket_path = _get_socket_path(synapse_tools_config, service_name) socket_proxy_path = _get_socket_path(synapse_tools_config, service_name, proxy_proto=True) endpoint_timeouts = service_info.get('endpoint_timeouts', {}) for (advertise_type, endpoint_name) in _get_backends_for_service( advertise_types, endpoint_timeouts, ): backend_identifier = get_backend_name(service_name, discover_type, advertise_type, endpoint_name) config = copy.deepcopy(base_watcher_cfg) config['discovery']['label_filters'] = [ { 'label': '%s:%s' % (advertise_type, get_current_location(advertise_type)), 'value': '', 'condition': 'equals', }, ] if endpoint_name != HAPROXY_DEFAULT_SECTION: endpoint_timeout = endpoint_timeouts[endpoint_name] # Override the 'timeout server' value timeout_index_list = [ i for i, v in enumerate(config['haproxy']['backend']) if v.startswith("timeout server ") ] if len(timeout_index_list) > 0: timeout_index = timeout_index_list[0] config['haproxy']['backend'][ timeout_index] = 'timeout server %dms' % endpoint_timeout else: config['haproxy']['backend'].append('timeout server %dms' % endpoint_timeout) if proxy_port is None: config['haproxy'] = {'disabled': True} if synapse_tools_config['listen_with_nginx']: config['nginx'] = {'disabled': True} else: if advertise_type == discover_type and endpoint_name == HAPROXY_DEFAULT_SECTION: # Specify a proxy port to create a frontend for this service if synapse_tools_config['listen_with_haproxy']: config['haproxy']['port'] = str(proxy_port) config['haproxy']['frontend'].extend([ 'bind {0}'.format(socket_path), 'bind {0} accept-proxy'.format(socket_proxy_path), ]) # If listen_with_haproxy is False, then have # HAProxy bind only to the socket. Nginx may or may not # be listening on ports based on listen_with_nginx values # at this stage. else: config['haproxy']['port'] = None config['haproxy']['bind_address'] = socket_path config['haproxy']['frontend'].append( 'bind {0} accept-proxy'.format(socket_proxy_path)) else: # The backend only watchers don't need frontend # because they have no listen port, so Synapse doens't # generate a frontend section for them at all del config['haproxy']['frontend'] # type: ignore config['haproxy']['backend_name'] = backend_identifier synapse_config['services'][backend_identifier] = config if proxy_port is not None: # If nginx is supported, include a single additional static # service watcher per service that listens on the right port and # proxies back to the unix socket exposed by HAProxy if synapse_tools_config['listen_with_nginx']: listener_name = '{0}.nginx_listener'.format(service_name) synapse_config['services'][listener_name] = ( _generate_nginx_for_watcher( service_name=service_name, service_info=cast(ServiceInfo, service_info), synapse_tools_config=synapse_tools_config, )) # Add HAProxy options for plugins for plugin_name in PLUGIN_REGISTRY: plugin_instance = PLUGIN_REGISTRY[plugin_name]( service_name=service_name, service_info=cast(ServiceInfo, service_info), synapse_tools_config=synapse_tools_config, ) config_to_opts = [ (synapse_config['services'][service_name]['haproxy'] ['frontend'], plugin_instance.frontend_options(), plugin_instance.prepend_options('frontend')), (synapse_config['services'][service_name]['haproxy'] ['backend'], plugin_instance.backend_options(), plugin_instance.prepend_options('backend')), (synapse_config['haproxy']['global'], plugin_instance.global_options(), plugin_instance.prepend_options('global')), (synapse_config['haproxy']['defaults'], plugin_instance.defaults_options(), plugin_instance.prepend_options('defaults')) ] for (cfg, opts, prepend_options) in config_to_opts: options = [x for x in opts if x not in cfg] if prepend_options: cfg[0:0] += options else: cfg.extend(options) # TODO(jlynch|2017-08-15): move this to a plugin! # populate the ACLs to route to the service backends, this must # happen last because ordering of use_backend ACLs matters. synapse_config['services'][service_name]['haproxy'][ 'frontend'].extend( generate_acls_for_service( service_name=service_name, discover_type=discover_type, advertise_types=advertise_types, endpoint_timeouts=endpoint_timeouts, )) return synapse_config
def generate_configuration(synapse_tools_config, zookeeper_topology, services): synapse_config = generate_base_config(synapse_tools_config) available_locations = available_location_types() location_depth_mapping = { loc: depth for depth, loc in enumerate(available_locations) } available_locations = set(available_locations) proxies = [ service_info['proxied_through'] for _, service_info in services if service_info.get('proxied_through') is not None ] for (service_name, service_info) in services: proxy_port = service_info.get('proxy_port', -1) # If we end up with the default value or a negative number in general, # then we know that the service does not want to be in SmartStack if proxy_port is not None and proxy_port < 0: continue # Note that at this point proxy_port can be: # * valid number: Wants Load balancing (HAProxy/Nginx) # * None: Wants discovery, but no load balancing (files) discover_type = service_info.get('discover', 'region') advertise_types = sorted( [ advertise_typ for advertise_typ in service_info.get('advertise', ['region']) # don't consider invalid advertise types if advertise_typ in available_locations ], key=lambda typ: location_depth_mapping[typ], reverse=True, # consider the most specific types first ) if discover_type not in advertise_types: return {} base_watcher_cfg = base_watcher_cfg_for_service( service_name=service_name, service_info=service_info, zookeeper_topology=zookeeper_topology, synapse_tools_config=synapse_tools_config, is_proxy=service_name in proxies, ) socket_path = _get_socket_path( synapse_tools_config, service_name ) for advertise_type in advertise_types: backend_identifier = get_backend_name( service_name, discover_type, advertise_type ) config = copy.deepcopy(base_watcher_cfg) config['discovery']['label_filters'] = [ { 'label': '%s:%s' % (advertise_type, get_current_location(advertise_type)), 'value': '', 'condition': 'equals', }, ] if proxy_port is None: config['haproxy'] = {'disabled': True} if synapse_tools_config['listen_with_nginx']: config['nginx'] = {'disabled': True} else: if advertise_type == discover_type: # Specify a proxy port to create a frontend for this service if synapse_tools_config['listen_with_haproxy']: config['haproxy']['port'] = str(proxy_port) config['haproxy']['frontend'].append( 'bind {0}'.format(socket_path) ) # If listen_with_haproxy is False, then have # HAProxy bind only to the socket. Nginx may or may not # be listening on ports based on listen_with_nginx values # at this stage. else: config['haproxy']['port'] = None config['haproxy']['bind_address'] = _get_socket_path( synapse_tools_config, service_name ) else: # The backend only watchers don't need frontend # because they have no listen port, so Synapse doens't # generate a frontend section for them at all del config['haproxy']['frontend'] config['haproxy']['backend_name'] = backend_identifier synapse_config['services'][backend_identifier] = config if proxy_port is not None: proxied_through = service_info.get('proxied_through') healthcheck_uri = service_info.get('healthcheck_uri', '/status') # populate the ACLs to route to the service backends synapse_config['services'][service_name]['haproxy']['frontend'].extend( generate_acls_for_service( service_name=service_name, discover_type=discover_type, advertise_types=advertise_types, proxied_through=proxied_through, healthcheck_uri=healthcheck_uri, ) ) # If nginx is supported, include a single additional static # service watcher per service that listens on the right port and # proxies back to the unix socket exposed by HAProxy if synapse_tools_config['listen_with_nginx']: listener_name = '{0}.nginx_listener'.format(service_name) synapse_config['services'][listener_name] = ( _generate_nginx_for_watcher( service_name, service_info, synapse_tools_config ) ) return synapse_config