def load(config): tunnels = [Tunnel(t) for t in util.flatten(config.get('tunnel', []))] _assert_no_duplicate_remote_endpoint(tunnels) internal_interfaces = [InternalInterface(i, tunnels) for i in util.flatten(config.get('internal_interface', []))] additional_config = _gather_firewall_config(tunnels, internal_interfaces) def run(outdir, diff, no_write, no_exec, no_git): if util.write_file(outdir, 'ipsec-tools.conf', _render_ipsec_tools(tunnels), no_write, diff): if os.geteuid() == 0: util.cmd(('/etc/ipsec-tools.conf',), no_exec) if any([ util.write_file(outdir, 'racoon/psk.txt', _render_racoon_psk(tunnels), no_write, diff), util.write_file(outdir, 'racoon/racoon.conf', _render_racoon(tunnels), no_write, diff), ]): if os.geteuid() == 0: util.cmd(('racoonctl', 'reload-config'), no_exec) if util.write_file(outdir, 'network/interfaces.d/caretakr.ipsec.conf', _render_interfaces(internal_interfaces), no_write, diff=True): logger.warn("Automatic activation of interface changes not implemented!") util.git_commit(outdir, ['ipsec-tools.conf', 'racoon/psk.txt', 'racoon/racoon.conf', 'network/interfaces.d/caretakr.ipsec.conf'], no_git, 'ipsec') return run, additional_config
def __init__(self, config, tunnels): self.name = config.pop('name') self.ovs = config.pop('ovs_gre', False) self.ip = [ipaddr.IPNetwork(i) for i in util.flatten(config.pop('ip', []))] if self.ovs is True: self.ovs_gre_peers = [util.HostIPPort(peer) for peer in util.flatten(config.pop('ovs_gre_peers', []))] self.ovs_bridge_eth = util.flatten(config.pop('ovs_bridge_eth', []))
def __init__(self, config): self.remote_endpoint = ipaddr.IPAddress(config.pop('remote_endpoint')) self.local_endpoint = ipaddr.IPAddress(config.pop('local_endpoint')) self.local_interface = config.pop('local_interface', 'eth0') self.racoon = Racoon(self, config.get('racoon')) self.policy_level = config.pop('policy_level', 'unique') self.remote_subnets = [ipaddr.IPNetwork(s, strict=True) for s in util.flatten(config.pop('remote_subnets'))] self.local_subnets = [ipaddr.IPNetwork(s, strict=True) for s in util.flatten(config.pop('local_subnets'))] self.allow_incoming_traffic = config.pop('allow_incoming_traffic', False) self.snat = config.pop('snat', None)
def _load_virtual_server_group(config): virtual_server_group = [] virtual_server = [] for group_name in sorted(config.keys()): group_config = config[group_name] vs = {} vs['hostips'] = [util.HostIPPort(x) for x in group_config.pop('virtual_server')] vs['ipv4'] = sorted(util.flatten([hostip.ipv4 for hostip in vs['hostips']])) vs['ipv6'] = sorted(util.flatten([hostip.ipv6 for hostip in vs['hostips']])) rs = {} rs['hostips'] = [util.HostIPPort(x) for x in group_config.pop('real_server')] rs['ipv4'] = sorted(util.flatten([hostip.ipv4 for hostip in rs['hostips']])) rs['ipv6'] = sorted(util.flatten([hostip.ipv6 for hostip in rs['hostips']])) drain = {} if 'drain' in group_config: drain['hostip'] = util.HostIPPort(group_config.pop('drain', [])) drain['ipv4'] = drain['hostip'].ipv4 drain['ipv6'] = drain['hostip'].ipv6 for l3proto in ('ipv4', 'ipv6'): invalid_drains = set(drain[l3proto]) - set(rs[l3proto]) if len(invalid_drains) > 0: raise Exception("Drain IP not a real_server: %s" % ','.join(map(str, invalid_drains))) else: drain['hostip'] = [] drain['ipv4'] = [] drain['ipv6'] = [] for l7proto in util.flatten(group_config.pop('proto')): port = port_by_l7proto[l7proto] for l3proto in ('ipv4', 'ipv6'): for l4proto in l4proto_by_l7proto[l7proto]: name = '%s_%s_%s_%s' % (group_name, l3proto, l4proto, l7proto) if len(vs[l3proto]) == 0: logger.debug("Ignoring %s, no virtual ips!" % name) continue if len(rs[l3proto]) == 0: logger.debug("Ignoring %s, no real ips!" % name) continue virtual_server_group.append(VirtualServerGroup(name, vs[l3proto], l3proto, l4proto, l7proto, port)) virtual_server.append(VirtualServer(name, l3proto, l4proto, l7proto, port, rs[l3proto], drain[l3proto])) return virtual_server_group, virtual_server
def check_wildcard(a): if a is None: return None if '*' in util.flatten(a): logger.debug("Emptying address list because it has a wildcard: %s" % a) return [] return a
def add(self, things): for thing in util.flatten(things): if isinstance(thing, firewall.IPTablesRuleset): self.fw.add(thing) elif isinstance(thing, firewall.HostsAllowRuleset): self.ha.add(thing) else: raise Exception("Must be IPTablesRuleset or HostsAllowRuleset: %s" % str(thing))
def __init__(self, name, app_dict, appnode_catchall, ipfilters): if not nginx_util.looks_like_app_name(name): raise ValueError("Invalid application name: %s", name) self.name = name self.server_name = [util.HostIPPort(server_name) for server_name in util.flatten(app_dict.pop('fqdn'))] self.catchall = appnode_catchall[app_dict.pop('catchall')] runtime_port = app_dict.pop('runtime_port', None) if runtime_port is not None: if not nginx_util.looks_like_port(runtime_port): raise ValueError("Invalid runtime port: %s", runtime_port) self.runtime_port = int(runtime_port) else: self.runtime_port = None self.staticfilexas = app_dict.pop('staticfilexas', True) self.mxclientsystem = app_dict.pop('mxclientsystem', None) self.x_frame_options = app_dict.pop('x_frame_options', None) self.authenticate = app_dict.pop('authenticate', 'none') self.mobile = app_dict.pop('mobile', None) if 'ipfilter' in app_dict: self.ipfilter = ipfilters[app_dict.pop('ipfilter')] else: self.ipfilter = None if 'root' in app_dict: self.root = app_dict.pop('root') if not nginx_util.looks_like_path(self.root): raise ValueError("Invalid root path: %s" % self.root) else: self.root = None if 'index' in app_dict: self.index = app_dict.pop('index') if not nginx_util.looks_like_index_html(self.index): raise ValueError("Invalid index name: %s" % self.index) else: self.index = None self.backup_servers = app_dict.pop('backup_server', []) self.client_max_body_size = app_dict.pop('client_max_body_size', '1G') if not nginx_util.looks_like_offset_number(self.client_max_body_size): raise ValueError("Invalid size for client_max_body_size: %s" % self.client_max_body_size) self.unavailable = app_dict.pop('unavailable', False) if not isinstance(self.unavailable, bool): raise ValueError("unavailable option needs True or False as value, " "%s given" % self.unavailable) self.request_handler = [] for suburl, rh_dict in app_dict.pop('request_handler', {}).items(): self.request_handler.append(AppnodeAppRequestHandler(suburl, rh_dict, ipfilters))
def virtual_ip_address(self, ips): ips = util.flatten(ips) ips = (sorted(set([ip for ip in ips if isinstance(ip, ipaddr.IPv4Address)])) + sorted(set([ip for ip in ips if isinstance(ip, ipaddr.IPv6Address)]))) if len(ips) > 0: self._virtual_ipaddress = ips[0] else: logger.error("No VRRP virtual_ipaddress available at all!") self._virtual_ipaddress = None self._virtual_ipaddress_excluded = ips[1:]
def _expand_interfaces(interfaces, depth=0, maxdepth=10): done = True logger.trace("interfaces before: (depth %s)\n%s" % (depth, yaml.dump(interfaces))) interfaces = {k: util.flatten(v) for (k, v) in interfaces.iteritems()} for name in list(sorted(interfaces.keys())): interfaces[name] = util.flatten(interfaces[name]) logger.trace("looking at %s: %s" % (name, interfaces[name])) for i in range(len(interfaces[name])): iface = interfaces[name][i] if isinstance(iface, str) and iface in interfaces and iface != name: interfaces[name][i] = interfaces[iface] logger.trace("replacing %s with %s" % (iface, interfaces[iface])) done = False logger.trace("interfaces after: (depth %s)\n%s" % (depth, yaml.dump(interfaces))) if not done: if depth < maxdepth: interfaces = _expand_interfaces(interfaces, depth+1, maxdepth) else: raise Exception("Maximum recursion depth exceeded for _expand_interfaces") return interfaces
def _custom_rules(custom, interfaces): _expand_interface = lambda iface: interfaces.get(iface, iface) stages = defaultdict(dict) for table, chains in custom.iteritems(): for chain, rules in chains.iteritems(): for rule in rules: stage = rule.pop('stage', 'middle') comment = rule.pop('comment', None) command = rule.pop('command', 'A') i = map(_expand_interface, util.flatten(rule.pop('i', []))) o = map(_expand_interface, util.flatten(rule.pop('o', []))) s = check_wildcard(rule.pop('s', [])) d = check_wildcard(rule.pop('d', [])) r = rule.pop('r', '-j ACCEPT') if len(rule) > 0: logger.warn("Unrecognized options in custom rule: %s" % rule) ruleset = stages[stage].setdefault((table, chain), firewall.IPTablesRuleset(table, chain)) ruleset.add(comment=comment, command=command, i=i, o=o, s=s, d=d, r=r) return stages
def _gather_firewall_config(tunnels, internal_interfaces): ovs_gre_peers = util.flatten([internal_interface.ovs_gre_peer_ips() for internal_interface in internal_interfaces]) if len(ovs_gre_peers) == 0: ovs_gre_peers = None customer_ipsec_gateway = [tunnel.generate_firewall_rules() for tunnel in tunnels] return { 'firewall': { 'gre_peers': ovs_gre_peers, 'customer_ipsec_gateway': customer_ipsec_gateway, } }
def _figure_out_modules(modules_to_load, modules_to_run): if modules_to_load is None or len(modules_to_load) == 0: modules_to_load = known_modules else: modules_to_load = util.flatten(modules_to_load) filtered_to_load = [] for module_name in modules_to_load: if module_name not in known_modules: raise Exception("Unknown module %s!" % module_name) else: filtered_to_load.append(module_name) if modules_to_run is None or len(modules_to_run) == 0: modules_to_run = filtered_to_load else: modules_to_run = util.flatten(modules_to_run) filtered_to_run = [] for module_name in modules_to_run: if module_name not in known_modules: logger.error("Ignoring unknown module %s!" % module_name) elif module_name not in modules_to_load: logger.error("Skipping module run for %s, it's not being loaded!" % module_name) else: filtered_to_run.append(module_name) return filtered_to_load, filtered_to_run
def __init__(self, name, catchall_dict): self.name = name self.ssl = catchall_dict.pop('ssl', name) self.x_frame_options = catchall_dict.pop('x_frame_options', None) self.ssl_client_certificate = catchall_dict.pop('ssl_client_certificate', None) self.ssl_crl = catchall_dict.pop('ssl_crl', None) self.ssl_verify_client = catchall_dict.pop('ssl_verify_client', 'on') self.listen = [] for listen_item in util.flatten(catchall_dict.pop('listen', [])): listen_hostipport = util.HostIPPort(listen_item) if len(listen_hostipport.addresses) == 0: raise Exception("No listen addresses found for: %s (%s)!" % (listen_item, self._name)) self.listen.append(listen_hostipport)
def __init__(self, name, redir_dict, ff_ssl_catchall): self.name = name self.server_name = util.HostIPPort(redir_dict.pop('fqdn')) self.redirect = redir_dict.pop('redirect') self.catchall = find_catchall_for_app(self.server_name.fqdn, redir_dict.pop('catchall', None), ff_ssl_catchall) if 'hsts_max_age' in redir_dict: self.hsts_max_age = int(redir_dict.pop('hsts_max_age')) else: self.hsts_max_age = None self.ssl = redir_dict.pop('ssl', self.catchall.ssl) server_name_ips = self.server_name.addresses catchall_ips = util.flatten([listen.addresses for listen in self.catchall.listen]) if len(set(server_name_ips) - set(catchall_ips)) > 0: logger.warn("%s %s (%s) resolves to [%s] but catchall (%s) listens on [%s]" % (type(self).__name__, self._name, self.server_name.fqdn, ', '.join([str(x) for x in server_name_ips]), self.catchall.name, ', '.join([str(x) for x in catchall_ips])))
def __init__(self, name, app_dict, ff_ssl_catchall): self.name = name self.appnode = util.HostIPPort(app_dict.pop('appnode')) self.server_name = util.HostIPPort(app_dict.pop('fqdn')) self.catchall = find_catchall_for_app(self.server_name.fqdn, app_dict.pop('catchall', None), ff_ssl_catchall) self.client_max_body_size = app_dict.pop('client_max_body_size', '1024M') self.hsts_max_age = app_dict.pop('hsts_max_age', None) self.x_frame_options = app_dict.pop('x_frame_options', None) self.ssl = app_dict.pop('ssl', self.catchall.ssl) self.ssl_client_certificate = app_dict.pop('ssl_client_certificate', None) self.ssl_crl = app_dict.pop('ssl_crl', None) self.ssl_verify_client = app_dict.pop('ssl_verify_client', 'on') server_name_ips = self.server_name.addresses catchall_ips = util.flatten([listen.addresses for listen in self.catchall.listen]) if len(set(server_name_ips) - set(catchall_ips)) > 0: logger.warn("%s %s (%s) resolves to [%s] but catchall (%s) listens on [%s]" % (type(self).__name__, self.name, self.server_name.fqdn, ', '.join([str(x) for x in server_name_ips]), self.catchall.name, ', '.join([str(x) for x in catchall_ips])))
def load(config): logger.trace("config: %s" % config) icmp_stateful = config.pop('icmp_stateful', True) ipv6_nat = config.pop('ipv6_nat', False) interfaces = _expand_interfaces(config.pop('interfaces', {})) input_ssh_from = check_wildcard(config.pop('input_ssh_from', [])) input_munin_from = check_wildcard(config.pop('input_munin_from', None)) input_nrpe_from = check_wildcard(config.pop('input_nrpe_from', None)) input_dns_from = check_wildcard(config.pop('input_dns_from', None)) input_dns_notrack = config.pop('input_dns_notrack', False) input_http_from = check_wildcard(config.pop('input_http_from', None)) input_http_port = check_wildcard(config.pop('input_http_port', [80, 443])) input_http_notrack = config.pop('input_http_notrack', False) input_smtp_from = check_wildcard(config.pop('input_smtp_from', None)) input_postgresql_from = check_wildcard(config.pop('input_postgresql_from', None)) input_xmpp_c2s_from = check_wildcard(config.pop('input_xmpp_c2s_from', None)) input_nsca_from = check_wildcard(config.pop('input_nsca_from', None)) output_dns_to = check_wildcard(config.pop('output_dns_to', [])) output_ntp_to = check_wildcard(config.pop('output_ntp_to', [])) output_http_to = check_wildcard(config.pop('output_http_to', None)) output_http_port = check_wildcard(config.pop('output_http_port', [80, 443])) output_smtp_to = check_wildcard(config.pop('output_smtp_to', [])) output_munin_to = check_wildcard(config.pop('output_munin_to', None)) output_nsca_to = check_wildcard(config.pop('output_nsca_to', None)) output_postgresql_to = check_wildcard(config.pop('output_postgresql_to', None)) output_xmpp_c2s_to = check_wildcard(config.pop('output_xmpp_c2s_to', None)) output_user = config.pop('output_user', None) output_group = config.pop('output_group', None) xmpp_s2s_fromto = check_wildcard(config.pop('xmpp_s2s_fromto', None)) gre_peers = check_wildcard(config.pop('gre_peers', None)) if any([input_dns_notrack, input_http_notrack]): logger.debug("notrack option set, early accepting icmp") icmp_stateful = False input_tcp = config.pop('input_tcp', []) input_udp = config.pop('input_udp', []) output_tcp = config.pop('output_tcp', []) output_udp = config.pop('output_udp', []) custom = _custom_rules(config.pop('custom', {}), interfaces) input_log_ignore = config.pop('input_log_ignore', None) input_log_limit = config.pop('input_log_limit', None) input_log_burst = config.pop('input_log_burst', None) output_log_ignore = config.pop('output_log_ignore', None) output_log_limit = config.pop('output_log_limit', None) output_log_burst = config.pop('output_log_burst', None) forwarding = False stateful_forward = config.pop('stateful_forward', True) customer_ipsec_gateway = util.flatten(config.pop('customer_ipsec_gateway', [])) if len(customer_ipsec_gateway) > 0: logger.debug("customer ipsec gateway set, enabling forwarding") forwarding = True fh = IPTablesHostsAllow() fh.fw.enable_ipv6_nat(ipv6_nat) logger.debug("creating rules for early phase") fh.add(libow.early_lo()) if input_dns_from is not None and input_dns_notrack is True: fh.add(libow.input_dns(input_dns_from, input_dns_notrack)) if input_http_from is not None and input_http_notrack is True: fh.add(libow.input_http(input_http_from, input_http_port, input_http_notrack)) fh.add(custom['early'].values()) logger.debug("creating rules for start phase") if icmp_stateful is False: fh.add(libow.input_icmp()) fh.add(libow.output_icmp()) fh.add(libow.start('INPUT')) fh.add(libow.start('OUTPUT')) if icmp_stateful is True: fh.add(libow.stateful_input_icmp()) fh.add(libow.output_icmp()) if forwarding is True: if stateful_forward is True: fh.add(libow.start('FORWARD')) fh.add(libow.forward_icmp()) fh.add(custom['start'].values()) logger.debug("creating rules for middle phase") if input_ssh_from is not None: fh.add(libow.input_ssh(input_ssh_from)) if input_munin_from is not None: fh.add(libow.input_tcp(input_munin_from, 4949)) if input_nrpe_from is not None: fh.add(libow.input_nrpe(input_nrpe_from)) if input_dns_from is not None and input_dns_notrack is False: fh.add(libow.input_dns(input_dns_from, input_dns_notrack)) if input_http_from is not None and input_http_notrack is False: fh.add(libow.input_http(input_http_from, input_http_port, input_http_notrack)) if input_smtp_from is not None: fh.add(libow.input_tcp(input_smtp_from, 25)) if input_postgresql_from is not None: fh.add(libow.input_tcp(input_postgresql_from, 5432)) if input_xmpp_c2s_from is not None: fh.add(libow.input_tcp(input_xmpp_c2s_from, 5222)) fh.add(libow.input_tcp([], 7777)) # XXX: unfiltered? for rule in input_tcp: comment = rule.get('comment', None) s = check_wildcard(rule.get('from', [])) port = rule.get('port', None) fh.add(libow.input_tcp(comment=comment, s=s, port=port)) for rule in input_udp: comment = rule.get('comment', None) s = check_wildcard(rule.get('from', [])) port = rule.get('port', None) fh.add(libow.input_udp(comment=comment, s=s, port=port)) if input_nsca_from is not None: fh.add(libow.input_tcp(input_nsca_from, 5667)) if xmpp_s2s_fromto is not None: fh.add(libow.input_tcp(xmpp_s2s_fromto, 5269)) if gre_peers is not None: fh.add(libow.input_gre(gre_peers)) if output_dns_to is not None: fh.add(libow.output_dns(output_dns_to)) if output_ntp_to is not None: fh.add(libow.output_udp(output_ntp_to, 123, uid='ntp')) if output_http_to is not None: fh.add(libow.output_http(output_http_to, output_http_port)) if output_smtp_to is not None: fh.add(libow.output_tcp(output_smtp_to, 25, uid='postfix')) if output_munin_to is not None: fh.add(libow.output_tcp(output_munin_to, 4949, uid='munin')) if output_nsca_to is not None: fh.add(libow.output_tcp(output_nsca_to, 5667)) if output_postgresql_to is not None: fh.add(libow.output_tcp(output_postgresql_to, 5432)) if output_xmpp_c2s_to is not None: fh.add(libow.output_tcp(output_xmpp_c2s_to, 5222)) if xmpp_s2s_fromto is not None: fh.add(libow.output_tcp(xmpp_s2s_fromto, 5269, uid='ejabberd')) if gre_peers is not None: fh.add(libow.output_gre(gre_peers)) for rule in output_tcp: comment = rule.get('comment', None) d = check_wildcard(rule.get('to', [])) port = rule.get('port', None) uid = rule.get('uid', None) fh.add(libow.output_tcp(comment=comment, d=d, port=port, uid=uid)) for rule in output_udp: comment = rule.get('comment', None) d = check_wildcard(rule.get('to', [])) port = rule.get('port', None) uid = rule.get('uid', None) fh.add(libow.output_udp(comment=comment, d=d, port=port, uid=uid)) if output_user is not None: output_user = util.flatten(output_user) for user in output_user: if isinstance(user, str): fh.add(libow.output_tcp(d=[], uid=user)) elif isinstance(user, int): fh.add(libow.output_tcp(d=[], uid=str(user))) else: raise Exception("don't understand output_user %s" % user) if output_group is not None: output_group = util.flatten(output_group) for group in output_group: if isinstance(group, str): fh.add(libow.output_tcp(d=[], gid=group)) elif isinstance(group, int): fh.add(libow.output_tcp(d=[], gid=str(group))) else: raise Exception("don't understand output_group %s" % group) if len(customer_ipsec_gateway) > 0: for cig in customer_ipsec_gateway: fh.add(libow.customer_ipsec_gateway( remote_endpoint=cig['remote_endpoint'], remote_subnets=cig['remote_subnets'], local_subnets=cig['local_subnets'], allow_incoming_traffic=cig.get('allow_incoming_traffic', False), snat=cig.get('snat', None) )) fh.add(custom['middle'].values()) logger.debug("creating rules for end phase") fh.add(libow.input_traceroute()) fh.add(libow.output_beheer_root()) fh.add(custom['end'].values()) logger.debug("creating rules for late phase") fh.add(libow.log('INPUT', input_log_ignore, input_log_limit, input_log_burst)) fh.add(libow.log('OUTPUT', output_log_ignore, output_log_limit, output_log_burst)) fh.add(custom['late'].values()) for hosts_allow in config.pop('hosts_allow', []): daemon = hosts_allow.pop('daemon') s = hosts_allow.pop('from', None) if '*' in util.flatten(s): s = 'all' if len(hosts_allow) > 0: logger.warn("Unrecognized options in hosts_allow rule: %s" % hosts_allow) fh.add(libow.hosts_allow(daemon, s)) firewall_lines = fh.fw.get_iptables_restore_script(4).splitlines() firewall6_lines = fh.fw.get_iptables_restore_script(6).splitlines() hosts_allow_lines = fh.ha.get_hosts_allow_content().splitlines() additional_config = None def run(outdir, diff, no_write, no_exec, no_git): if any([ util.write_file(outdir, 'network/firewall', firewall_lines, no_write, diff), util.write_file(outdir, 'network/firewall6', firewall6_lines, no_write, diff), util.write_file(outdir, 'hosts.allow', hosts_allow_lines, no_write, diff), ]): if os.geteuid() == 0: if no_exec is True: util.cmd(('/sbin/iptables-restore', '-t'), dry_run=False, stdin=firewall_lines) util.cmd(('/sbin/ip6tables-restore', '-t'), dry_run=False, stdin=firewall6_lines) else: util.cmd(('/sbin/iptables-restore'), dry_run=False, stdin=firewall_lines) util.cmd(('/sbin/ip6tables-restore'), dry_run=False, stdin=firewall6_lines) util.git_commit(outdir, ['network/firewall', 'network/firewall6', 'hosts.allow'], no_git, 'firewall') return run, additional_config
def server_name(self, server_name): self._server_name = util.flatten(server_name)
def listen(self, listen): listen = util.flatten(listen) if not all([isinstance(l, util.HostIPPort) for l in listen]): raise ValueError("Server listen address needs to be HostIPPort") self._listen = listen
def server(self, server): self._server = util.flatten(server)
def backup_server(self, backup_server): self._backup_server = util.flatten(backup_server)