def make_vcl_query_regex(inp, match_function, ignore_case): reg_query_beg = "(^|&)" reg_query_param_end = "(=|&|$)" reg_query_end = "(&|$)" reg_query_wc = r"[^&]*" regex = None esc_param = vcl_escape_string_to_regex(inp["parameter"]) if match_function == "exist": regex = reg_query_beg + esc_param + reg_query_param_end else: value = inp["value"] if match_function == "regex": regex = (reg_query_beg + esc_param + "=" + vcl_escape_regex(value) + reg_query_end) else: esc_value = vcl_escape_string_to_regex(value) if match_function == "exact": regex = reg_query_beg + esc_param + "=" + esc_value + reg_query_end elif match_function == "begins_with": regex = (reg_query_beg + esc_param + "=" + esc_value + reg_query_wc + reg_query_end) elif match_function == "ends_with": regex = (reg_query_beg + esc_param + "=" + reg_query_wc + esc_value + reg_query_end) elif match_function == "contains": regex = (reg_query_beg + esc_param + "=" + reg_query_wc + esc_value + reg_query_wc + reg_query_end) if regex is None: raise ORMInternalRenderException("ERROR: unhandled query match " "function: " + match_function + ":" + str(inp)) return vcl_regex_add_opts(regex, ignore_case)
def make_trailing_slash_action(config_in, config_out, rule_id, indent_depth=0): # pylint:disable=unused-argument regex = "" action = [] reg_path_without_trailing = "(?:/[^/?#]+)*" reg_post_path = "[#?].*" if config_in == "add": # Will match all paths without trailing slash, # when the last part begins with a period or contains no periods. reg_path_end = r"/(?:\.?[^/?#.]+)" regex = ("^(" + reg_path_without_trailing + reg_path_end + ")" "(" + reg_post_path + ")?$") sub = r"\1/\2" elif config_in == "remove": # Will match all paths with a trailing slash. regex = "^(" + reg_path_without_trailing + ")/" "(" + reg_post_path + ")?$" sub = r"\1\2" elif config_in == "do_nothing": return config_out else: raise ORMInternalRenderException("ERROR: unhandled " + "trailing slash action: " + config_in) action.append(indent(indent_depth) + 'if (req.url ~ "' + regex + '") {') action.append( indent(indent_depth + 1) + "return (synth(307, " 'regsub(req.url, "' + regex + '", "' + sub + '")));') action.append(indent(indent_depth) + "}") config_out["sb"] = action return config_out
def make_trailing_slash_action(config_in, config_out, rule_id, indent_depth=0): #pylint:disable=unused-argument regex = '' action = [] reg_path_without_trailing = '(?:/[^/?#]+)*' reg_post_path = '[#?].*' if config_in == 'add': # Will match all paths without trailing slash, # when the last part begins with a period or contains no periods. reg_path_end = r'/(?:\.?[^/?#.]+)' regex = ('^(' + reg_path_without_trailing + reg_path_end + ')' '(' + reg_post_path + ')?$') sub = r'\1/\2' elif config_in == 'remove': # Will match all paths with a trailing slash. regex = ('^(' + reg_path_without_trailing + ')/' '(' + reg_post_path + ')?$') sub = r'\1\2' elif config_in == 'do_nothing': return config_out else: raise ORMInternalRenderException('ERROR: unhandled ' + 'trailing slash action: ' + config_in) action.append(indent(indent_depth) + 'if (req.url ~ "' + regex + '") {') action.append( indent(indent_depth + 1) + 'return (synth(307, ' 'regsub(req.url, "' + regex + '", "' + sub + '")));') action.append(indent(indent_depth) + '}') config_out['sb'] = action return config_out
def make_vcl_query_regex(inp, match_function, ignore_case): reg_query_beg = '(^|&)' reg_query_param_end = '(=|&|$)' reg_query_end = '(&|$)' reg_query_wc = r'[^&]*' regex = None esc_param = vcl_escape_string_to_regex(inp['parameter']) if match_function == 'exist': regex = reg_query_beg + esc_param + reg_query_param_end else: value = inp['value'] if match_function == 'regex': regex = (reg_query_beg + esc_param + '=' + vcl_escape_regex(value) + reg_query_end) else: esc_value = vcl_escape_string_to_regex(value) if match_function == 'exact': regex = (reg_query_beg + esc_param + '=' + esc_value + reg_query_end) elif match_function == 'begins_with': regex = (reg_query_beg + esc_param + '=' + esc_value + reg_query_wc + reg_query_end) elif match_function == 'ends_with': regex = (reg_query_beg + esc_param + '=' + reg_query_wc + esc_value + reg_query_end) elif match_function == 'contains': regex = (reg_query_beg + esc_param + '=' + reg_query_wc + esc_value + reg_query_wc + reg_query_end) if regex is None: raise ORMInternalRenderException('ERROR: unhandled query match ' 'function: ' + match_function + ':' + str(inp)) return vcl_regex_add_opts(regex, ignore_case)
def make_backend_action(self, backend_config, rule_id): origins = [] if "origin" in backend_config: origins.append(backend_config["origin"]) elif "servers" in backend_config: origins += backend_config["servers"] else: raise ORMInternalRenderException( "ERROR: unhandled backend type: " + str(backend_config.keys())) if origins: self.backend_acls.append(" use_backend " + rule_id + " " "if { hdr(X-ORM-ID) -m str " + rule_id + " }") self.backends.append("") self.backends.append("backend " + rule_id) for origin in origins: origin_instance = origin if isinstance(origin, str): origin_instance = {"server": origin} scheme, hostname, port = parser.extract_from_origin( origin_instance["server"]) server = (" server " + parser.normalize(origin_instance["server"]) + " " + hostname + ":" + port + " resolvers dns resolve-prefer ipv4" + " check") if scheme == "https": # TODO: add 'verify required sni ca-file verifyhost' server += " ssl verify none" elif scheme != "http": raise ORMInternalRenderException("ERROR: unhandled origin " "scheme: " + scheme) if origin_instance.get("max_connections", False): server += " maxconn {}".format( origin_instance["max_connections"]) if origin_instance.get("max_queued_connections", False): server += " maxqueue {}".format( origin_instance["max_queued_connections"]) self.backends.append(server)
def make_backend_action(self, backend_config, rule_id): origins = [] if 'origin' in backend_config: origins.append(backend_config['origin']) elif 'servers' in backend_config: origins += backend_config['servers'] else: raise ORMInternalRenderException( 'ERROR: unhandled backend type: ' + str(backend_config.keys())) if origins: self.backend_acls.append(' use_backend ' + rule_id + ' ' 'if { hdr(X-ORM-ID) -m str ' + rule_id + ' }') self.backends.append('') self.backends.append('backend ' + rule_id) for origin in origins: origin_instance = origin if isinstance(origin, str): origin_instance = {'server': origin} scheme, hostname, port = parser.extract_from_origin( origin_instance['server']) server = (' server ' + parser.normalize(origin_instance['server']) + ' ' + hostname + ':' + port + ' resolvers dns resolve-prefer ipv4' + ' check') if scheme == 'https': # TODO: add 'verify required sni ca-file verifyhost' server += ' ssl verify none' elif scheme != 'http': raise ORMInternalRenderException('ERROR: unhandled origin ' 'scheme: ' + scheme) if origin_instance.get('max_connections', False): server += ' maxconn {}'.format( origin_instance['max_connections']) if origin_instance.get('max_queued_connections', False): server += ' maxqueue {}'.format( origin_instance['max_queued_connections']) self.backends.append(server)
def make_condition_match(self, match): src = match["source"] fun = match["function"] inp = match["input"] if src == "path": return self.make_match_path(fun, inp) if src == "domain": return self.make_match_domain(fun, inp) if src == "query": return self.make_match_query(fun, inp) raise ORMInternalRenderException("ERROR: unhandled match source: " + src + ":" + fun + ":" + str(inp))
def make_condition_match(self, match): src = match['source'] fun = match['function'] inp = match['input'] if src == 'path': return self.make_match_path(fun, inp) if src == 'domain': return self.make_match_domain(fun, inp) if src == 'query': return self.make_match_query(fun, inp) raise ORMInternalRenderException('ERROR: unhandled match source: ' + src + ':' + fun + ':' + str(inp))
def make_path_mod_actions(config_in, config_out, rule_id, indent_depth=0): # pylint:disable=unused-argument actions = [] for action in config_in: if "replace" in action: conf = action["replace"] actions += make_path_replace_action(conf, indent_depth=indent_depth) elif "prefix" in action: conf = action["prefix"] actions += make_path_prefix_action(conf, indent_depth=indent_depth) else: raise ORMInternalRenderException("ERROR: unhandled path_mod " "action type: " + action) config_out["sb"] += actions return config_out
def make_path_mod_actions(config_in, config_out, rule_id, indent_depth=0): #pylint:disable=unused-argument actions = [] for action in config_in: if 'replace' in action: conf = action['replace'] actions += make_path_replace_action(conf, indent_depth=indent_depth) elif 'prefix' in action: conf = action['prefix'] actions += make_path_prefix_action(conf, indent_depth=indent_depth) else: raise ORMInternalRenderException('ERROR: unhandled path_mod ' 'action type: ' + action) config_out['sb'] += actions return config_out
def make_custom_internal_healthcheck(healthcheck_config): config = [] if not healthcheck_config: return config if 'http' in healthcheck_config: http_config = healthcheck_config['http'] path = http_config['path'] method = http_config.get('method', 'GET') check_option = 'option httpchk {} {}'.format(method, path) domain = http_config.get('domain', None) if domain: check_option += r' HTTP/1.1\nHost:\ ' + domain config.append(' ' + check_option) config.append(' http-check expect ! rstatus ^5') else: raise ORMInternalRenderException('ERROR: unhandled ' 'custom_internal_healthcheck type: ' + healthcheck_config.keys()) return config
def make_vcl_path_regex(value, match_function, ignore_case): regex = None if match_function == "regex": regex = "^{}$".format(vcl_escape_regex(value)) else: escaped_value = vcl_escape_string_to_regex(value) if match_function == "exact": regex = "^{}$".format(escaped_value) elif match_function == "begins_with": regex = "^{}.*$".format(escaped_value) elif match_function == "ends_with": regex = "^.*{}$".format(escaped_value) elif match_function == "contains": regex = "^.*{}.*$".format(escaped_value) if regex is None: raise ORMInternalRenderException("ERROR: unhandled path match " "function: " + match_function + ":" + str(value)) return vcl_regex_add_opts(regex, ignore_case)
def make_custom_internal_healthcheck(healthcheck_config): config = [] if not healthcheck_config: return config if "http" in healthcheck_config: http_config = healthcheck_config["http"] path = http_config["path"] method = http_config.get("method", "GET") check_option = "option httpchk {} {}".format(method, path) domain = http_config.get("domain", None) if domain: check_option += r" HTTP/1.1\nHost:\ " + domain config.append(" " + check_option) config.append(" http-check expect ! rstatus ^5") else: raise ORMInternalRenderException("ERROR: unhandled " "custom_internal_healthcheck type: " + healthcheck_config.keys()) return config
def make_vcl_path_regex(value, match_function, ignore_case): regex = None if match_function == 'regex': regex = '^{}$'.format(vcl_escape_regex(value)) else: escaped_value = vcl_escape_string_to_regex(value) if match_function == 'exact': regex = '^{}$'.format(escaped_value) elif match_function == 'begins_with': regex = '^{}.*$'.format(escaped_value) elif match_function == 'ends_with': regex = '^.*{}$'.format(escaped_value) elif match_function == 'contains': regex = '^.*{}.*$'.format(escaped_value) if regex is None: raise ORMInternalRenderException('ERROR: unhandled path match ' 'function: ' + match_function + ':' + str(value)) return vcl_regex_add_opts(regex, ignore_case)
def make_header_action(config_in, config_out, rule_id, indent_depth=0, southbound=True): # pylint:disable=unused-argument hdr_var = "req" if southbound else "resp" actions = [] for header_action in config_in: # field name is validated by schema in validator. I guess any # valid field name is valid to use in Varnish VCL as well. if "remove" in header_action: field = header_action["remove"] actions.append( indent(indent_depth) + "unset " + hdr_var + ".http." + field + ";") elif "set" in header_action: field = header_action["set"]["field"] value = header_action["set"]["value"] actions.append( indent(indent_depth) + "set " + hdr_var + ".http." + field + " = " + vcl_safe_string(value) + ";") elif "add" in header_action: field = header_action["add"]["field"] value = header_action["add"]["value"] actions.append( indent(indent_depth) + "if (" + hdr_var + ".http." + field + ") {") actions.append( indent(indent_depth + 1) + "set " + hdr_var + ".http." + field + " = " + hdr_var + ".http." + field + ' + ",";') actions.append(indent(indent_depth) + "}") actions.append( indent(indent_depth) + "set " + hdr_var + ".http." + field + " = " + hdr_var + ".http." + field + " + " + vcl_safe_string(value) + ";") else: raise ORMInternalRenderException("ERROR: unhandled header " "action type: " + str(header_action.keys())) config_out_key = "sb" if southbound else "nb" config_out[config_out_key] = actions return config_out
def make_header_action(config_in, config_out, rule_id, indent_depth=0, southbound=True): #pylint:disable=unused-argument hdr_var = 'req' if southbound else 'resp' actions = [] for header_action in config_in: # field name is validated by schema in validator. I guess any # valid field name is valid to use in Varnish VCL as well. if 'remove' in header_action: field = header_action['remove'] actions.append( indent(indent_depth) + 'unset ' + hdr_var + '.http.' + field + ';') elif 'set' in header_action: field = header_action['set']['field'] value = header_action['set']['value'] actions.append( indent(indent_depth) + 'set ' + hdr_var + '.http.' + field + ' = ' + vcl_safe_string(value) + ';') elif 'add' in header_action: field = header_action['add']['field'] value = header_action['add']['value'] actions.append( indent(indent_depth) + 'if (' + hdr_var + '.http.' + field + ') {') actions.append( indent(indent_depth + 1) + 'set ' + hdr_var + '.http.' + field + ' = ' + hdr_var + '.http.' + field + ' + ",";') actions.append(indent(indent_depth) + '}') actions.append( indent(indent_depth) + 'set ' + hdr_var + '.http.' + field + ' = ' + hdr_var + '.http.' + field + ' + ' + vcl_safe_string(value) + ';') else: raise ORMInternalRenderException('ERROR: unhandled header ' 'action type: ' + str(header_action.keys())) config_out_key = 'sb' if southbound else 'nb' config_out[config_out_key] = actions return config_out
def parse_match_tree(self, match_tree, indent_depth=0, negate=False): if "and" in match_tree: return self.handle_condition_list("&&", match_tree["and"], negate=negate, indent_depth=indent_depth) if "or" in match_tree: return self.handle_condition_list("||", match_tree["or"], negate=negate, indent_depth=indent_depth) if "match" in match_tree: opt_negate = "!" if negate else "" return opt_negate + self.make_condition_match(match_tree["match"]) if "not" in match_tree: return self.parse_match_tree(match_tree["not"], indent_depth=indent_depth, negate=True) raise ORMInternalRenderException( "ERROR: unhandled condition operator: " + str(match_tree.keys))
def make_path_replace_action(replace_config, indent_depth=0): # pylint:disable=unused-argument ignore_case = replace_config.get("ignore_case", False) vcl_regex = None vcl_sub = replace_config.get("to", None) if "from_regex" in replace_config: vcl_regex = make_vcl_value_regex("path", replace_config["from_regex"], "regex", ignore_case) vcl_sub = replace_config.get("to_regsub", vcl_sub) elif "from_exact" in replace_config: vcl_regex = make_vcl_value_regex("path", replace_config["from_exact"], "exact", ignore_case) if vcl_sub is None or vcl_regex is None: raise ORMInternalRenderException("ERROR: could not generate " "substitution using replace config " "keys: " + str(replace_config.keys())) return [ indent(indent_depth) + 'variable.set("path", ' 'regsub(variable.get("path"), {reg}, {sub}));'.format( reg=vcl_safe_string(vcl_regex), sub=vcl_safe_string(vcl_sub)) ]
def make_path_replace_action(replace_config, indent_depth=0): #pylint:disable=unused-argument ignore_case = replace_config.get('ignore_case', False) vcl_regex = None vcl_sub = replace_config.get('to', None) if 'from_regex' in replace_config: vcl_regex = make_vcl_path_regex(replace_config['from_regex'], 'regex', ignore_case) vcl_sub = replace_config.get('to_regsub', vcl_sub) elif 'from_exact' in replace_config: vcl_regex = make_vcl_path_regex(replace_config['from_exact'], 'exact', ignore_case) if vcl_sub is None or vcl_regex is None: raise ORMInternalRenderException('ERROR: could not generate ' 'substitution using replace config ' 'keys: ' + str(replace_config.keys())) return [ indent(indent_depth) + 'variable.set("path", ' 'regsub(variable.get("path"), {reg}, {sub}));'.format( reg=vcl_safe_string(vcl_regex), sub=vcl_safe_string(vcl_sub)) ]
def parse_match_tree(self, match_tree, indent_depth=0, negate=False): if 'and' in match_tree: return self.handle_condition_list('&&', match_tree['and'], negate=negate, indent_depth=indent_depth) if 'or' in match_tree: return self.handle_condition_list('||', match_tree['or'], negate=negate, indent_depth=indent_depth) if 'match' in match_tree: opt_negate = '!' if negate else '' return (opt_negate + self.make_condition_match(match_tree['match'])) if 'not' in match_tree: return self.parse_match_tree(match_tree['not'], indent_depth=indent_depth, negate=True) raise ORMInternalRenderException( 'ERROR: unhandled condition operator: ' + str(match_tree.keys))
def make_actions( self, action_config, rule_id, domain=None, match_sub_name=None, indent_depth=0, is_global=False, ): # pylint:disable=too-many-locals,too-many-arguments,too-many-branches # pylint:disable=too-many-statements if not domain and not is_global: raise ORMInternalRenderException("ERROR: One of domain and " + "is_global must be set!") # Order is important supported_actions = [ { "name": "https_redirection", "func": make_https_redir_action, }, # Must be first { "name": "trailing_slash", "func": make_trailing_slash_action, }, # Must be second { "name": "synthetic_response", "func": make_synth_resp_action }, { "name": "redirect", "func": make_redirect_action }, { "name": "header_southbound", "func": make_sb_header_action }, { "name": "req_path", "func": make_path_mod_actions }, { "name": "backend", "func": make_backend_action }, { "name": "header_northbound", "func": make_nb_header_action }, ] for action_name in action_config: present = False for supported_action in supported_actions: if action_name == supported_action["name"]: present = True if not present: raise ORMInternalRenderException("ERROR: unhandled " + "action type: " + action_name) if "backend" in action_config: self.uses_sub_use_backend = True if "redirect" in action_config and "url" not in action_config: self.uses_sub_reconstruct_requrl = True sb = [] nb = [] for action in supported_actions: action_name = action["name"] action_function = action["func"] if action_name not in action_config: continue config_out = {"sb": [], "nb": [], "synth": []} action_function( config_in=action_config[action_name], config_out=config_out, rule_id=rule_id, indent_depth=indent_depth + 1, ) sb += config_out["sb"] nb += config_out["nb"] self.synthetic_responses += config_out["synth"] if is_global: if sb: self.global_actions_southbound += sb if nb: self.global_actions_northbound += nb else: if sb: sb.insert( 0, indent(indent_depth + 1) + "call global_actions_southbound;") if match_sub_name: actions = [] actions.append((indent(indent_depth) + "call " + match_sub_name + ";")) actions += make_action_if_clause(sb, rule_id, indent_depth=indent_depth) self.actions_southbound.setdefault(domain, []) self.actions_southbound[domain] += actions else: # If there is no match_sub_name, it is a default rule actions = [] # Set variable when default actions are reached in # vcl_recv (southbound) so we know whether to perform # the default northbound actions actions.append( indent(indent_depth + 1) + make_vcl_set_match_variable(rule_id)) actions += sb self.default_actions_southbound.setdefault(domain, []) self.default_actions_southbound[domain] += actions if nb: if match_sub_name: actions = make_action_if_clause(nb, rule_id, indent_depth=indent_depth) self.actions_northbound.setdefault(domain, []) self.actions_northbound[domain] += actions else: # If there is no match_sub_name, it is a default rule # Only perform default northbound actions if variable is set # (that is, only when the default southbound actions did) actions = make_action_if_clause(nb, rule_id, indent_depth=indent_depth) self.default_actions_northbound.setdefault(domain, []) self.default_actions_northbound[domain] += actions # Return True if we generated an action return config_out["sb"] or config_out["nb"] or config_out["synth"]
def make_match_domain(self, fun, inp): if fun == "exact": return "req.http.host == " + vcl_safe_string(inp) raise ORMInternalRenderException("ERROR: unhandled domain match: " + fun + ":" + str(inp))
def make_match_domain(self, fun, inp): if fun == 'exact': return 'req.http.host == ' + vcl_safe_string(inp) raise ORMInternalRenderException('ERROR: unhandled domain match: ' + fun + ':' + str(inp))
def make_actions(self, action_config, rule_id, domain=None, match_sub_name=None, indent_depth=0, is_global=False): #pylint:disable=too-many-locals,too-many-arguments,too-many-branches #pylint:disable=too-many-statements if not domain and not is_global: raise ORMInternalRenderException('ERROR: One of domain and ' + 'is_global must be set!') # Order is important supported_actions = [ { 'name': 'https_redirection', 'func': make_https_redir_action }, # Must be first { 'name': 'trailing_slash', 'func': make_trailing_slash_action }, # Must be second { 'name': 'synthetic_response', 'func': make_synth_resp_action }, { 'name': 'redirect', 'func': make_redirect_action }, { 'name': 'header_southbound', 'func': make_sb_header_action }, { 'name': 'req_path', 'func': make_path_mod_actions }, { 'name': 'backend', 'func': make_backend_action }, { 'name': 'header_northbound', 'func': make_nb_header_action } ] for action_name in action_config: present = False for supported_action in supported_actions: if action_name == supported_action['name']: present = True if not present: raise ORMInternalRenderException('ERROR: unhandled ' + 'action type: ' + action_name) if 'backend' in action_config: self.uses_sub_use_backend = True if 'redirect' in action_config and 'url' not in action_config: self.uses_sub_reconstruct_requrl = True sb = [] nb = [] for action in supported_actions: action_name = action['name'] action_function = action['func'] if action_name not in action_config: continue config_out = {'sb': [], 'nb': [], 'synth': []} action_function(config_in=action_config[action_name], config_out=config_out, rule_id=rule_id, indent_depth=indent_depth + 1) sb += config_out['sb'] nb += config_out['nb'] self.synthetic_responses += config_out['synth'] if is_global: if sb: self.global_actions_southbound += sb if nb: self.global_actions_northbound += nb else: if sb: sb.insert( 0, indent(indent_depth + 1) + 'call global_actions_southbound;') if match_sub_name: actions = [] actions.append((indent(indent_depth) + 'call ' + match_sub_name + ';')) actions += make_action_if_clause(sb, rule_id, indent_depth=indent_depth) self.actions_southbound.setdefault(domain, []) self.actions_southbound[domain] += actions else: # If there is no match_sub_name, it is a default rule actions = [] # Set variable when default actions are reached in # vcl_recv (southbound) so we know whether to perform # the default northbound actions actions.append( indent(indent_depth + 1) + make_vcl_set_match_variable(rule_id)) actions += sb self.default_actions_southbound.setdefault(domain, []) self.default_actions_southbound[domain] += actions if nb: if match_sub_name: actions = make_action_if_clause(nb, rule_id, indent_depth=indent_depth) self.actions_northbound.setdefault(domain, []) self.actions_northbound[domain] += actions else: # If there is no match_sub_name, it is a default rule # Only perform default northbound actions if variable is set # (that is, only when the default southbound actions did) actions = make_action_if_clause(nb, rule_id, indent_depth=indent_depth) self.default_actions_northbound.setdefault(domain, []) self.default_actions_northbound[domain] += actions # Return True if we generated an action return (config_out['sb'] or config_out['nb'] or config_out['synth'])