def __init__(self, rule_docs, globals_doc=None):
# pylint:disable=too-many-locals,too-many-statements
super().__init__(rule_docs=rule_docs,
globals_doc=globals_doc,
output_file="varnish.vcl")
self.names = {}
self.matches = []
self.actions_southbound = {}
self.actions_northbound = {}
self.default_actions_southbound = {}
self.default_actions_northbound = {}
self.global_actions_southbound = []
self.global_actions_northbound = []
self.synthetic_responses = []
self.uses_sub_use_backend = False
self.uses_sub_reconstruct_requrl = False
self.directors = {}
for domain, rules in rule_docs.items():
self.actions_southbound.setdefault(domain, [])
self.actions_northbound.setdefault(domain, [])
self.default_actions_southbound.setdefault(domain, [])
self.default_actions_northbound.setdefault(domain, [])
for rule in rules:
description = rule.get("description")
orm_file = rule.get("_orm_source_file").split("/", 1)[1:][0]
rule_id = rule.get("_rule_id")
config_debug_line = "\n# " + orm_file + " - " + description
if rule.get("domain_default", False):
# Create default southbound and northbound actions
actions = rule.get("actions")
self.make_actions(actions,
rule_id,
domain=domain,
indent_depth=1)
else:
# Create a Varnish subroutine (sub) to match current rule
matches = rule.get("matches")
match_tree = parser.get_match_tree(matches)
self.matches.append(config_debug_line)
match_sub_name = self.get_unique_vcl_name("match", rule_id)
match_sub = self.make_match_sub(match_tree, rule_id,
match_sub_name)
# Create regular southbound and northbound actions
# (using the matching sub).
actions = rule.get("actions")
if self.make_actions(
actions,
rule_id,
domain=domain,
match_sub_name=match_sub_name,
indent_depth=1,
):
self.matches += match_sub
# Create global southbound and northbound actions
global_actions = self.globals_doc.get("global_actions", None)
if global_actions:
rule_id = "global"
self.make_actions(global_actions,
rule_id,
indent_depth=0,
is_global=True)
if self.uses_sub_use_backend:
self.uses_sub_reconstruct_requrl = True
southbound_config = self.make_southbound_actions(rule_docs.keys())
northbound_config = self.make_northbound_actions(rule_docs.keys())
haproxy = self.globals_doc.get("haproxy", {})
self.config = self.jinja.get_template("varnish.vcl.j2").render(
global_actions_southbound=self.global_actions_southbound,
global_actions_northbound=self.global_actions_northbound,
matches=self.matches,
synthetic_responses=self.synthetic_responses,
uses_sub_use_backend=self.uses_sub_use_backend,
uses_sub_reconstruct_requrl=self.uses_sub_reconstruct_requrl,
southbound_actions=southbound_config,
northbound_actions=northbound_config,
haproxy_address=haproxy.get("address", "localhost"),
)
def validate_constraints_rule_collision(domain_rules, cache_path=None):
# pylint:disable=too-many-locals,too-many-branches,too-many-statements
# Assert all paths are unique (collision check) per domain
print("Checking for path collisions...")
fsm_gen_start = time.time()
cpu_count = os.cpu_count()
print("Using a pool of {} workers".format(cpu_count))
worker_pool = concurrent.futures.ProcessPoolExecutor(max_workers=cpu_count)
admin_pool = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count)
fsm_futures = {}
fsm_cache = {}
fsm_cache_hits = {}
if cache_path and os.path.exists(cache_path):
with open(cache_path, "rb") as fsm_cache_file:
fsm_cache = pickle.load(fsm_cache_file)
for domain, rules in domain_rules.items():
for rule in rules:
# Create an FSM for each rule's match tree paths
if rule.get("domain_default", False):
continue
matches = rule["matches"]
match_tree = parser.get_match_tree(matches)
fsm_cache_key = domain + str(match_tree)
if fsm_cache_key in fsm_cache:
fsm_cache_hits[fsm_cache_key] = fsm_cache.pop(fsm_cache_key)
continue
future = admin_pool.submit(get_match_path_fsm,
match_tree,
worker_pool=worker_pool)
fsm_futures[future] = {
"desc": rule["description"],
"file": rule["_orm_source_file"],
"domain": domain,
"cache_key": fsm_cache_key,
}
# Set the cache to hits only to purge unused entries.
fsm_cache = fsm_cache_hits
rule_fsm_list = []
for fsm_future in concurrent.futures.as_completed(fsm_futures):
fsm = fsm_future.result()
fsm_entry = fsm_futures[fsm_future]
fsm_entry["fsm"] = fsm
print("Generated FSM for " + fsm_entry["file"] + ": " +
fsm_entry["desc"])
rule_fsm_list.append(fsm_entry)
print("Got {} FSM:s. {} from cache. {} freshly generated.".format(
str(len(rule_fsm_list) + len(fsm_cache.keys())),
str(len(fsm_cache.keys())),
str(len(rule_fsm_list)),
))
print("FSM generation took: {}s".format(
str(round(time.time() - fsm_gen_start, 2))))
collision_check_start = time.time()
collision_futures = {}
for i, fsm_one in enumerate(rule_fsm_list):
# First check the new FSM:s against the other new FSM:s
j = i + 1
while j < len(rule_fsm_list):
fsm_two = rule_fsm_list[j]
if fsm_one["domain"] == fsm_two["domain"]:
future = worker_pool.submit(fsm_collision, fsm_one["fsm"],
fsm_two["fsm"])
collision_futures[future] = {
"fsm_one": fsm_one,
"fsm_two": fsm_two
}
j += 1
# Then check the new FSM:s against the cached FSM:s
for cache_key in fsm_cache:
fsm_two = fsm_cache[cache_key]
if fsm_one["domain"] == fsm_two["domain"]:
future = worker_pool.submit(fsm_collision, fsm_one["fsm"],
fsm_two["fsm"])
collision_futures[future] = {
"fsm_one": fsm_one,
"fsm_two": fsm_two
}
collision_messages = []
colliding_fsm_cache_keys = []
for collision_future in concurrent.futures.as_completed(collision_futures):
if collision_future.result():
collision_data = collision_futures[collision_future]
fsm_one = collision_data["fsm_one"]
fsm_two = collision_data["fsm_two"]
colliding_fsm_cache_keys += [
fsm_one["cache_key"], fsm_two["cache_key"]
]
collision_messages.append(
"\nFound path collision for domain: {domain}\n"
"{first_file} ({first_desc})\n"
"collides with\n"
"{second_file} ({second_desc})\n".format(
domain=fsm_one["domain"],
first_file=fsm_one["file"],
first_desc=fsm_one["desc"],
second_file=fsm_two["file"],
second_desc=fsm_two["desc"],
))
print("Path collision check took: " +
str(round(time.time() - collision_check_start, 2)) + "s")
if cache_path:
print("Writing FSM cache to {}".format(cache_path))
# Add newly generated FSM:s to cache
for fsm_entry in rule_fsm_list:
cache_key = fsm_entry["cache_key"]
# Only add to cache if it did not collide with anything.
# The cache must only contain FSM:s which does not collide
# with any other FSM in the cache.
if cache_key not in colliding_fsm_cache_keys:
fsm_cache[cache_key] = fsm_entry
with open(cache_path, "wb") as fsm_cache_file:
pickle.dump(fsm_cache, fsm_cache_file)
if collision_messages:
for msg in collision_messages:
print(msg)
return False
return True