def build(self) -> Tuple[IR, EnvoyConfig]: # Do a build, return IR & econf, but also stash them in self.builds. yaml_data = yaml.safe_dump_all(self.resources.values()) aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(yaml_data, k8s=True) aconf.load_all(fetcher.sorted()) ir = IR(aconf, cache=self.cache, file_checker=lambda path: True, secret_handler=self.secret_handler) assert ir, "could not create an IR" econf = EnvoyConfig.generate(ir, "V2", cache=self.cache) assert econf, "could not create an econf" self.builds.append((ir, econf)) return ir, econf
def _load_ir(self, aconf: Config, fetcher: ResourceFetcher, secret_reader: Callable[['IRTLSContext', str, str], SavedSecret], snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-%s.json" % snapshot) open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_reader=secret_reader) ir_path = os.path.join(app.snapshot_path, "ir-%s.json" % snapshot) open(ir_path, "w").write(ir.as_json()) check_scout(app, "update", ir) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) return self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated") if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5)
def _get_envoy_config(yaml): aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(yaml) aconf.load_all(fetcher.sorted()) secret_handler = NullSecretHandler(logger, None, None, "0") ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) assert ir return EnvoyConfig.generate(ir, "V2")
def _get_envoy_config(yaml, version='V3'): aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_yaml(default_listener_manifests() + yaml, k8s=True) aconf.load_all(fetcher.sorted()) secret_handler = NullSecretHandler(logger, None, None, "0") ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) assert ir return EnvoyConfig.generate(ir, version)
def load_config(self, url): snapshot = url.split('/')[-1] aconf_path = os.path.join(app.config_dir_prefix, "snapshot-%s.yaml" % snapshot) ir_path = os.path.join(app.config_dir_prefix, "ir-%s.json" % snapshot) self.logger.info("copying configuration from %s to %s" % (url, aconf_path)) saved = save_url_contents(self.logger, "%s/services" % url, aconf_path) if saved: scc = SecretSaver(app.logger, url, app.config_dir_prefix) aconf = Config() # Yeah yeah yeah. It's not really a directory. Whatever. aconf.load_from_directory(aconf_path, k8s=app.k8s, recurse=True) ir = IR(aconf, secret_reader=scc.url_reader) open(ir_path, "w").write(ir.as_json()) check_scout(app, "update", ir) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write( json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated")
def _test_compiler(test_name, envoy_api_version="V2"): test_path = os.path.join(testfolder, test_name) basename = os.path.basename(test_path) with open(os.path.join(test_path, 'bootstrap-ads.json'), 'r') as f: expected_bootstrap = json.loads(f.read()) node_name = expected_bootstrap.get('node', {}).get('cluster', None) assert node_name namespace = node_name.replace(test_name + '-', '', 1) with open(os.path.join(test_path, 'snapshot.yaml'), 'r') as f: watt = f.read() Config.ambassador_id = basename Config.ambassador_namespace = namespace os.environ['AMBASSADOR_ID'] = basename os.environ['AMBASSADOR_NAMESPACE'] = namespace aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_watt(watt) aconf.load_all(fetcher.sorted()) secret_handler = MockSecretHandler(logger, "mockery", "/tmp/ambassador/snapshots", "v1") ir = IR(aconf, file_checker=lambda path: True, secret_handler=secret_handler) expected_econf_file = 'econf.json' if ir.edge_stack_allowed: expected_econf_file = 'econf-aes.json' with open(os.path.join(test_path, expected_econf_file), 'r') as f: expected_econf = json.loads(f.read()) assert ir econf = EnvoyConfig.generate(ir, version=envoy_api_version) bootstrap_config, ads_config, _ = econf.split_config() ads_config.pop('@type', None) assert bootstrap_config == expected_bootstrap assert ads_config == expected_econf assert_valid_envoy_config(ads_config) assert_valid_envoy_config(bootstrap_config) # @pytest.mark.compilertest # @pytest.mark.parametrize("test_name", testdata) # def test_compiler_v3(test_name): # _test_compiler(test_name, envoy_api_version="V3")
def show_overview(reqid=None): app.logger.debug("OV %s - showing overview" % reqid) aconf = get_aconf(app) ir = IR(aconf, secret_reader=app.scc.secret_reader) check_scout(app, "overview", ir) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) if app.verbose: app.logger.debug("OV %s: DIAG" % reqid) app.logger.debug("%s" % json.dumps(diag.as_dict(), sort_keys=True, indent=4)) ov = diag.overview(request, app.estats) if app.verbose: app.logger.debug("OV %s: OV" % reqid) app.logger.debug("%s" % json.dumps(ov, sort_keys=True, indent=4)) app.logger.debug("OV %s: collecting errors" % reqid) ddict = collect_errors_and_notices(request, reqid, "overview", diag) tvars = dict(system=system_info(), envoy_status=envoy_status(app.estats), loginfo=app.estats.loginfo, notices=app.notices.notices, **ov, **ddict) if request.args.get('json', None): key = request.args.get('filter', None) if key: return jsonify(tvars.get(key, None)) else: return jsonify(tvars) else: return render_template("overview.html", **tvars)
def show_intermediate(source=None, reqid=None): app.logger.debug("SRC %s - getting intermediate for '%s'" % (reqid, source)) aconf = get_aconf(app) ir = IR(aconf, secret_reader=app.scc.secret_reader) check_scout(app, "detail: %s" % source, ir) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) method = request.args.get('method', None) resource = request.args.get('resource', None) result = diag.lookup(request, source, app.estats) if app.verbose: app.logger.debug("RESULT %s" % json.dumps(result, sort_keys=True, indent=4)) ddict = collect_errors_and_notices(request, reqid, "detail %s" % source, diag) tvars = dict(system=system_info(), envoy_status=envoy_status(app.estats), loginfo=app.estats.loginfo, notices=app.notices.notices, method=method, resource=resource, **result, **ddict) if request.args.get('json', None): key = request.args.get('filter', None) if key: return jsonify(tvars.get(key, None)) else: return jsonify(tvars) else: return render_template("diag.html", **tvars)
def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, secret_handler: SecretHandler, snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json") open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_handler=secret_handler) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) app.check_scout("attempted bad update") self._respond( rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot) return snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4")) snaplist: List[Tuple[str, str]] = [] if snapcount > 0: self.logger.debug("rotating snapshots for snapshot %s" % snapshot) # If snapcount is 4, this range statement becomes range(-4, -1) # which gives [ -4, -3, -2 ], which the list comprehension turns # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]... # which is the list of suffixes to rename to rotate the snapshots. snaplist += [(str(x + 1), str(x)) for x in range(-1 * snapcount, -1)] # After dealing with that, we need to rotate the current file into -1. snaplist.append(('', '-1')) # Whether or not we do any rotation, we need to cycle in the '-tmp' file. snaplist.append(('-tmp', '')) for from_suffix, to_suffix in snaplist: for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) try: self.logger.debug("rotate: %s -> %s" % (from_path, to_path)) os.rename(from_path, to_path) except IOError as e: self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e)) pass except Exception as e: self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e)) self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated from snapshot %s" % snapshot) self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot) if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5) # app.scout_updater = PeriodicTrigger(lambda: app.watcher.check_scout("30s"), period=30) # Don't use app.check_scout; it will deadlock. And don't bother doing the Scout # update until after we've taken care of Envoy. self.check_scout("update")
def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: bool, source: List[str], labels: List[str], namespace: Optional[str], watch: str, include_ir: bool, include_aconf: bool, diff_path: Optional[str] = None, kat_name: Optional[str] = None) -> None: loglevel = logging.DEBUG if debug else logging.INFO logging.basicConfig( level=loglevel, format="%(asctime)s mockery %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") logger = logging.getLogger('mockery') logger.debug(f"reading from {k8s_yaml_paths}") if not source: source = [ "Host", "service", "ingresses", "AuthService", "LogService", "Mapping", "Module", "RateLimitService", "TCPMapping", "TLSContext", "TracingService", "ConsulResolver", "KubernetesEndpointResolver", "KubernetesServiceResolver" ] if namespace: os.environ['AMBASSADOR_NAMESPACE'] = namespace # Make labels a list, instead of a tuple. labels = list(labels) labels_to_force = {l: True for l in labels or []} if kat_name: logger.debug(f"KAT name {kat_name}") # First set up some labels to force. labels_to_force["scope=AmbassadorTest"] = True labels_to_force[f"service={kat_name}"] = True kat_amb_id_label = f"kat-ambassador-id={kat_name}" if kat_amb_id_label not in labels_to_force: labels_to_force[kat_amb_id_label] = True labels.append(kat_amb_id_label) os.environ['AMBASSADOR_ID'] = kat_name # Forcibly override the cached ambassador_id. Config.ambassador_id = kat_name logger.debug(f"namespace {namespace or '*'}") logger.debug(f"labels to watch {', '.join(labels)}") logger.debug( f"labels to force {', '.join(sorted(labels_to_force.keys()))}") logger.debug(f"watch hook {watch}") logger.debug(f"sources {', '.join(source)}") for key in sorted(os.environ.keys()): if key.startswith('AMBASSADOR'): logger.debug(f"${key}={os.environ[key]}") if force_pod_labels: try: os.makedirs("/tmp/ambassador-pod-info") except OSError as e: if e.errno != errno.EEXIST: raise with open("/tmp/ambassador-pod-info/labels", "w", encoding="utf-8") as outfile: for l in labels_to_force: outfile.write(l) outfile.write("\n") # Pull in the YAML. input_yaml = ''.join([open(x, "r").read() for x in k8s_yaml_paths]) manifest = parse_yaml(input_yaml) w = Mockery(logger, debug, source, ",".join(labels), namespace, watch) iteration = 0 while True: iteration += 1 if iteration > 10: print(f"!!!! Not stable after 10 iterations, failing") logger.error("Not stable after 10 iterations, failing") sys.exit(1) logger.info(f"======== START ITERATION {iteration}") w.load(manifest) logger.info(f"WATT_K8S: {w.snapshot}") hook_ok, any_changes = w.run_hook() if not hook_ok: raise Exception("hook failed") if any_changes: logger.info( f"======== END ITERATION {iteration}: watches changed!") else: logger.info(f"======== END ITERATION {iteration}: stable!") break # Once here, we should be good to go. try: os.makedirs("/tmp/ambassador/snapshots") except OSError as e: if e.errno != errno.EEXIST: raise scc = MockSecretHandler(logger, "mockery", "/tmp/ambassador/snapshots", f"v{iteration}") aconf = Config() logger.debug(f"Config.ambassador_id {Config.ambassador_id}") logger.debug(f"Config.ambassador_namespace {Config.ambassador_namespace}") logger.info(f"STABLE WATT_K8S: {w.snapshot}") fetcher = ResourceFetcher(logger, aconf) fetcher.parse_watt(w.snapshot) aconf.load_all(fetcher.sorted()) open("/tmp/ambassador/snapshots/aconf.json", "w", encoding="utf-8").write(aconf.as_json()) ir = IR(aconf, secret_handler=scc) open("/tmp/ambassador/snapshots/ir.json", "w", encoding="utf-8").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") bootstrap_config, ads_config, clustermap = econf.split_config() ads_config.pop('@type', None) with open("/tmp/ambassador/snapshots/econf.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(ads_config, pretty=True)) with open("/tmp/ambassador/snapshots/bootstrap.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(bootstrap_config, pretty=True)) diag = Diagnostics(ir, econf) with open("/tmp/ambassador/snapshots/diag.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(diag.as_dict(), pretty=True)) if diff_path: diffs = False pairs_to_check = [(os.path.join(diff_path, 'snapshots', 'econf.json'), '/tmp/ambassador/snapshots/econf.json'), (os.path.join(diff_path, 'bootstrap-ads.json'), '/tmp/ambassador/snapshots/bootstrap.json')] if include_ir: pairs_to_check.append( (os.path.join(diff_path, 'snapshots', 'ir.json'), '/tmp/ambassador/snapshots/ir.json')) if include_aconf: pairs_to_check.append((os.path.join(diff_path, 'snapshots', 'aconf.json'), '/tmp/ambassador/snapshots/aconf.json')) for gold_path, check_path in pairs_to_check: if update: logger.info(f"mv {check_path} {gold_path}") shutil.move(check_path, gold_path) elif not filecmp.cmp(gold_path, check_path): diffs = True gold_lines = open(gold_path, "r", encoding="utf-8").readlines() check_lines = open(check_path, "r", encoding="utf-8").readlines() for line in difflib.unified_diff(gold_lines, check_lines, fromfile=gold_path, tofile=check_path): sys.stdout.write(line) if diffs: sys.exit(1)
def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, secret_handler: SecretHandler, snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json") open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_handler=secret_handler) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config, retries=self.app.validation_retries): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) # Don't use app.check_scout; it will deadlock. self.check_scout("attempted bad update") self._respond( rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot) return snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4")) snaplist: List[Tuple[str, str]] = [] if snapcount > 0: self.logger.debug("rotating snapshots for snapshot %s" % snapshot) # If snapcount is 4, this range statement becomes range(-4, -1) # which gives [ -4, -3, -2 ], which the list comprehension turns # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]... # which is the list of suffixes to rename to rotate the snapshots. snaplist += [(str(x + 1), str(x)) for x in range(-1 * snapcount, -1)] # After dealing with that, we need to rotate the current file into -1. snaplist.append(('', '-1')) # Whether or not we do any rotation, we need to cycle in the '-tmp' file. snaplist.append(('-tmp', '')) for from_suffix, to_suffix in snaplist: for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) try: self.logger.debug("rotate: %s -> %s" % (from_path, to_path)) os.rename(from_path, to_path) except IOError as e: self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e)) pass except Exception as e: self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e)) app.latest_snapshot = snapshot self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) if app.ir.k8s_status_updates: for name in app.ir.k8s_status_updates.keys(): kind, update = app.ir.k8s_status_updates[name] self.logger.info( f"doing K8s status update for {kind} {name}...") text = json.dumps(update) with open(f'/tmp/kstat-{kind}-{name}', 'w') as out: out.write(text) cmd = [ '/ambassador/kubestatus', kind, '-f', f'metadata.name={name}', '-u', '/dev/fd/0' ] self.logger.info(f"Running command: {cmd}") try: rc = subprocess.run(cmd, input=text.encode('utf-8'), timeout=5) self.logger.info(f'...update finished, rc {rc.returncode}') except subprocess.TimeoutExpired as e: self.logger.error(f'...update timed out, {e}') self.logger.info("configuration updated from snapshot %s" % snapshot) self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot) if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5) # Check our environment... self.check_environment() self.chime()
def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, secret_reader: Callable[['IRTLSContext', str, str], SavedSecret], snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json") open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_reader=secret_reader) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) app.check_scout("attempted bad update") self._respond( rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot) return self.logger.info("rotating snapshots for snapshot %s" % snapshot) for from_suffix, to_suffix in [('-3', '-4'), ('-2', '-3'), ('-1', '-2'), ('', '-1'), ('-tmp', '')]: for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: try: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) self.logger.debug("rotate: %s -> %s" % (from_path, to_path)) os.rename(from_path, to_path) except IOError as e: self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e)) except Exception as e: self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e)) self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated from snapshot %s" % snapshot) self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot) if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5) # Don't use app.check_scout; it will deadlock. And don't bother doing the Scout # update until after we've taken care of Envoy. self.check_scout("update")