def _load_ir(self, aconf: Config, fetcher: ResourceFetcher, secret_reader: Callable[['IRTLSContext', str, str], SavedSecret], snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-%s.json" % snapshot) open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_reader=secret_reader) ir_path = os.path.join(app.snapshot_path, "ir-%s.json" % snapshot) open(ir_path, "w").write(ir.as_json()) check_scout(app, "update", ir) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) return self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated") if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5)
def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, secret_handler: SecretHandler, snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json") open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_handler=secret_handler) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) app.check_scout("attempted bad update") self._respond( rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot) return snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4")) snaplist: List[Tuple[str, str]] = [] if snapcount > 0: self.logger.debug("rotating snapshots for snapshot %s" % snapshot) # If snapcount is 4, this range statement becomes range(-4, -1) # which gives [ -4, -3, -2 ], which the list comprehension turns # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]... # which is the list of suffixes to rename to rotate the snapshots. snaplist += [(str(x + 1), str(x)) for x in range(-1 * snapcount, -1)] # After dealing with that, we need to rotate the current file into -1. snaplist.append(('', '-1')) # Whether or not we do any rotation, we need to cycle in the '-tmp' file. snaplist.append(('-tmp', '')) for from_suffix, to_suffix in snaplist: for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) try: self.logger.debug("rotate: %s -> %s" % (from_path, to_path)) os.rename(from_path, to_path) except IOError as e: self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e)) pass except Exception as e: self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e)) self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated from snapshot %s" % snapshot) self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot) if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5) # app.scout_updater = PeriodicTrigger(lambda: app.watcher.check_scout("30s"), period=30) # Don't use app.check_scout; it will deadlock. And don't bother doing the Scout # update until after we've taken care of Envoy. self.check_scout("update")
def main(k8s_yaml_paths: List[str], debug: bool, force_pod_labels: bool, update: bool, source: List[str], labels: List[str], namespace: Optional[str], watch: str, include_ir: bool, include_aconf: bool, diff_path: Optional[str] = None, kat_name: Optional[str] = None) -> None: loglevel = logging.DEBUG if debug else logging.INFO logging.basicConfig( level=loglevel, format="%(asctime)s mockery %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S") logger = logging.getLogger('mockery') logger.debug(f"reading from {k8s_yaml_paths}") if not source: source = [ "Host", "service", "ingresses", "AuthService", "LogService", "Mapping", "Module", "RateLimitService", "TCPMapping", "TLSContext", "TracingService", "ConsulResolver", "KubernetesEndpointResolver", "KubernetesServiceResolver" ] if namespace: os.environ['AMBASSADOR_NAMESPACE'] = namespace # Make labels a list, instead of a tuple. labels = list(labels) labels_to_force = {l: True for l in labels or []} if kat_name: logger.debug(f"KAT name {kat_name}") # First set up some labels to force. labels_to_force["scope=AmbassadorTest"] = True labels_to_force[f"service={kat_name}"] = True kat_amb_id_label = f"kat-ambassador-id={kat_name}" if kat_amb_id_label not in labels_to_force: labels_to_force[kat_amb_id_label] = True labels.append(kat_amb_id_label) os.environ['AMBASSADOR_ID'] = kat_name # Forcibly override the cached ambassador_id. Config.ambassador_id = kat_name logger.debug(f"namespace {namespace or '*'}") logger.debug(f"labels to watch {', '.join(labels)}") logger.debug( f"labels to force {', '.join(sorted(labels_to_force.keys()))}") logger.debug(f"watch hook {watch}") logger.debug(f"sources {', '.join(source)}") for key in sorted(os.environ.keys()): if key.startswith('AMBASSADOR'): logger.debug(f"${key}={os.environ[key]}") if force_pod_labels: try: os.makedirs("/tmp/ambassador-pod-info") except OSError as e: if e.errno != errno.EEXIST: raise with open("/tmp/ambassador-pod-info/labels", "w", encoding="utf-8") as outfile: for l in labels_to_force: outfile.write(l) outfile.write("\n") # Pull in the YAML. input_yaml = ''.join([open(x, "r").read() for x in k8s_yaml_paths]) manifest = parse_yaml(input_yaml) w = Mockery(logger, debug, source, ",".join(labels), namespace, watch) iteration = 0 while True: iteration += 1 if iteration > 10: print(f"!!!! Not stable after 10 iterations, failing") logger.error("Not stable after 10 iterations, failing") sys.exit(1) logger.info(f"======== START ITERATION {iteration}") w.load(manifest) logger.info(f"WATT_K8S: {w.snapshot}") hook_ok, any_changes = w.run_hook() if not hook_ok: raise Exception("hook failed") if any_changes: logger.info( f"======== END ITERATION {iteration}: watches changed!") else: logger.info(f"======== END ITERATION {iteration}: stable!") break # Once here, we should be good to go. try: os.makedirs("/tmp/ambassador/snapshots") except OSError as e: if e.errno != errno.EEXIST: raise scc = MockSecretHandler(logger, "mockery", "/tmp/ambassador/snapshots", f"v{iteration}") aconf = Config() logger.debug(f"Config.ambassador_id {Config.ambassador_id}") logger.debug(f"Config.ambassador_namespace {Config.ambassador_namespace}") logger.info(f"STABLE WATT_K8S: {w.snapshot}") fetcher = ResourceFetcher(logger, aconf) fetcher.parse_watt(w.snapshot) aconf.load_all(fetcher.sorted()) open("/tmp/ambassador/snapshots/aconf.json", "w", encoding="utf-8").write(aconf.as_json()) ir = IR(aconf, secret_handler=scc) open("/tmp/ambassador/snapshots/ir.json", "w", encoding="utf-8").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") bootstrap_config, ads_config, clustermap = econf.split_config() ads_config.pop('@type', None) with open("/tmp/ambassador/snapshots/econf.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(ads_config, pretty=True)) with open("/tmp/ambassador/snapshots/bootstrap.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(bootstrap_config, pretty=True)) diag = Diagnostics(ir, econf) with open("/tmp/ambassador/snapshots/diag.json", "w", encoding="utf-8") as outfile: outfile.write(dump_json(diag.as_dict(), pretty=True)) if diff_path: diffs = False pairs_to_check = [(os.path.join(diff_path, 'snapshots', 'econf.json'), '/tmp/ambassador/snapshots/econf.json'), (os.path.join(diff_path, 'bootstrap-ads.json'), '/tmp/ambassador/snapshots/bootstrap.json')] if include_ir: pairs_to_check.append( (os.path.join(diff_path, 'snapshots', 'ir.json'), '/tmp/ambassador/snapshots/ir.json')) if include_aconf: pairs_to_check.append((os.path.join(diff_path, 'snapshots', 'aconf.json'), '/tmp/ambassador/snapshots/aconf.json')) for gold_path, check_path in pairs_to_check: if update: logger.info(f"mv {check_path} {gold_path}") shutil.move(check_path, gold_path) elif not filecmp.cmp(gold_path, check_path): diffs = True gold_lines = open(gold_path, "r", encoding="utf-8").readlines() check_lines = open(check_path, "r", encoding="utf-8").readlines() for line in difflib.unified_diff(gold_lines, check_lines, fromfile=gold_path, tofile=check_path): sys.stdout.write(line) if diffs: sys.exit(1)
def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, secret_handler: SecretHandler, snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json") open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_handler=secret_handler) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config, retries=self.app.validation_retries): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) # Don't use app.check_scout; it will deadlock. self.check_scout("attempted bad update") self._respond( rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot) return snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4")) snaplist: List[Tuple[str, str]] = [] if snapcount > 0: self.logger.debug("rotating snapshots for snapshot %s" % snapshot) # If snapcount is 4, this range statement becomes range(-4, -1) # which gives [ -4, -3, -2 ], which the list comprehension turns # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]... # which is the list of suffixes to rename to rotate the snapshots. snaplist += [(str(x + 1), str(x)) for x in range(-1 * snapcount, -1)] # After dealing with that, we need to rotate the current file into -1. snaplist.append(('', '-1')) # Whether or not we do any rotation, we need to cycle in the '-tmp' file. snaplist.append(('-tmp', '')) for from_suffix, to_suffix in snaplist: for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) try: self.logger.debug("rotate: %s -> %s" % (from_path, to_path)) os.rename(from_path, to_path) except IOError as e: self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e)) pass except Exception as e: self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e)) app.latest_snapshot = snapshot self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) if app.ir.k8s_status_updates: for name in app.ir.k8s_status_updates.keys(): kind, update = app.ir.k8s_status_updates[name] self.logger.info( f"doing K8s status update for {kind} {name}...") text = json.dumps(update) with open(f'/tmp/kstat-{kind}-{name}', 'w') as out: out.write(text) cmd = [ '/ambassador/kubestatus', kind, '-f', f'metadata.name={name}', '-u', '/dev/fd/0' ] self.logger.info(f"Running command: {cmd}") try: rc = subprocess.run(cmd, input=text.encode('utf-8'), timeout=5) self.logger.info(f'...update finished, rc {rc.returncode}') except subprocess.TimeoutExpired as e: self.logger.error(f'...update timed out, {e}') self.logger.info("configuration updated from snapshot %s" % snapshot) self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot) if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5) # Check our environment... self.check_environment() self.chime()
def config(config_dir_path: Parameter.REQUIRED, output_json_path: Parameter.REQUIRED, *, debug=False, debug_scout=False, check=False, k8s=False, ir=None, aconf=None, exit_on_error=False): """ Generate an Envoy configuration :param config_dir_path: Configuration directory to scan for Ambassador YAML files :param output_json_path: Path to output envoy.json :param debug: If set, generate debugging output :param debug_scout: If set, generate debugging output when talking to Scout :param check: If set, generate configuration only if it doesn't already exist :param k8s: If set, assume configuration files are annotated K8s manifests :param exit_on_error: If set, will exit with status 1 on any configuration error :param ir: Pathname to which to dump the IR (not dumped if not present) :param aconf: Pathname to which to dump the aconf (not dumped if not present) """ if debug: logger.setLevel(logging.DEBUG) if debug_scout: logging.getLogger('ambassador.scout').setLevel(logging.DEBUG) try: logger.debug("CHECK MODE %s" % check) logger.debug("CONFIG DIR %s" % config_dir_path) logger.debug("OUTPUT PATH %s" % output_json_path) dump_aconf: Optional[str] = aconf dump_ir: Optional[str] = ir # Bypass the existence check... output_exists = False if check: # ...oh no wait, they explicitly asked for the existence check! # Assume that the file exists (ie, we'll do nothing) unless we # determine otherwise. output_exists = True try: parse_json(open(output_json_path, "r").read()) except FileNotFoundError: logger.debug("output file does not exist") output_exists = False except OSError: logger.warning("output file is not sane?") output_exists = False except json.decoder.JSONDecodeError: logger.warning("output file is not valid JSON") output_exists = False logger.info("Output file %s" % ("exists" if output_exists else "does not exist")) rc = RichStatus.fromError("impossible error") if not output_exists: # Either we didn't need to check, or the check didn't turn up # a valid config. Regenerate. logger.info("Generating new Envoy configuration...") aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.load_from_filesystem(config_dir_path, k8s=k8s) aconf.load_all(fetcher.sorted()) if dump_aconf: with open(dump_aconf, "w") as output: output.write(aconf.as_json()) output.write("\n") # If exit_on_error is set, log _errors and exit with status 1 if exit_on_error and aconf.errors: raise Exception("errors in: {0}".format(', '.join( aconf.errors.keys()))) secret_handler = NullSecretHandler(logger, config_dir_path, config_dir_path, "0") ir = IR(aconf, file_checker=file_checker, secret_handler=secret_handler) if dump_ir: with open(dump_ir, "w") as output: output.write(ir.as_json()) output.write("\n") # clize considers kwargs with False for default value as flags, # resulting in the logic below. # https://clize.readthedocs.io/en/stable/basics.html#accepting-flags logger.info("Writing envoy V2 configuration") v2config = V2Config(ir) rc = RichStatus.OK(msg="huh_v2") if rc: with open(output_json_path, "w") as output: output.write(v2config.as_json()) output.write("\n") else: logger.error("Could not generate new Envoy configuration: %s" % rc.error) scout = Scout() result = scout.report(action="config", mode="cli") show_notices(result) except Exception as e: handle_exception("EXCEPTION from config", e, config_dir_path=config_dir_path, output_json_path=output_json_path) # This is fatal. sys.exit(1)
def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher, secret_reader: Callable[['IRTLSContext', str, str], SavedSecret], snapshot: str) -> None: aconf.load_all(fetcher.sorted()) aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json") open(aconf_path, "w").write(aconf.as_json()) ir = IR(aconf, secret_reader=secret_reader) ir_path = os.path.join(app.snapshot_path, "ir-tmp.json") open(ir_path, "w").write(ir.as_json()) econf = EnvoyConfig.generate(ir, "V2") diag = Diagnostics(ir, econf) bootstrap_config, ads_config = econf.split_config() if not self.validate_envoy_config(config=ads_config): self.logger.info( "no updates were performed due to invalid envoy configuration, continuing with current configuration..." ) app.check_scout("attempted bad update") self._respond( rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot) return self.logger.info("rotating snapshots for snapshot %s" % snapshot) for from_suffix, to_suffix in [('-3', '-4'), ('-2', '-3'), ('-1', '-2'), ('', '-1'), ('-tmp', '')]: for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]: try: from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix)) to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix)) self.logger.debug("rotate: %s -> %s" % (from_path, to_path)) os.rename(from_path, to_path) except IOError as e: self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e)) except Exception as e: self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e)) self.logger.info("saving Envoy configuration for snapshot %s" % snapshot) with open(app.bootstrap_path, "w") as output: output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4)) with open(app.ads_path, "w") as output: output.write(json.dumps(ads_config, sort_keys=True, indent=4)) app.aconf = aconf app.ir = ir app.econf = econf app.diag = diag if app.kick: self.logger.info("running '%s'" % app.kick) os.system(app.kick) elif app.ambex_pid != 0: self.logger.info("notifying PID %d ambex" % app.ambex_pid) os.kill(app.ambex_pid, signal.SIGHUP) self.logger.info("configuration updated from snapshot %s" % snapshot) self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot) if app.health_checks and not app.stats_updater: app.logger.info("starting Envoy status updater") app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5) # Don't use app.check_scout; it will deadlock. And don't bother doing the Scout # update until after we've taken care of Envoy. self.check_scout("update")
# Never cache anything. def cache_secret(self, context: 'IRTLSContext', secret_info: SecretInfo): return SavedSecret(secret_info.name, secret_info.namespace, '-crt-path-', '-key-path-', { 'tls_crt': '-crt-', 'tls_key': '-key-' }) scc = SecretHandler(logger, "test-dump", "ss") yamlpath = sys.argv[1] if len(sys.argv) > 1 else "consul-3.yaml" aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.parse_watt(open(yamlpath, "r").read()) aconf.load_all(fetcher.sorted()) open("test-aconf.json", "w").write(aconf.as_json()) # sys.exit(0) ir = IR(aconf, secret_handler=scc) open("test-ir.json", "w").write(ir.as_json()) econf = V2Config(ir) open("test-v2.json", "w").write(econf.as_json())