def load_config_fs(self, path: str) -> None: snapshot = re.sub(r'[^A-Za-z0-9_-]', '_', path) self.logger.info("loading configuration from disk: %s" % path) scc = SecretSaver(app.logger, path, app.snapshot_path) aconf = Config() fetcher = ResourceFetcher(app.logger, aconf) fetcher.load_from_filesystem(path, k8s=False, recurse=True) if not fetcher.elements: self.logger.info("no configuration found at %s" % path) return self._load_ir(aconf, fetcher, scc.null_reader, snapshot)
def load_config_fs(self, rqueue: queue.Queue, path: str) -> None: self.logger.info("loading configuration from disk: %s" % path) snapshot = re.sub(r'[^A-Za-z0-9_-]', '_', path) scc = FSSecretHandler(app.logger, path, app.snapshot_path, 0) aconf = Config() fetcher = ResourceFetcher(app.logger, aconf) fetcher.load_from_filesystem(path, k8s=app.k8s, recurse=True) if not fetcher.elements: self.logger.debug("no configuration resources found at %s" % path) # self._respond(rqueue, 204, 'ignoring empty configuration') # return self._load_ir(rqueue, aconf, fetcher, scc, snapshot)
def load_config_fs(self, rqueue: queue.Queue, path: str) -> None: self.logger.info("loading configuration from disk: %s" % path) # The "path" here can just be a path, but it can also be a command for testing, # if the user has chosen to allow that. if self.app.allow_fs_commands and (':' in path): pfx, rest = path.split(':', 1) if pfx.lower() == 'cmd': fields = rest.split(':', 1) cmd = fields[0].upper() args = fields[1:] if (len(fields) > 1) else None if cmd.upper() == 'CHIME': self.logger.info('CMD: Chiming') self.chime() self._respond(rqueue, 200, 'Chimed') elif cmd.upper() == 'CHIME_RESET': self.chimed = False self.last_chime = False self.env_good = False self.app.scout.reset_events() self.app.scout.report(mode="boot", action="boot1", no_cache=True) self.logger.info('CMD: Reset chime state') self._respond(rqueue, 200, 'CMD: Reset chime state') elif cmd.upper() == 'SCOUT_CACHE_RESET': self.app.scout.reset_cache_time() self.logger.info('CMD: Reset Scout cache time') self._respond(rqueue, 200, 'CMD: Reset Scout cache time') elif cmd.upper() == 'ENV_OK': self.env_good = True self.failure_list = None self.logger.info('CMD: Marked environment good') self._respond(rqueue, 200, 'CMD: Marked environment good') elif cmd.upper() == 'ENV_BAD': self.env_good = False self.failure_list = ['failure forced'] self.logger.info('CMD: Marked environment bad') self._respond(rqueue, 200, 'CMD: Marked environment bad') else: self.logger.info(f'CMD: no such command "{cmd}"') self._respond(rqueue, 400, f'CMD: no such command "{cmd}"') return else: self.logger.info(f'CONFIG_FS: invalid prefix "{pfx}"') self._respond(rqueue, 400, f'CONFIG_FS: invalid prefix "{pfx}"') return snapshot = re.sub(r'[^A-Za-z0-9_-]', '_', path) scc = FSSecretHandler(app.logger, path, app.snapshot_path, "0") aconf = Config() fetcher = ResourceFetcher(app.logger, aconf) fetcher.load_from_filesystem(path, k8s=app.k8s, recurse=True) if not fetcher.elements: self.logger.debug("no configuration resources found at %s" % path) # self._respond(rqueue, 204, 'ignoring empty configuration') # return self._load_ir(rqueue, aconf, fetcher, scc, snapshot)
def test_config(testname, dirpath, configdir): # pytest.xfail("old V1 tests are disabled") # return global logger errors = [] if not os.path.isdir(configdir): errors.append("configdir %s is not a directory" % configdir) print("==== loading resources") aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.load_from_filesystem(configdir, recurse=True) aconf.load_all(fetcher.sorted()) ir = IR(aconf, file_checker=file_always_exists, secret_reader=atest_secret_reader) v1config = V1Config(ir) print("==== checking IR") current = get_old_intermediate(aconf, ir, v1config) current['envoy_config'] = filtered_overview(current['envoy_config']) current = sanitize_errors(current) current_path = os.path.join(dirpath, "intermediate.json") json.dump(current, open(current_path, "w"), sort_keys=True, indent=4) # Check the IR against its gold file, if that gold file exists. gold_path = os.path.join(dirpath, "gold.intermediate.json") if os.path.exists(gold_path): gold_parsed = None try: gold_parsed = json.load(open(gold_path, "r")) except json.decoder.JSONDecodeError as e: errors.append("%s was unparseable?" % gold_path) gold_no_yaml = normalize_gold(gold_parsed) gold_no_yaml_path = os.path.join(dirpath, "gold.no_yaml.json") json.dump(gold_no_yaml, open(gold_no_yaml_path, "w"), sort_keys=True, indent=4) udiff = unified_diff(gold_no_yaml_path, current_path) if udiff: errors.append("gold.intermediate.json and intermediate.json do not match!\n\n%s" % "\n".join(udiff)) print("==== checking V1") # Check the V1 config against its gold file, if it exists (and it should). gold_path = os.path.join(dirpath, "gold.json") if os.path.exists(gold_path): v1path = os.path.join(dirpath, "v1.json") json.dump(v1config.as_dict(), open(v1path, "w"), sort_keys=True, indent=4) udiff = unified_diff(gold_path, v1path) if udiff: errors.append("gold.json and v1.json do not match!\n\n%s" % "\n".join(udiff)) # if ambassador.code != 0: # errors.append('ambassador failed! %s' % ambassador.code) # else: # envoy = shell([ 'docker', 'run', # '--rm', # '-v', '%s:/etc/ambassador-config' % dirpath, # VALIDATOR_IMAGE, # '/usr/local/bin/envoy', # '--base-id', '1', # '--mode', 'validate', # '--service-cluster', 'test', # '-c', '/etc/ambassador-config/envoy.json' ], # verbose=True) # # envoy_succeeded = (envoy.code == 0) # # if not envoy_succeeded: # errors.append('envoy failed! %s' % envoy.code) # # envoy_output = list(envoy.output()) # # if envoy_succeeded: # if not envoy_output[-1].strip().endswith(' OK'): # errors.append('envoy validation failed!') # # print("==== checking short-circuit with existing config") # # ambassador = shell([ 'ambassador', 'config', '--check', configdir, envoy_json_out ]) # # print(ambassador.errors(raw=True)) # # if ambassador.code != 0: # errors.append('ambassador repeat check failed! %s' % ambassador.code) # # if 'Output file exists' not in ambassador.errors(raw=True): # errors.append('ambassador repeat check did not short circuit??') if errors: print("---- ERRORS") print("%s" % "\n".join(errors)) assert not errors, ("failing, _errors: %d" % len(errors))
def config(config_dir_path: Parameter.REQUIRED, output_json_path: Parameter.REQUIRED, *, debug=False, debug_scout=False, check=False, k8s=False, ir=None, aconf=None, exit_on_error=False): """ Generate an Envoy configuration :param config_dir_path: Configuration directory to scan for Ambassador YAML files :param output_json_path: Path to output envoy.json :param debug: If set, generate debugging output :param debug_scout: If set, generate debugging output when talking to Scout :param check: If set, generate configuration only if it doesn't already exist :param k8s: If set, assume configuration files are annotated K8s manifests :param exit_on_error: If set, will exit with status 1 on any configuration error :param ir: Pathname to which to dump the IR (not dumped if not present) :param aconf: Pathname to which to dump the aconf (not dumped if not present) """ if debug: logger.setLevel(logging.DEBUG) if debug_scout: logging.getLogger('ambassador.scout').setLevel(logging.DEBUG) try: logger.debug("CHECK MODE %s" % check) logger.debug("CONFIG DIR %s" % config_dir_path) logger.debug("OUTPUT PATH %s" % output_json_path) dump_aconf: Optional[str] = aconf dump_ir: Optional[str] = ir # Bypass the existence check... output_exists = False if check: # ...oh no wait, they explicitly asked for the existence check! # Assume that the file exists (ie, we'll do nothing) unless we # determine otherwise. output_exists = True try: json.loads(open(output_json_path, "r").read()) except FileNotFoundError: logger.debug("output file does not exist") output_exists = False except OSError: logger.warning("output file is not sane?") output_exists = False except json.decoder.JSONDecodeError: logger.warning("output file is not valid JSON") output_exists = False logger.info("Output file %s" % ("exists" if output_exists else "does not exist")) rc = RichStatus.fromError("impossible error") if not output_exists: # Either we didn't need to check, or the check didn't turn up # a valid config. Regenerate. logger.info("Generating new Envoy configuration...") aconf = Config() fetcher = ResourceFetcher(logger, aconf) fetcher.load_from_filesystem(config_dir_path, k8s=k8s) aconf.load_all(fetcher.sorted()) if dump_aconf: with open(dump_aconf, "w") as output: output.write(aconf.as_json()) output.write("\n") # If exit_on_error is set, log _errors and exit with status 1 if exit_on_error and aconf.errors: raise Exception("errors in: {0}".format(', '.join(aconf.errors.keys()))) secret_handler = NullSecretHandler(logger, config_dir_path, config_dir_path, "0") ir = IR(aconf, file_checker=file_checker, secret_handler=secret_handler) if dump_ir: with open(dump_ir, "w") as output: output.write(ir.as_json()) output.write("\n") # clize considers kwargs with False for default value as flags, # resulting in the logic below. # https://clize.readthedocs.io/en/stable/basics.html#accepting-flags logger.info("Writing envoy V2 configuration") v2config = V2Config(ir) rc = RichStatus.OK(msg="huh_v2") if rc: with open(output_json_path, "w") as output: output.write(v2config.as_json()) output.write("\n") else: logger.error("Could not generate new Envoy configuration: %s" % rc.error) scout = Scout() result = scout.report(action="config", mode="cli") show_notices(result) except Exception as e: handle_exception("EXCEPTION from config", e, config_dir_path=config_dir_path, output_json_path=output_json_path) # This is fatal. sys.exit(1)
def dump(config_dir_path: Parameter.REQUIRED, *, secret_dir_path=None, watt=False, debug=False, debug_scout=False, k8s=False, recurse=False, aconf=False, ir=False, v2=False, diag=False, features=False): """ Dump various forms of an Ambassador configuration for debugging Use --aconf, --ir, and --envoy to control what gets dumped. If none are requested, the IR will be dumped. :param config_dir_path: Configuration directory to scan for Ambassador YAML files :param secret_dir_path: Directory into which to save secrets :param watt: If set, input must be a WATT snapshot :param debug: If set, generate debugging output :param debug_scout: If set, generate debugging output :param k8s: If set, assume configuration files are annotated K8s manifests :param recurse: If set, recurse into directories below config_dir_path :param aconf: If set, dump the Ambassador config :param ir: If set, dump the IR :param v2: If set, dump the Envoy V2 config :param diag: If set, dump the Diagnostics overview :param features: If set, dump the feature set """ if not secret_dir_path: secret_dir_path = config_dir_path if not os.path.isdir(secret_dir_path): secret_dir_path = os.path.dirname(secret_dir_path) if debug: logger.setLevel(logging.DEBUG) if debug_scout: logging.getLogger('ambassador.scout').setLevel(logging.DEBUG) if not (aconf or ir or v2 or diag or features): aconf = True ir = True v2 = True diag = False features = False dump_aconf = aconf dump_ir = ir dump_v2 = v2 dump_diag = diag dump_features = features od = {} diagconfig: Optional[EnvoyConfig] = None try: aconf = Config() fetcher = ResourceFetcher(logger, aconf) if watt: fetcher.parse_watt(open(config_dir_path, "r").read()) else: fetcher.load_from_filesystem(config_dir_path, k8s=k8s, recurse=True) aconf.load_all(fetcher.sorted()) # aconf.post_error("Error from string, boo yah") # aconf.post_error(RichStatus.fromError("Error from RichStatus")) if dump_aconf: od['aconf'] = aconf.as_dict() secret_handler = NullSecretHandler(logger, config_dir_path, secret_dir_path, "0") ir = IR(aconf, file_checker=file_checker, secret_handler=secret_handler) if dump_ir: od['ir'] = ir.as_dict() if dump_v2: v2config = V2Config(ir) diagconfig = v2config od['v2'] = v2config.as_dict() if dump_diag: if not diagconfig: diagconfig = V2Config(ir) econf = typecast(EnvoyConfig, diagconfig) diag = Diagnostics(ir, econf) od['diag'] = diag.as_dict() od['elements'] = econf.elements if dump_features: od['features'] = ir.features() # scout = Scout() # scout_args = {} # # if ir and not os.environ.get("AMBASSADOR_DISABLE_FEATURES", None): # scout_args["features"] = ir.features() # # result = scout.report(action="dump", mode="cli", **scout_args) # show_notices(result) json.dump(od, sys.stdout, sort_keys=True, indent=4) sys.stdout.write("\n") except Exception as e: handle_exception("EXCEPTION from dump", e, config_dir_path=config_dir_path) # This is fatal. sys.exit(1)