Esempio n. 1
0
def main(k8s_yaml_path: str, debug: bool, force_pod_labels: bool, update: bool,
         source: List[str], labels: List[str], namespace: Optional[str], watch: str,
         include_ir: bool, include_aconf: bool,
         diff_path: Optional[str]=None, kat_name: Optional[str]=None) -> None:
    loglevel = logging.DEBUG if debug else logging.INFO

    logging.basicConfig(
        level=loglevel,
        format="%(asctime)s mockery %(levelname)s: %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S"
    )

    logger = logging.getLogger('mockery')

    logger.debug(f"reading from {k8s_yaml_path}")

    if not source:
        source = [
            "Host", "service", "ingresses",
            "AuthService", "LogService", "Mapping", "Module", "RateLimitService",
            "TCPMapping", "TLSContext", "TracingService",
            "ConsulResolver", "KubernetesEndpointResolver", "KubernetesServiceResolver"
        ]

    if namespace:
        os.environ['AMBASSADOR_NAMESPACE'] = namespace

    # Make labels a list, instead of a tuple.
    labels = list(labels)
    labels_to_force = { l: True for l in labels or [] }

    if kat_name:
        logger.debug(f"KAT name {kat_name}")

        # First set up some labels to force.

        labels_to_force["scope=AmbassadorTest"] = True
        labels_to_force[f"service={kat_name}"] = True

        kat_amb_id_label = f"kat-ambassador-id={kat_name}"

        if kat_amb_id_label not in labels_to_force:
            labels_to_force[kat_amb_id_label] = True
            labels.append(kat_amb_id_label)

        os.environ['AMBASSADOR_ID'] = kat_name

        # Forcibly override the cached ambassador_id.
        Config.ambassador_id = kat_name

    logger.debug(f"namespace {namespace or '*'}")
    logger.debug(f"labels to watch {', '.join(labels)}")
    logger.debug(f"labels to force {', '.join(sorted(labels_to_force.keys()))}")
    logger.debug(f"watch hook {watch}")
    logger.debug(f"sources {', '.join(source)}")

    for key in sorted(os.environ.keys()):
        if key.startswith('AMBASSADOR'):
            logger.debug(f"${key}={os.environ[key]}")

    if force_pod_labels:
        try:
            os.makedirs("/tmp/ambassador-pod-info")
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        with open("/tmp/ambassador-pod-info/labels", "w", encoding="utf-8") as outfile:
            for l in labels_to_force:
                outfile.write(l)
                outfile.write("\n")

    # Pull in the YAML.
    manifest = parse_yaml(open(k8s_yaml_path, "r").read())

    w = Mockery(logger, debug, source, ",".join(labels), namespace, watch)

    iteration = 0

    while True:
        iteration += 1

        if iteration > 10:
            print(f"!!!! Not stable after 10 iterations, failing")
            logger.error("Not stable after 10 iterations, failing")
            sys.exit(1)

        print(f"==== ITERATION {iteration}")

        watt_k8s = w.load(manifest)

        logger.info(f"WATT_K8S: {json.dumps(watt_k8s, sort_keys=True, indent=4)}")

        hook_ok, any_changes = w.run_hook(watt_k8s)
        
        if not hook_ok:
            raise Exception("hook failed")
            
        if any_changes:
            logger.info("WATT_K8S: some changes from the hook!")
            print("====== watches changed!")
        else:
            print("====== stable!")
            break

    # Once here, we should be good to go.
    try:
        os.makedirs("/tmp/ambassador/snapshots")
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    scc = SecretHandler(logger, "mockery", "/tmp/ambassador/snapshots", f"v{iteration}")

    aconf = Config()

    logger.debug(f"Config.ambassador_id {Config.ambassador_id}")
    logger.debug(f"Config.ambassador_namespace {Config.ambassador_namespace}")

    fetcher = ResourceFetcher(logger, aconf)
    fetcher.parse_watt(open("/tmp/mockery.json", "r", encoding="utf-8").read())
    aconf.load_all(fetcher.sorted())

    open("/tmp/ambassador/snapshots/aconf.json", "w", encoding="utf-8").write(aconf.as_json())

    ir = IR(aconf, secret_handler=scc)

    open("/tmp/ambassador/snapshots/ir.json", "w", encoding="utf-8").write(ir.as_json())

    econf = EnvoyConfig.generate(ir, "V2")
    bootstrap_config, ads_config = econf.split_config()

    ads_config.pop('@type', None)
    with open("/tmp/ambassador/snapshots/econf.json", "w", encoding="utf-8") as outfile:
        outfile.write(json.dumps(ads_config, sort_keys=True, indent=4))

    with open("/tmp/ambassador/snapshots/bootstrap.json", "w", encoding="utf-8") as outfile:
        outfile.write(json.dumps(bootstrap_config, sort_keys=True, indent=4))

    diag = Diagnostics(ir, econf)

    with open("/tmp/ambassador/snapshots/diag.json", "w", encoding="utf-8") as outfile:
        outfile.write(json.dumps(diag.as_dict(), sort_keys=True, indent=4))

    if diff_path:
        diffs = False

        pairs_to_check = [
            (os.path.join(diff_path, 'snapshots', 'econf.json'), '/tmp/ambassador/snapshots/econf.json'),
            (os.path.join(diff_path, 'bootstrap-ads.json'), '/tmp/ambassador/snapshots/bootstrap.json')
        ]

        if include_ir:
            pairs_to_check.append(
                ( os.path.join(diff_path, 'snapshots', 'ir.json'), '/tmp/ambassador/snapshots/ir.json' )
            )

        if include_aconf:
            pairs_to_check.append(
                ( os.path.join(diff_path, 'snapshots', 'aconf.json'), '/tmp/ambassador/snapshots/aconf.json' )
            )

        for gold_path, check_path in pairs_to_check:
            if update:
                logger.info(f"mv {check_path} {gold_path}")
                shutil.move(check_path, gold_path)
            elif not filecmp.cmp(gold_path, check_path):
                diffs = True

                gold_lines = open(gold_path, "r", encoding="utf-8").readlines()
                check_lines = open(check_path, "r", encoding="utf-8").readlines()

                for line in difflib.unified_diff(gold_lines, check_lines, fromfile=gold_path, tofile=check_path):
                    sys.stdout.write(line)

        if diffs:
            sys.exit(1)
Esempio n. 2
0
    def _load_ir(self, rqueue: queue.Queue, aconf: Config,
                 fetcher: ResourceFetcher, secret_handler: SecretHandler,
                 snapshot: str) -> None:
        aconf.load_all(fetcher.sorted())

        aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json")
        open(aconf_path, "w").write(aconf.as_json())

        ir = IR(aconf, secret_handler=secret_handler)

        ir_path = os.path.join(app.snapshot_path, "ir-tmp.json")
        open(ir_path, "w").write(ir.as_json())

        econf = EnvoyConfig.generate(ir, "V2")
        diag = Diagnostics(ir, econf)

        bootstrap_config, ads_config = econf.split_config()

        if not self.validate_envoy_config(config=ads_config,
                                          retries=self.app.validation_retries):
            self.logger.info(
                "no updates were performed due to invalid envoy configuration, continuing with current configuration..."
            )
            # Don't use app.check_scout; it will deadlock.
            self.check_scout("attempted bad update")
            self._respond(
                rqueue, 500,
                'ignoring: invalid Envoy configuration in snapshot %s' %
                snapshot)
            return

        snapcount = int(os.environ.get('AMBASSADOR_SNAPSHOT_COUNT', "4"))
        snaplist: List[Tuple[str, str]] = []

        if snapcount > 0:
            self.logger.debug("rotating snapshots for snapshot %s" % snapshot)

            # If snapcount is 4, this range statement becomes range(-4, -1)
            # which gives [ -4, -3, -2 ], which the list comprehension turns
            # into [ ( "-3", "-4" ), ( "-2", "-3" ), ( "-1", "-2" ) ]...
            # which is the list of suffixes to rename to rotate the snapshots.

            snaplist += [(str(x + 1), str(x))
                         for x in range(-1 * snapcount, -1)]

            # After dealing with that, we need to rotate the current file into -1.
            snaplist.append(('', '-1'))

        # Whether or not we do any rotation, we need to cycle in the '-tmp' file.
        snaplist.append(('-tmp', ''))

        for from_suffix, to_suffix in snaplist:
            for fmt in [
                    "aconf{}.json", "econf{}.json", "ir{}.json",
                    "snapshot{}.yaml"
            ]:
                from_path = os.path.join(app.snapshot_path,
                                         fmt.format(from_suffix))
                to_path = os.path.join(app.snapshot_path,
                                       fmt.format(to_suffix))

                try:
                    self.logger.debug("rotate: %s -> %s" %
                                      (from_path, to_path))
                    os.rename(from_path, to_path)
                except IOError as e:
                    self.logger.debug("skip %s -> %s: %s" %
                                      (from_path, to_path, e))
                    pass
                except Exception as e:
                    self.logger.debug("could not rename %s -> %s: %s" %
                                      (from_path, to_path, e))

        app.latest_snapshot = snapshot
        self.logger.info("saving Envoy configuration for snapshot %s" %
                         snapshot)

        with open(app.bootstrap_path, "w") as output:
            output.write(json.dumps(bootstrap_config, sort_keys=True,
                                    indent=4))

        with open(app.ads_path, "w") as output:
            output.write(json.dumps(ads_config, sort_keys=True, indent=4))

        app.aconf = aconf
        app.ir = ir
        app.econf = econf
        app.diag = diag

        if app.kick:
            self.logger.info("running '%s'" % app.kick)
            os.system(app.kick)
        elif app.ambex_pid != 0:
            self.logger.info("notifying PID %d ambex" % app.ambex_pid)
            os.kill(app.ambex_pid, signal.SIGHUP)

        self.logger.info("configuration updated from snapshot %s" % snapshot)
        self._respond(rqueue, 200,
                      'configuration updated from snapshot %s' % snapshot)

        if app.health_checks and not app.stats_updater:
            app.logger.info("starting Envoy status updater")
            app.stats_updater = PeriodicTrigger(app.watcher.update_estats,
                                                period=5)

        # Check our environment...
        self.check_environment()

        self.chime()
Esempio n. 3
0
def dump(config_dir_path: Parameter.REQUIRED,
         *,
         secret_dir_path=None,
         watt=False,
         debug=False,
         debug_scout=False,
         k8s=False,
         recurse=False,
         stats=False,
         nopretty=False,
         everything=False,
         aconf=False,
         ir=False,
         v2=False,
         diag=False,
         features=False,
         profile=False):
    """
    Dump various forms of an Ambassador configuration for debugging

    Use --aconf, --ir, and --envoy to control what gets dumped. If none are requested, the IR
    will be dumped.

    :param config_dir_path: Configuration directory to scan for Ambassador YAML files
    :param secret_dir_path: Directory into which to save secrets
    :param watt: If set, input must be a WATT snapshot
    :param debug: If set, generate debugging output
    :param debug_scout: If set, generate debugging output
    :param k8s: If set, assume configuration files are annotated K8s manifests
    :param recurse: If set, recurse into directories below config_dir_path
    :param stats: If set, dump statistics to stderr
    :param nopretty: If set, do not pretty print the dumped JSON
    :param aconf: If set, dump the Ambassador config
    :param ir: If set, dump the IR
    :param v2: If set, dump the Envoy V2 config
    :param diag: If set, dump the Diagnostics overview
    :param everything: If set, dump everything
    :param features: If set, dump the feature set
    :param profile: If set, profile with the cProfile module
    """

    if not secret_dir_path:
        secret_dir_path = "/tmp/cli-secrets"

        if not os.path.isdir(secret_dir_path):
            secret_dir_path = os.path.dirname(secret_dir_path)

    if debug:
        logger.setLevel(logging.DEBUG)

    if debug_scout:
        logging.getLogger('ambassador.scout').setLevel(logging.DEBUG)

    if everything:
        aconf = True
        ir = True
        v2 = True
        diag = True
        features = True
    elif not (aconf or ir or v2 or diag or features):
        aconf = True
        ir = True
        v2 = True
        diag = False
        features = False

    dump_aconf = aconf
    dump_ir = ir
    dump_v2 = v2
    dump_diag = diag
    dump_features = features

    od = {}
    diagconfig: Optional[EnvoyConfig] = None

    _profile: Optional[cProfile.Profile] = None
    _rc = 0

    if profile:
        _profile = cProfile.Profile()
        _profile.enable()

    try:
        total_timer = Timer("total")
        total_timer.start()

        fetch_timer = Timer("fetch resources")
        with fetch_timer:
            aconf = Config()

            fetcher = ResourceFetcher(logger, aconf)

            if watt:
                fetcher.parse_watt(open(config_dir_path, "r").read())
            else:
                fetcher.load_from_filesystem(config_dir_path,
                                             k8s=k8s,
                                             recurse=True)

        load_timer = Timer("load fetched resources")
        with load_timer:
            aconf.load_all(fetcher.sorted())

        # aconf.post_error("Error from string, boo yah")
        # aconf.post_error(RichStatus.fromError("Error from RichStatus"))

        irgen_timer = Timer("ir generation")
        with irgen_timer:
            secret_handler = NullSecretHandler(logger, config_dir_path,
                                               secret_dir_path, "0")

            ir = IR(aconf,
                    file_checker=file_checker,
                    secret_handler=secret_handler)

        aconf_timer = Timer("aconf")
        with aconf_timer:
            if dump_aconf:
                od['aconf'] = aconf.as_dict()

        ir_timer = Timer("ir")
        with ir_timer:
            if dump_ir:
                od['ir'] = ir.as_dict()

        v2_timer = Timer("v2")
        with v2_timer:
            if dump_v2:
                v2config = V2Config(ir)
                diagconfig = v2config
                od['v2'] = v2config.as_dict()

        diag_timer = Timer("diag")
        with diag_timer:
            if dump_diag:
                if not diagconfig:
                    diagconfig = V2Config(ir)
                econf = typecast(EnvoyConfig, diagconfig)
                diag = Diagnostics(ir, econf)
                od['diag'] = diag.as_dict()
                od['elements'] = econf.elements

        features_timer = Timer("features")
        with features_timer:
            if dump_features:
                od['features'] = ir.features()

        # scout = Scout()
        # scout_args = {}
        #
        # if ir and not os.environ.get("AMBASSADOR_DISABLE_FEATURES", None):
        #     scout_args["features"] = ir.features()
        #
        # result = scout.report(action="dump", mode="cli", **scout_args)
        # show_notices(result)

        dump_timer = Timer("dump JSON")

        with dump_timer:
            js = dump_json(od, pretty=not nopretty)
            jslen = len(js)

        write_timer = Timer("write JSON")
        with write_timer:
            sys.stdout.write(js)
            sys.stdout.write("\n")

        total_timer.stop()

        route_count = 0
        vhost_count = 0
        filter_chain_count = 0
        filter_count = 0
        for listener in od['v2']['static_resources']['listeners']:
            for fc in listener['filter_chains']:
                filter_chain_count += 1
                for f in fc['filters']:
                    filter_count += 1
                    for vh in f['typed_config']['route_config'][
                            'virtual_hosts']:
                        vhost_count += 1
                        route_count += len(vh['routes'])

        if stats:
            sys.stderr.write("STATS:\n")
            sys.stderr.write("  config bytes:  %d\n" % jslen)
            sys.stderr.write("  vhosts:        %d\n" % vhost_count)
            sys.stderr.write("  filter chains: %d\n" % filter_chain_count)
            sys.stderr.write("  filters:       %d\n" % filter_count)
            sys.stderr.write("  routes:        %d\n" % route_count)
            sys.stderr.write("  routes/vhosts: %.3f\n" %
                             float(float(route_count) / float(vhost_count)))
            sys.stderr.write("TIMERS:\n")
            sys.stderr.write("  fetch resources:  %.3fs\n" %
                             fetch_timer.average)
            sys.stderr.write("  load resources:   %.3fs\n" %
                             load_timer.average)
            sys.stderr.write("  ir generation:    %.3fs\n" %
                             irgen_timer.average)
            sys.stderr.write("  aconf:            %.3fs\n" %
                             aconf_timer.average)
            sys.stderr.write("  envoy v2:         %.3fs\n" % v2_timer.average)
            sys.stderr.write("  diag:             %.3fs\n" %
                             diag_timer.average)
            sys.stderr.write("  features:         %.3fs\n" %
                             features_timer.average)
            sys.stderr.write("  dump json:        %.3fs\n" %
                             dump_timer.average)
            sys.stderr.write("  write json:       %.3fs\n" %
                             write_timer.average)
            sys.stderr.write("  ----------------------\n")
            sys.stderr.write("  total: %.3fs\n" % total_timer.average)
    except Exception as e:
        handle_exception("EXCEPTION from dump",
                         e,
                         config_dir_path=config_dir_path)
        _rc = 1

    if _profile:
        _profile.disable()
        _profile.dump_stats("ambassador.profile")

    sys.exit(_rc)
Esempio n. 4
0
def dump(config_dir_path: Parameter.REQUIRED, *,
         secret_dir_path=None, watt=False, debug=False, debug_scout=False, k8s=False, recurse=False,
         aconf=False, ir=False, v2=False, diag=False, features=False):
    """
    Dump various forms of an Ambassador configuration for debugging

    Use --aconf, --ir, and --envoy to control what gets dumped. If none are requested, the IR
    will be dumped.

    :param config_dir_path: Configuration directory to scan for Ambassador YAML files
    :param secret_dir_path: Directory into which to save secrets
    :param watt: If set, input must be a WATT snapshot
    :param debug: If set, generate debugging output
    :param debug_scout: If set, generate debugging output
    :param k8s: If set, assume configuration files are annotated K8s manifests
    :param recurse: If set, recurse into directories below config_dir_path
    :param aconf: If set, dump the Ambassador config
    :param ir: If set, dump the IR
    :param v2: If set, dump the Envoy V2 config
    :param diag: If set, dump the Diagnostics overview
    :param features: If set, dump the feature set
    """

    if not secret_dir_path:
        secret_dir_path = config_dir_path

        if not os.path.isdir(secret_dir_path):
            secret_dir_path = os.path.dirname(secret_dir_path)

    if debug:
        logger.setLevel(logging.DEBUG)

    if debug_scout:
        logging.getLogger('ambassador.scout').setLevel(logging.DEBUG)

    if not (aconf or ir or v2 or diag or features):
        aconf = True
        ir = True
        v2 = True
        diag = False
        features = False

    dump_aconf = aconf
    dump_ir = ir
    dump_v2 = v2
    dump_diag = diag
    dump_features = features

    od = {}
    diagconfig: Optional[EnvoyConfig] = None

    try:
        aconf = Config()
        fetcher = ResourceFetcher(logger, aconf)

        if watt:
            fetcher.parse_watt(open(config_dir_path, "r").read())
        else:
            fetcher.load_from_filesystem(config_dir_path, k8s=k8s, recurse=True)

        aconf.load_all(fetcher.sorted())

        # aconf.post_error("Error from string, boo yah")
        # aconf.post_error(RichStatus.fromError("Error from RichStatus"))

        if dump_aconf:
            od['aconf'] = aconf.as_dict()

        secret_handler = NullSecretHandler(logger, config_dir_path, secret_dir_path, "0")

        ir = IR(aconf, file_checker=file_checker, secret_handler=secret_handler)

        if dump_ir:
            od['ir'] = ir.as_dict()

        if dump_v2:
            v2config = V2Config(ir)
            diagconfig = v2config
            od['v2'] = v2config.as_dict()

        if dump_diag:
            if not diagconfig:
                diagconfig = V2Config(ir)

            econf = typecast(EnvoyConfig, diagconfig)
            diag = Diagnostics(ir, econf)
            od['diag'] = diag.as_dict()
            od['elements'] = econf.elements

        if dump_features:
            od['features'] = ir.features()

        # scout = Scout()
        # scout_args = {}
        #
        # if ir and not os.environ.get("AMBASSADOR_DISABLE_FEATURES", None):
        #     scout_args["features"] = ir.features()
        #
        # result = scout.report(action="dump", mode="cli", **scout_args)
        # show_notices(result)

        json.dump(od, sys.stdout, sort_keys=True, indent=4)
        sys.stdout.write("\n")
    except Exception as e:
        handle_exception("EXCEPTION from dump", e,
                         config_dir_path=config_dir_path)

        # This is fatal.
        sys.exit(1)
Esempio n. 5
0
    def _load_ir(self, rqueue: queue.Queue, aconf: Config, fetcher: ResourceFetcher,
                 secret_reader: Callable[['IRTLSContext', str, str], SavedSecret],
                 snapshot: str) -> None:
        aconf.load_all(fetcher.sorted())

        aconf_path = os.path.join(app.snapshot_path, "aconf-tmp.json")
        open(aconf_path, "w").write(aconf.as_json())

        ir = IR(aconf, secret_reader=secret_reader)

        ir_path = os.path.join(app.snapshot_path, "ir-tmp.json")
        open(ir_path, "w").write(ir.as_json())

        econf = EnvoyConfig.generate(ir, "V2")
        diag = Diagnostics(ir, econf)

        bootstrap_config, ads_config = econf.split_config()

        if not self.validate_envoy_config(config=ads_config):
            self.logger.info("no updates were performed due to invalid envoy configuration, continuing with current configuration...")
            app.check_scout("attempted bad update")
            self._respond(rqueue, 500, 'ignoring: invalid Envoy configuration in snapshot %s' % snapshot)
            return

        self.logger.info("rotating snapshots for snapshot %s" % snapshot)

        for from_suffix, to_suffix in [ ('-3', '-4'), ('-2', '-3'), ('-1', '-2'), ('', '-1'), ('-tmp', '') ]:
            for fmt in [ "aconf{}.json", "econf{}.json", "ir{}.json", "snapshot{}.yaml" ]:
                try:
                    from_path = os.path.join(app.snapshot_path, fmt.format(from_suffix))
                    to_path = os.path.join(app.snapshot_path, fmt.format(to_suffix))

                    # self.logger.debug("rotate: %s -> %s" % (from_path, to_path))
                    os.rename(from_path, to_path)
                except IOError as e:
                    # self.logger.debug("skip %s -> %s: %s" % (from_path, to_path, e))
                    pass
                except Exception as e:
                    self.logger.debug("could not rename %s -> %s: %s" % (from_path, to_path, e))

        self.logger.info("saving Envoy configuration for snapshot %s" % snapshot)

        with open(app.bootstrap_path, "w") as output:
            output.write(json.dumps(bootstrap_config, sort_keys=True, indent=4))

        with open(app.ads_path, "w") as output:
            output.write(json.dumps(ads_config, sort_keys=True, indent=4))

        app.aconf = aconf
        app.ir = ir
        app.econf = econf
        app.diag = diag

        if app.kick:
            self.logger.info("running '%s'" % app.kick)
            os.system(app.kick)
        elif app.ambex_pid != 0:
            self.logger.info("notifying PID %d ambex" % app.ambex_pid)
            os.kill(app.ambex_pid, signal.SIGHUP)

        self.logger.info("configuration updated from snapshot %s" % snapshot)
        self._respond(rqueue, 200, 'configuration updated from snapshot %s' % snapshot)

        if app.health_checks and not app.stats_updater:
            app.logger.info("starting Envoy status updater")
            app.stats_updater = PeriodicTrigger(app.watcher.update_estats, period=5)
            # app.scout_updater = PeriodicTrigger(lambda: app.watcher.check_scout("30s"), period=30)

        # Don't use app.check_scout; it will deadlock. And don't bother doing the Scout
        # update until after we've taken care of Envoy.
        self.check_scout("update")