Beispiel #1
0
    def test_compile_config_dot_square(self, fname_param_config):
        """Folder location.

        This is tricky because it depends on whether or not the user specified
        a config file and whether or not he also specified the "--folder" flag.

        """
        _, param, _ = fname_param_config
        param.configfile = None

        assert Filepath(".square.yaml").exists()

        # User specified "--no-config": use `.square.yaml` if it exists.
        param.no_config = False
        cfg, err = main.compile_config(param)
        assert not err and cfg.kubecontext == "dot-square"

        # User did not specify "--no-config" and`.square.yaml` exists too. This must
        # still use the default configuration.
        param.no_config = True
        cfg, err = main.compile_config(param)
        assert not err and cfg.kubecontext is None

        # User did not specify "--no-config": use `.square.yaml` if it exists.
        Filepath(".square.yaml").rename(".square.yaml.bak")
        assert not Filepath(".square.yaml").exists()
        param.no_config = False
        cfg, err = main.compile_config(param)
        assert not err and cfg.kubecontext is None
Beispiel #2
0
    def test_common_filters(self, tmp_path):
        """Deal with empty or non-existing `_common_` filter."""
        fname_ref = Filepath("tests/support/config.yaml")

        # ----------------------------------------------------------------------
        # Empty _common_ filters.
        # ----------------------------------------------------------------------
        # Clear the "_common_" filter and save the configuration again.
        ref = yaml.safe_load(fname_ref.read_text())
        ref["filters"]["_common_"].clear()
        fout = tmp_path / "corrupt.yaml"
        fout.write_text(yaml.dump(ref))

        # Load the new configuration. This must succeed and the filters must
        # match the ones defined in the file because there the "_common_"
        # filter was empty.
        cfg, err = cfgfile.load(fout)
        assert not err and ref["filters"] == cfg.filters

        # ----------------------------------------------------------------------
        # Missing _common_ filters.
        # ----------------------------------------------------------------------
        # Remove the "_common_" filter and save the configuration again.
        ref = yaml.safe_load(fname_ref.read_text())
        del ref["filters"]["_common_"]
        fout = tmp_path / "valid.yaml"
        fout.write_text(yaml.dump(ref))

        # Load the new configuration. This must succeed and the filters must
        # match the ones defined in the file because there was no "_common_"
        # filter to merge.
        cfg, err = cfgfile.load(fout)
        assert cfg.filters["_common_"] == []
        del cfg.filters["_common_"]
        assert not err and ref["filters"] == cfg.filters
Beispiel #3
0
def load_gke_config(fname: Filepath,
                    context: Optional[str],
                    disable_warnings: bool = False) -> Tuple[K8sConfig, bool]:
    """Return K8s access config for GKE cluster described in `kubeconfig`.

    Returns None if `kubeconfig` does not exist or could not be parsed.

    Inputs:
        kubconfig: str
            Name of kubeconfig file.
        context: str
            Kubeconf context. Use `None` to use default context.
        disable_warnings: bool
            Whether or not do disable GCloud warnings.

    Returns:
        Config

    """
    # Parse the kubeconfig file.
    name, user, cluster, err = load_kubeconfig(fname, context)
    if err:
        return (K8sConfig(), True)

    # Unpack the self signed certificate (Google does not register the K8s API
    # server certificate with a public CA).
    try:
        ssl_ca_cert_data = base64.b64decode(
            cluster["certificate-authority-data"])
    except KeyError:
        logit.debug(f"Context {context} in <{fname}> is not a GKE config")
        return (K8sConfig(), True)

    # Save the certificate to a temporary file. This is only necessary because
    # the requests library will need a path to the CA file - unfortunately, we
    # cannot just pass it the content.
    _, tmp = tempfile.mkstemp(text=False)
    ssl_ca_cert = Filepath(tmp)
    ssl_ca_cert.write_bytes(ssl_ca_cert_data)

    with warnings.catch_warnings(record=disable_warnings):
        try:
            cred, project_id = google.auth.default(
                scopes=['https://www.googleapis.com/auth/cloud-platform'])
            cred.refresh(google.auth.transport.requests.Request())
            token = cred.token
        except google.auth.exceptions.DefaultCredentialsError as e:
            logit.error(str(e))
            return (K8sConfig(), True)

    # Return the config data.
    logit.info("Assuming GKE cluster.")
    return K8sConfig(
        url=cluster["server"],
        token=token,
        ca_cert=ssl_ca_cert,
        client_cert=None,
        version="",
        name=cluster["name"],
    ), False
Beispiel #4
0
    def test_compile_config_default_folder(self, fname_param_config):
        """Folder location.

        This is tricky because it depends on whether or not the user specified
        a config file and whether or not he also specified the "--folder" flag.

        """
        _, param, _ = fname_param_config

        # Config file and no "--folder": config file entry wins.
        param.configfile = str(TEST_CONFIG_FNAME)
        param.folder = None
        cfg, err = main.compile_config(param)
        assert not err
        assert cfg.folder == TEST_CONFIG_FNAME.parent.absolute() / "some/path"

        # Config file and "--folder some/where": `--folder` wins.
        param.configfile = str(TEST_CONFIG_FNAME)
        param.folder = "some/where"
        cfg, err = main.compile_config(param)
        assert not err and cfg.folder == Filepath("some/where")

        # No config file and no "--folder": manifest folder must be CWD.
        param.configfile = None
        param.folder = None
        cfg, err = main.compile_config(param)
        assert not err and cfg.folder == Filepath.cwd() / "foo" / "bar"

        # No config file and "--folder some/where": must point to `some/where`.
        param.configfile = None
        param.folder = "some/where"
        cfg, err = main.compile_config(param)
        assert not err and cfg.folder == Filepath("some/where")
Beispiel #5
0
    def test_parse_commandline_get_grouping(self, tmp_path):
        """GET supports file hierarchy options."""
        kubeconfig = tmp_path / "kubeconfig.yaml"
        kubeconfig.write_text("")
        base_cmd = ("square.py", "get", "all",
                    "--kubeconfig", str(tmp_path / "kubeconfig.yaml"))

        # ----------------------------------------------------------------------
        # Default file system hierarchy.
        # ----------------------------------------------------------------------
        with mock.patch("sys.argv", base_cmd):
            param = main.parse_commandline_args()
            assert param.groupby is None

        # ----------------------------------------------------------------------
        # User defined file system hierarchy.
        # ----------------------------------------------------------------------
        cmd = ("--groupby", "ns", "kind")
        with mock.patch("sys.argv", base_cmd + cmd):
            param = main.parse_commandline_args()
            assert param.groupby == ["ns", "kind"]

        cfg, err = main.compile_config(param)
        assert not err
        assert cfg.groupby == GroupBy(label="", order=["ns", "kind"])

        # ----------------------------------------------------------------------
        # Include a label into the hierarchy and use "ns" twice.
        # ----------------------------------------------------------------------
        cmd = ("--groupby", "ns", "label=foo", "ns")
        with mock.patch("sys.argv", base_cmd + cmd):
            param = main.parse_commandline_args()
            assert param.groupby == ["ns", "label=foo", "ns"]

        cfg, err = main.compile_config(param)
        assert not err
        assert cfg.groupby == GroupBy(label="foo", order=["ns", "label", "ns"])

        # ----------------------------------------------------------------------
        # The label resource, unlike "ns" or "kind", can only be specified
        # at most once.
        # ----------------------------------------------------------------------
        cmd = ("--groupby", "ns", "label=foo", "label=bar")
        with mock.patch("sys.argv", base_cmd + cmd):
            param = main.parse_commandline_args()
            assert param.groupby == ["ns", "label=foo", "label=bar"]

        expected = Config(
            folder=Filepath(""),
            kubeconfig=Filepath(""),
            kubecontext=None,
            selectors=Selectors(set(), [], []),
            groupby=GroupBy("", []),
            priorities=[],
        )
        assert main.compile_config(param) == (expected, True)
Beispiel #6
0
def filename_for_manifest(meta: MetaManifest, manifest: dict,
                          grouping: GroupBy) -> Tuple[Filepath, bool]:
    """Return the file for the manifest based on `groupby`.

    Inputs:
        meta: MetaManifest
        manifest: dict
        groupby: GroupBy

    Output:
        Filepath

    """
    # --- Sanity checks ---
    if not set(grouping.order).issubset({"ns", "kind", "label"}):
        logit.error(f"Invalid resource ordering: {grouping.order}")
        return Filepath(), True

    if "label" in grouping.order:
        if len(grouping.label) == 0:
            logit.error("Must specify a non-empty label when grouping by it")
            return Filepath(), True

    # Convenience: reliably extract a label dictionary even when the original
    # manifest has none.
    labels = manifest.get("metadata", {}).get("labels", {})

    # Helper LookUpTable that contains the values for all those groups the
    # "--groupby" command line option accepts. We will use this LUT below to
    # assemble the full manifest path.
    lut = {
        # Get the namespace. Use "_global_" for non-namespaced resources.
        # The only exception are `Namespaces` themselves because it is neater
        # to save their manifest in the relevant namespace folder, together
        # with all the other resources that are in that namespace.
        "ns": (meta.name if meta.kind == "Namespace" else None)
        or meta.namespace or "_global_",  # noqa
        "kind":
        meta.kind.lower(),
        # Try to find the user specified label. If the current resource lacks
        # that label then put it into the catchall file.
        "label":
        labels.get(grouping.label, "_other"),
    }

    # Concatenate the components according to `grouping.order` to produce the
    # full file name. This order is what the user can specify via the
    # "--groupby" option on the command line.
    path_constituents = [lut[_] for _ in grouping.order]
    path = str.join("/", path_constituents)

    # Default to the catch-all `_other.yaml` resource if the order did not
    # produce a file name. This typically happens when `grouping.order = []`.
    path = "_other.yaml" if path == "" else f"{path}.yaml"
    return Filepath(path), False
Beispiel #7
0
 def test_compile_config_missing_k8s_credentials(self, fname_param_config):
     """Gracefully abort if kubeconfig does not exist"""
     _, param, _ = fname_param_config
     param.kubeconfig += "does-not-exist"
     assert main.compile_config(param) == (
         Config(
             folder=Filepath(""),
             kubeconfig=Filepath(""),
             kubecontext=None,
             selectors=Selectors(set(), [], []),
             groupby=GroupBy("", []),
             priorities=[],
         ), True)
Beispiel #8
0
    def test_load_err(self, tmp_path):
        """Gracefully handle missing file, corrupt content etc."""
        # Must gracefully handle a corrupt configuration file.
        fname = tmp_path / "does-not-exist.yaml"
        _, err = cfgfile.load(fname)
        assert err

        # YAML error.
        fname = tmp_path / "corrupt-yaml.yaml"
        fname.write_text("[foo")
        _, err = cfgfile.load(fname)
        assert err

        # Does not match the definition of `dtypes.Config`.
        fname = tmp_path / "invalid-pydantic-schema.yaml"
        fname.write_text("foo: bar")
        _, err = cfgfile.load(fname)
        assert err

        # YAML file is valid but not a map. This special case is important
        # because the test function will expand the content as **kwargs.
        fname = tmp_path / "invalid-pydantic-schema.yaml"
        fname.write_text("")
        _, err = cfgfile.load(fname)
        assert err

        # Load the sample configuration and corrupt the label selector. Instead
        # of a list of 2-tuples we make it a list of 3-tuples.
        cfg = yaml.safe_load(Filepath("tests/support/config.yaml").read_text())
        cfg["selectors"]["labels"] = [["foo", "bar", "foobar"]]
        fout = tmp_path / "corrupt.yaml"
        fout.write_text(yaml.dump(cfg))
        _, err = cfgfile.load(fout)
        assert err
Beispiel #9
0
    def test_load_kind_config_ok(self):
        # Load the K8s configuration for a Kind cluster.
        fname = Filepath("tests/support/kubeconf.yaml")

        ret, err = k8s.load_kind_config(fname, "kind")
        assert not err
        assert ret.url == "https://localhost:8443"
        assert ret.token == ""
        assert ret.version == ""
        assert ret.name == "kind"

        # Function must also accept pathlib.Path instances.
        ret, err = k8s.load_kind_config(pathlib.Path(fname), "kind")
        assert not err
        assert ret.url == "https://localhost:8443"
        assert ret.token == ""
        assert ret.version == ""
        assert ret.name == "kind"

        # Function must have create the credential files.
        assert ret.ca_cert.exists()
        assert ret.client_cert.crt.exists()
        assert ret.client_cert.key.exists()

        # Try to load a GKE context - must fail.
        assert k8s.load_kind_config(fname, "gke") == (K8sConfig(), True)
Beispiel #10
0
    def test_translate_resource_kinds(self, k8sconfig):
        """Translate various spellings in `selectors.kinds`"""
        cfg = Config(
            folder=Filepath('/tmp'),
            kubeconfig="",
            kubecontext=None,
            selectors=Selectors(
                kinds={"svc", 'DEPLOYMENT', "Secret"},
                namespaces=['default'],
                labels=["app=morty", "foo=bar"],
            ),
            groupby=GroupBy("", []),
            priorities=["ns", "DEPLOYMENT"],
        )

        # Convert the resource names to their correct K8s kind.
        ret = sq.translate_resource_kinds(cfg, k8sconfig)
        assert ret.selectors.kinds == {"Service", "Deployment", "Secret"}
        assert ret.priorities == ["Namespace", "Deployment"]

        # Add two invalid resource names. This must succeed but return the
        # resource names without having changed them.
        cfg.selectors.kinds.clear()
        cfg.selectors.kinds.update({"invalid", "k8s-resource-kind"})
        cfg.priorities.clear()
        cfg.priorities.extend(["invalid", "k8s-resource-kind"])
        ret = sq.translate_resource_kinds(cfg, k8sconfig)
        assert ret.selectors.kinds == {"invalid", "k8s-resource-kind"}
        assert ret.priorities == ["invalid", "k8s-resource-kind"]
Beispiel #11
0
    def test_compile_hierarchy_ok(self, fname_param_config):
        """Parse the `--groupby` argument."""
        _, param, _ = fname_param_config

        err_resp = Config(
            folder=Filepath(""),
            kubeconfig=Filepath(""),
            kubecontext=None,
            selectors=Selectors(set(), [], []),
            groupby=GroupBy("", []),
            priorities=[],
        ), True

        # ----------------------------------------------------------------------
        # Default hierarchy.
        # ----------------------------------------------------------------------
        for cmd in ["apply", "get", "plan"]:
            param.parser = cmd
            ret, err = main.compile_config(param)
            assert not err
            assert ret.groupby == GroupBy(label="app", order=["ns", "label", "kind"])
            del cmd, ret, err

        # ----------------------------------------------------------------------
        # User defined hierarchy with a valid label.
        # ----------------------------------------------------------------------
        param.parser = "get"
        param.groupby = ("ns", "kind", "label=app", "ns")
        ret, err = main.compile_config(param)
        assert not err
        assert ret.groupby == GroupBy(label="app", order=["ns", "kind", "label", "ns"])

        # ----------------------------------------------------------------------
        # User defined hierarchy with invalid labels.
        # ----------------------------------------------------------------------
        param.parser = "get"
        invalid_labels = ["label", "label=", "label=foo=bar"]
        for label in invalid_labels:
            param.groupby = ("ns", "kind", label, "ns")
            assert main.compile_config(param) == err_resp

        # ----------------------------------------------------------------------
        # User defined hierarchy with invalid resource types.
        # ----------------------------------------------------------------------
        param.parser = "get"
        param.groupby = ("ns", "unknown")
        assert main.compile_config(param) == err_resp
Beispiel #12
0
def main() -> int:
    param = parse_commandline_args()

    # Print version information and quit.
    if param.parser == "version":
        print(__version__)
        return 0

    # Create a default ".square.yaml" in the current folder and quit.
    if param.parser == "config":
        fname = Filepath(param.folder or ".") / ".square.yaml"
        fname.parent.mkdir(parents=True, exist_ok=True)
        fname.write_text(DEFAULT_CONFIG_FILE.read_text())
        print(
            f"Created configuration file <{fname}>.\n"
            "Please open the file in an editor and adjust the values, most notably "
            "<kubeconfig>, <kubecontext>, <folder> and <selectors.labels>."
        )
        return 0

    # Initialise logging.
    square.square.setup_logging(param.verbosity)

    # Create Square configuration from command line arguments.
    cfg, err1 = compile_config(param)
    cfg, err2 = expand_all_kinds(cfg)
    if err1 or err2:
        return 1

    # Do what the user asked us to do.
    if param.info:
        err = show_info(cfg)
    elif param.parser == "get":
        err = square.square.get_resources(cfg)
    elif param.parser == "plan":
        plan, err = square.square.make_plan(cfg)
        square.square.show_plan(plan)
    elif param.parser == "apply":
        err = apply_plan(cfg, "yes")
    else:
        logit.error(f"Unknown command <{param.parser}>")
        return 1

    # Return error code.
    return 1 if err else 0
Beispiel #13
0
def load(fname: Filepath) -> Tuple[Config, bool]:
    """Parse the Square configuration file `fname` and return it as a `Config`."""
    err_resp = Config(Filepath(""), kubeconfig=Filepath(""),
                      kubecontext=None), True
    fname = Filepath(fname)

    # Load the configuration file.
    try:
        raw = yaml.safe_load(Filepath(fname).read_text())
    except FileNotFoundError as e:
        logit.error(f"Cannot load config file <{fname}>: {e.args[1]}")
        return err_resp
    except yaml.YAMLError as exc:
        msg = f"Could not parse YAML file {fname}"

        # Special case: parser supplied location information.
        mark = getattr(exc, "problem_mark", SimpleNamespace(line=-1,
                                                            column=-1))
        line, col = (mark.line + 1, mark.column + 1)
        msg = f"YAML format error in {fname}: Line {line} Column {col}"
        logit.error(msg)
        return err_resp

    if not isinstance(raw, dict):
        logit.error(f"Config file <{fname}> has invalid structure")
        return err_resp

    # Parse the configuration into `ConfigFile` structure.
    try:
        cfg = Config(**raw)
    except (pydantic.ValidationError, TypeError) as e:
        logit.error(f"Schema is invalid: {e}")
        return err_resp

    # Remove the "_common_" filter and merge it into all the other filters.
    common = cfg.filters.pop("_common_", [])
    cfg.filters = {k: merge(common, v) for k, v in cfg.filters.items()}
    cfg.filters["_common_"] = common

    # Ensure the path is an absolute path.
    cfg.folder = fname.parent.absolute() / cfg.folder

    # Convert the list to a set. No functional reason.
    cfg.selectors.kinds = set(cfg.selectors.kinds)
    return cfg, False
Beispiel #14
0
    def test_cluster_config(self, m_compile_endpoints, m_version, m_load_auto, k8sconfig):
        kubeconfig = Filepath("kubeconfig")
        kubecontext = False

        m_load_auto.return_value = (k8sconfig, False)
        m_version.return_value = (k8sconfig, False)
        m_compile_endpoints.return_value = False

        assert k8s.cluster_config(kubeconfig, kubecontext) == (k8sconfig, False)
Beispiel #15
0
    def test_load(self):
        """Load and parse configuration file."""
        # Load the sample that ships with Square.
        fname = Filepath("tests/support/config.yaml")
        cfg, err = cfgfile.load(fname)
        assert not err and isinstance(cfg, Config)

        assert cfg.folder == fname.parent.absolute() / "some/path"
        assert cfg.kubeconfig == Filepath("/path/to/kubeconfig")
        assert cfg.kubecontext is None
        assert cfg.priorities == list(DEFAULT_PRIORITIES)
        assert cfg.selectors.kinds == set(DEFAULT_PRIORITIES)
        assert cfg.selectors.namespaces == ["default", "kube-system"]
        assert cfg.selectors.labels == ["app=square"]
        assert set(cfg.filters.keys()) == {
            "_common_", "ConfigMap", "Deployment", "HorizontalPodAutoscaler",
            "Service"
        }
Beispiel #16
0
def load_kind_config(fname: Filepath,
                     context: Optional[str]) -> Tuple[K8sConfig, bool]:
    """Load Kind configuration from `fname`.

    https://github.com/bsycorp/kind

    Kind is just another Minikube cluster. The only notable difference
    is that it does not store its credentials as files but directly in
    the Kubeconfig file. This function will copy those files into /tmp.

    Return None on error.

    Inputs:
        kubconfig: str
            Path to kubeconfig for Kind cluster.
        context: str
            Kubeconf context. Use `None` to use default context.

    Returns:
        Config

    """
    # Parse the kubeconfig file.
    name, user, cluster, err = load_kubeconfig(fname, context)
    if err:
        return (K8sConfig(), True)

    # Kind and Minikube use client certificates to authenticate. We need to
    # pass those to the HTTP client of our choice when we create the session.
    try:
        client_crt = base64.b64decode(user["client-certificate-data"]).decode()
        client_key = base64.b64decode(user["client-key-data"]).decode()
        client_ca = base64.b64decode(
            cluster["certificate-authority-data"]).decode()
        path = Filepath(tempfile.mkdtemp())
        p_client_crt = path / "kind-client.crt"
        p_client_key = path / "kind-client.key"
        p_ca = path / "kind.ca"
        p_client_crt.write_text(client_crt)
        p_client_key.write_text(client_key)
        p_ca.write_text(client_ca)
        client_cert = K8sClientCert(crt=p_client_crt, key=p_client_key)

        # Return the config data.
        logit.debug("Assuming Minikube/Kind cluster.")
        return K8sConfig(
            url=cluster["server"],
            token="",
            ca_cert=p_ca,
            client_cert=client_cert,
            version="",
            name=cluster["name"],
        ), False
    except KeyError:
        logit.debug(f"Context {context} in <{fname}> is not a Minikube config")
        return (K8sConfig(), True)
Beispiel #17
0
    def test_load_folder_paths(self, tmp_path):
        """The folder paths must always be relative to the config file."""
        fname = tmp_path / ".square.yaml"
        fname_ref = Filepath("tests/support/config.yaml")

        # The parsed folder must point to "tmp_path".
        ref = yaml.safe_load(fname_ref.read_text())
        fname.write_text(yaml.dump(ref))
        cfg, err = cfgfile.load(fname)
        assert not err and cfg.folder == tmp_path / "some/path"

        # The parsed folder must point to "tmp_path/folder".
        ref = yaml.safe_load(fname_ref.read_text())
        ref["folder"] = "my-folder"
        fname.write_text(yaml.dump(ref))
        cfg, err = cfgfile.load(fname)
        assert not err and cfg.folder == tmp_path / "my-folder"

        # An absolute path must ignore the position of ".square.yaml".
        # No idea how to handle this on Windows.
        if not sys.platform.startswith("win"):
            ref = yaml.safe_load(fname_ref.read_text())
            ref["folder"] = "/absolute/path"
            fname.write_text(yaml.dump(ref))
            cfg, err = cfgfile.load(fname)
            assert not err and cfg.folder == Filepath("/absolute/path")
Beispiel #18
0
def load_incluster_config(
        fname_token: Filepath = FNAME_TOKEN,
        fname_cert: Filepath = FNAME_CERT) -> Tuple[K8sConfig, bool]:
    """Return K8s access config from Pod service account.

    Returns None if we are not running in a Pod.

    Inputs:
        kubconfig: str
            Name of kubeconfig file.
    Returns:
        Config

    """
    # Every K8s pod has this.
    server_ip = os.getenv('KUBERNETES_PORT_443_TCP_ADDR', None)

    fname_cert = pathlib.Path(fname_cert)
    fname_token = pathlib.Path(fname_token)

    # Sanity checks: URL and service account files either exist, or we are not
    # actually inside a Pod.
    try:
        assert server_ip is not None
        assert fname_cert.exists()
        assert fname_token.exists()
    except AssertionError:
        logit.debug("Could not find incluster (service account) credentials.")
        return K8sConfig(), True

    # Return the compiled K8s access configuration.
    logit.info("Use incluster (service account) credentials.")
    return K8sConfig(
        url=f'https://{server_ip}',
        token=fname_token.read_text(),
        ca_cert=fname_cert,
        client_cert=None,
        version="",
        name="",
    ), False
Beispiel #19
0
def save_files(folder: Filepath, file_data: Dict[Filepath, str]) -> bool:
    """Save all `file_data` under `folder`.

    All paths in `file_data` are relative to `folder`.

    Inputs:
        folder: Filepath
        file_data: Dict[Filepath, str]
            The file name (relative to `folder`) and its content.

    Returns:
        None

    """
    # Python's `pathlib.Path` objects are simply nicer to work with...
    folder = pathlib.Path(folder)

    # Delete all YAML files under `folder`. This avoids stale manifests.
    try:
        for fp in folder.rglob("*.yaml"):
            logit.info(f"Removing stale <{fp}>")
            fp.unlink()
    except (IOError, PermissionError) as err:
        logit.error(f"{err}")
        return True

    # Iterate over the dict and write each file. Abort on error.
    for fname, yaml_str in file_data.items():
        # Skip the file if its content would be empty.
        if yaml_str == '':
            continue

        # Construct absolute file path.
        fname_abs = folder / fname
        logit.debug(f"Creating path for <{fname}>")

        # Create the parent directories and write the file. Abort on error.
        logit.info(f"Saving YAML file <{fname_abs}>")
        try:
            fname_abs.parent.mkdir(parents=True, exist_ok=True)
            fname_abs.write_text(yaml_str)
        except (IOError, PermissionError) as err:
            logit.error(f"{err}")
            return True

    # Tell caller that all files were successfully written.
    return False
Beispiel #20
0
def load(
        folder: Filepath, selectors: Selectors
) -> Tuple[ServerManifests, LocalManifestLists, bool]:
    """Recursively load all "*.yaml" files under `folder`.

    Ignores all files not ending in ".yaml". Also removes all manifests that do
    not match the `selectors`.

    Returns no data in the case of an error.

    NOTE: this is merely a wrapper around the various low-level functions to
    load and parse the YAML files.

    Input:
        folder: Filepath
            Source folder.
        selectors: Selectors

    Returns:
        (local manifest without file info, local manifests with file info)

    """
    # Python's `pathlib.Path` objects are simply nicer to work with...
    folder = pathlib.Path(folder)

    # Compile the list of all YAML files in `folder` but only store their path
    # relative to `folder`.
    fnames = [(_.relative_to(folder), _.name) for _ in folder.rglob("*.yaml")]
    fnames = [path for path, name in fnames if not name.startswith(".")]

    try:
        # Load the files and abort on error.
        fdata_raw, err = load_files(folder, fnames)
        assert not err and fdata_raw is not None

        # Return the YAML parsed manifests.
        man_files, err = parse(fdata_raw, selectors)
        assert not err and man_files is not None

        # Remove the Filepath dimension.
        man_meta, err = unpack(man_files)
        assert not err and man_meta is not None
    except AssertionError:
        return ({}, {}, True)

    # Return the file based manifests and unpacked manifests.
    return (man_meta, man_files, False)
Beispiel #21
0
def cluster_config(kubeconfig: Filepath,
                   context: Optional[str]) -> Tuple[K8sConfig, bool]:
    """Return web session to K8s API.

    This will read the Kubernetes credentials, contact Kubernetes to
    interrogate its version and then return the configuration and web-session.

    Inputs:
        kubeconfig: str
            Path to kubeconfig file.
        context: str
            Kubernetes context to use (can be `None` to use default).

    Returns:
        K8sConfig

    """
    # Read Kubeconfig file and use it to create a `requests` client session.
    # That session will have the proper security certificates and headers so
    # that subsequent calls to K8s need not deal with it anymore.
    kubeconfig = kubeconfig.expanduser()
    try:
        # Parse Kubeconfig file.
        k8sconfig, err = load_auto_config(kubeconfig,
                                          context,
                                          disable_warnings=True)
        assert not err

        # Configure web session.
        k8sconfig = k8sconfig._replace(client=session(k8sconfig))
        assert k8sconfig.client

        # Contact the K8s API to update version field in `k8sconfig`.
        k8sconfig, err = version(k8sconfig)
        assert not err and k8sconfig

        # Populate the `k8sconfig.apis` field.
        err = compile_api_endpoints(k8sconfig)
        assert not err
    except AssertionError:
        return (K8sConfig(), True)

    # Log the K8s API address and version.
    logit.info(f"Kubernetes server at {k8sconfig.url}")
    logit.info(f"Kubernetes version is {k8sconfig.version}")
    return (k8sconfig, False)
Beispiel #22
0
    def test_expand_all_kinds_err_config(self, k8sconfig):
        """Abort if the kubeconfig file does not exist."""
        cfg = Config(
            folder=pathlib.Path('/tmp'),
            kubeconfig=Filepath("/does/not/exist"),
            kubecontext=None,
            selectors=Selectors(
                kinds=set(),
                namespaces=['default'],
                labels=["app=morty", "foo=bar"],
            ),
            groupby=GroupBy("", tuple()),
            priorities=("Namespace", "Deployment"),
        )

        _, err = main.expand_all_kinds(cfg)
        assert err
Beispiel #23
0
    def test_cluster_config(self):
        """Basic success/failure test for K8s configuration."""
        # Only show INFO and above or otherwise this test will produce a
        # humongous amount of logs from all the K8s calls.
        square.square.setup_logging(2)

        # Fixtures.
        fun = square.k8s.cluster_config
        fname = Filepath("/tmp/kubeconfig-kind.yaml")

        # Must produce a valid K8s configuration.
        cfg, err = fun(fname, None)
        assert not err and isinstance(cfg, K8sConfig)

        # Gracefully handle connection errors to K8s.
        with mock.patch.object(square.k8s, "session") as m_sess:
            m_sess.return_value = None
            assert fun(fname, None) == (K8sConfig(), True)
Beispiel #24
0
    def test_nonpreferred_api(self, tmp_path):
        """Sync `autoscaling/v1` and `autoscaling/v2beta` at the same time.

        This test is designed to verify that Square will interrogate the
        correct K8s endpoint versions to download the manifest.

        """
        # Only show INFO and above or otherwise this test will produce a
        # humongous amount of useless logs from all the K8s calls.
        square.square.setup_logging(2)

        config = Config(
            folder=tmp_path,
            groupby=GroupBy(label="app", order=[]),
            kubecontext=None,
            kubeconfig=Filepath("/tmp/kubeconfig-kind.yaml"),
            selectors=Selectors(
                kinds={"Namespace", "HorizontalPodAutoscaler"},
                namespaces=["test-hpa"],
                labels=[],
            ),
        )

        # Copy the manifest with the namespace and the two HPAs to the temporary path.
        manifests = list(
            yaml.safe_load_all(open("tests/support/k8s-test-hpa.yaml")))
        man_path = tmp_path / "manifest.yaml"
        man_path.write_text(yaml.dump_all(manifests))
        assert len(manifests) == 3

        # ---------------------------------------------------------------------
        # Deploy the resources: one namespace with two HPAs in it. On will be
        # deployed via `autoscaling/v1` the other via `autoscaling/v2beta2`.
        # ---------------------------------------------------------------------
        sh.kubectl("apply", "--kubeconfig", config.kubeconfig, "-f",
                   str(man_path))

        # ---------------------------------------------------------------------
        # Sync all manifests. This must do nothing. In particular, it must not
        # change the `apiVersion` of either HPA.
        # ---------------------------------------------------------------------
        assert not square.square.get_resources(config)
        assert list(yaml.safe_load_all(man_path.read_text())) == manifests
Beispiel #25
0
def config(k8sconfig, tmp_path) -> Generator[square.dtypes.Config, None, None]:
    """Return a valid and fully populated `Config` structure.

    The data in the structure matches `tests/support/config.yaml` except for
    the `kubeconfig` file. That one is different and points to an actual
    (dummy) file in a temporary folder for this test.

    """
    # Load the sample configuration.
    cfg, err = square.cfgfile.load(Filepath("tests/support/config.yaml"))
    assert not err

    # Point the folder and kubeconfig to temporary versions.
    cfg.folder = tmp_path
    cfg.kubeconfig = (tmp_path / "kubeconf")

    # Ensure the dummy kubeconfig file exists.
    cfg.kubeconfig.write_text("")

    yield cfg
Beispiel #26
0
    def k8sconfig(self, integrationtest, ref_config):
        """Return a valid K8sConfig for a dummy or the real integration test cluster.

        The `config` is a reference K8s config. Return it if `integrationtest
        is False`, otherwise go to the real cluster. Skip the integration test
        if there is no real cluster available right now.

        """
        # Use a fake or genuine K8s cluster.
        if not integrationtest:
            return ref_config

        if not kind_available():
            pytest.skip()
        kubeconfig = Filepath("/tmp/kubeconfig-kind.yaml")

        # Create a genuine K8s config from our integration test cluster.
        k8sconfig, err = k8s.cluster_config(kubeconfig, None)
        assert not err and k8sconfig and k8sconfig.client
        return k8sconfig
Beispiel #27
0
    def test_nondefault_resources(self, tmp_path):
        """Manage an `autoscaling/v1` and `autoscaling/v2beta` at the same time.

        This test is designed to verify that Square will interrogate the
        correct K8s endpoint versions to compute the plan for a resource.

        """
        # Only show INFO and above or otherwise this test will produce a
        # humongous amount of logs from all the K8s calls.
        square.square.setup_logging(2)

        config = Config(
            folder=tmp_path,
            groupby=GroupBy(label="app", order=[]),
            kubecontext=None,
            kubeconfig=Filepath("/tmp/kubeconfig-kind.yaml"),
            selectors=Selectors(kinds={"Namespace", "HorizontalPodAutoscaler"},
                                namespaces=["test-hpa"],
                                labels=[]),
        )

        # Copy the manifest with the namespace and the two HPAs to the temporary path.
        manifests = list(
            yaml.safe_load_all(open("tests/support/k8s-test-hpa.yaml")))
        man_path = tmp_path / "manifest.yaml"
        man_path.write_text(yaml.dump_all(manifests))
        assert len(manifests) == 3

        # ---------------------------------------------------------------------
        # Deploy the resources: one namespace with two HPAs in it. On will be
        # deployed via `autoscaling/v1` the other via `autoscaling/v2beta2`.
        # ---------------------------------------------------------------------
        sh.kubectl("apply", "--kubeconfig", config.kubeconfig, "-f",
                   str(man_path))

        # ---------------------------------------------------------------------
        # The plan must be empty because Square must have interrogated the
        # correct API endpoints for each HPA.
        # ---------------------------------------------------------------------
        plan_1, err = square.square.make_plan(config)
        assert not err
        assert plan_1.create == plan_1.patch == plan_1.delete == []
        del plan_1

        # ---------------------------------------------------------------------
        # Modify the v2beta2 HPA manifest and verify that Square now wants to
        # patch that resource.
        # ---------------------------------------------------------------------
        # Make a change to the manifest and save it.
        tmp_manifests = copy.deepcopy(manifests)
        assert tmp_manifests[2]["apiVersion"] == "autoscaling/v2beta2"
        tmp_manifests[2]["spec"]["metrics"][0]["external"]["metric"][
            "name"] = "foo"
        man_path.write_text(yaml.dump_all(tmp_manifests))

        # The plan must report one patch.
        plan_2, err = square.square.make_plan(config)
        assert not err
        assert plan_2.create == plan_2.delete == [] and len(plan_2.patch) == 1
        assert plan_2.patch[0].meta.name == "hpav2beta2"
        assert plan_2.patch[0].meta.apiVersion == "autoscaling/v2beta2"
        del plan_2

        # ---------------------------------------------------------------------
        # Delete both HPAs with Square.
        # ---------------------------------------------------------------------
        # Keep only the namespace manifest and save the file.
        tmp_manifests = copy.deepcopy(manifests[:1])
        man_path.write_text(yaml.dump_all(tmp_manifests))

        # Square must now want to delete both HPAs.
        plan_3, err = square.square.make_plan(config)
        assert not err
        assert plan_3.create == plan_3.patch == [] and len(plan_3.delete) == 2
        assert {_.meta.name for _ in plan_3.delete} == {"hpav1", "hpav2beta2"}
        assert not square.square.apply_plan(config, plan_3)
        del plan_3

        # ---------------------------------------------------------------------
        # Re-create both HPAs with Square.
        # ---------------------------------------------------------------------
        # Restore the original manifest file.
        man_path.write_text(yaml.dump_all(manifests))

        # Create a plan. That plan must want to restore both HPAs.
        plan_4, err = square.square.make_plan(config)
        assert not err
        assert plan_4.delete == plan_4.patch == [] and len(plan_4.create) == 2
        assert {_.meta.name for _ in plan_4.create} == {"hpav1", "hpav2beta2"}
        assert {_.meta.apiVersion
                for _ in plan_4.create
                } == {"autoscaling/v1", "autoscaling/v2beta2"}
        assert not square.square.apply_plan(config, plan_4)
        del plan_4

        # Apply the plan.
        plan_5, err = square.square.make_plan(config)
        assert not err
        assert plan_5.create == plan_5.patch == plan_5.delete == []
        del plan_5

        # ---------------------------------------------------------------------
        # Verify that a change in the `apiVersion` would mean a patch.
        # ---------------------------------------------------------------------
        # Manually change the API version of one of the HPAs.
        tmp_manifests = copy.deepcopy(manifests)
        assert tmp_manifests[1]["apiVersion"] == "autoscaling/v1"
        tmp_manifests[1]["apiVersion"] = "autoscaling/v2beta2"
        man_path.write_text(yaml.dump_all(tmp_manifests))

        # Square must now produce a single non-empty patch.
        plan_6, err = square.square.make_plan(config)
        assert not err
        assert plan_6.delete == plan_6.create == [] and len(plan_6.patch) == 1
        assert plan_6.patch[0].meta.name == "hpav1"
        assert plan_6.patch[0].meta.apiVersion == "autoscaling/v2beta2"
Beispiel #28
0
import backoff
import google.auth
import google.auth.exceptions
import google.auth.transport.requests
import requests
import yaml

from square.dtypes import (
    Filepath,
    K8sClientCert,
    K8sConfig,
    K8sResource,
    MetaManifest,
)

FNAME_TOKEN = Filepath("/var/run/secrets/kubernetes.io/serviceaccount/token")
FNAME_CERT = Filepath("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt")

# Convenience: global logger instance to avoid repetitive code.
logit = logging.getLogger("square")


def load_kubeconfig(fname: Filepath,
                    context: Optional[str]) -> Tuple[str, dict, dict, bool]:
    """Return user name and user- and cluster information.

    Return None on error.

    Inputs:
        fname: str
            Path to kubeconfig file, eg "~/.kube/config.yaml"
Beispiel #29
0
def compile_config(cmdline_param) -> Tuple[Config, bool]:
    """Return `Config` from `cmdline_param`.

    Inputs:
        cmdline_param: SimpleNamespace

    Returns:
        Config, err

    """
    err_resp = Config(
        folder=Filepath(""),
        kubeconfig=Filepath(""),
        kubecontext=None,
        selectors=Selectors(set(), namespaces=[], labels=[]),
        groupby=GroupBy("", []),
        priorities=[],
    ), True

    # Convenience.
    p = cmdline_param

    # Load the default configuration unless the user specified an explicit one.
    if p.configfile:
        logit.info(f"Loading configuration file <{p.configfile}>")
        cfg, err = square.cfgfile.load(p.configfile)

        # Use the folder the `load()` function determined.
        folder = cfg.folder

        # Look for `--kubeconfig`. Default to the value in config file.
        kubeconfig = p.kubeconfig or cfg.kubeconfig
    else:
        # Pick the configuration file, depending on whether the user specified
        # `--no-config`, `--config` and whether a `.square.yaml` file exists.
        default_cfg = DEFAULT_CONFIG_FILE
        dot_square = Filepath(".square.yaml")
        if p.no_config:
            cfg_file = default_cfg
        else:
            cfg_file = dot_square if dot_square.exists() else default_cfg

        logit.info(f"Loading configuration file <{cfg_file}>")
        cfg, err = square.cfgfile.load(cfg_file)

        # Determine which Kubeconfig to use. The order is: `--kubeconfig`,
        # `--config`, `.square`, `KUBECONFIG` environment variable.
        if cfg_file == default_cfg:
            kubeconfig = p.kubeconfig or os.getenv("KUBECONFIG", "")
        else:
            kubeconfig = p.kubeconfig or str(cfg.kubeconfig) or os.getenv("KUBECONFIG", "")  # noqa
        del dot_square, default_cfg

        # Use the current working directory as the folder directory because the
        # user did not specify an explicit configuration file which would
        # contain the desired folder.
        folder = Filepath.cwd()

        # Abort if neither `--kubeconfig` nor KUBECONFIG env var.
        if not kubeconfig:
            logit.error("Must specify a Kubernetes config file.")
            return err_resp
    if err:
        return err_resp

    # Override the folder if user specified "--folder".
    folder = folder if p.folder is None else p.folder

    # Expand the user's home folder, ie if the path contains a "~".
    folder = Filepath(folder).expanduser()
    kubeconfig = Filepath(kubeconfig).expanduser()

    # ------------------------------------------------------------------------
    # GroupBy (determines the folder hierarchy that GET will create).
    # In : `--groupby ns kind label=app`
    # Out: GroupBy(order=["ns", "kind", "label"], label="app")
    # ------------------------------------------------------------------------
    # Unpack the ordering and replace all `label=*` with `label`.
    # NOTE: `p.groups` does not necessarily exist because the option only makes
    #        sense for eg `square GET` and is thus not implemented for `square apply`.
    order = getattr(p, "groupby", None)
    if order is None:
        groupby = cfg.groupby
    else:
        clean_order = [_ if not _.startswith("label") else "label" for _ in order]
        if not set(clean_order).issubset({"ns", "kind", "label"}):
            logit.error("Invalid definition of `groupby`")
            return err_resp

        labels = [_ for _ in order if _.startswith("label")]
        if len(labels) > 1:
            logit.error("Can only specify one `label=<name>` in `--groupby`.")
            return err_resp

        # Unpack the label name from the `--groupby` argument.
        # Example, if user specified `--groupby label=app` then we need to extract
        # the `app` part. This also includes basic sanity checks.
        label_name = ""
        if len(labels) == 1:
            try:
                _, label_name = labels[0].split("=")
                assert len(label_name) > 0
            except (ValueError, AssertionError):
                logit.error(f"Invalid label specification <{labels[0]}>")
                return err_resp

        # Compile the final `GroupBy` structure.
        groupby = GroupBy(order=list(clean_order), label=label_name)
        del order, clean_order, label_name

    # ------------------------------------------------------------------------
    # General: folder, kubeconfig, kubecontext, ...
    # ------------------------------------------------------------------------
    kinds = set(p.kinds) if p.kinds else cfg.selectors.kinds

    # Use the value from the (default) config file unless the user overrode
    # them on the command line.
    kubeconfig = Filepath(kubeconfig)
    kubecontext = p.kubecontext or cfg.kubecontext
    namespaces = cfg.selectors.namespaces if p.namespaces is None else p.namespaces
    sel_labels = cfg.selectors.labels if p.labels is None else p.labels
    priorities = p.priorities or cfg.priorities
    selectors = Selectors(kinds, namespaces, sel_labels)

    # Use filters from (default) config file because they cannot be specified
    # on the command line.
    filters = cfg.filters

    # ------------------------------------------------------------------------
    # Verify inputs.
    # ------------------------------------------------------------------------
    # Abort without credentials.
    if not kubeconfig.exists():
        logit.error(f"Cannot find Kubernetes config file <{kubeconfig}>")
        return err_resp

    # -------------------------------------------------------------------------
    # Assemble the full configuration and return it.
    # -------------------------------------------------------------------------
    cfg = Config(
        folder=folder,
        kubeconfig=kubeconfig,
        kubecontext=kubecontext,
        selectors=selectors,
        groupby=groupby,
        priorities=priorities,
        filters=filters,
    )
    return cfg, False
Beispiel #30
0
def load_eks_config(fname: Filepath,
                    context: Optional[str],
                    disable_warnings: bool = False) -> Tuple[K8sConfig, bool]:
    """Return K8s access config for EKS cluster described in `kubeconfig`.

    Returns None if `kubeconfig` does not exist or could not be parsed.

    Inputs:
        fname: Filepath
            Kubeconfig file.
        context: str
            Kubeconf context. Use `None` to use default context.
        disable_warnings: bool
            Whether or not do disable GCloud warnings.

    Returns:
        Config

    """
    # Parse the kubeconfig file.
    name, user, cluster, err = load_kubeconfig(fname, context)
    if err:
        return (K8sConfig(), True)

    # Get a copy of all env vars. We will pass that one along to the
    # sub-process, plus the env vars specified in the kubeconfig file.
    env = os.environ.copy()

    # Unpack the self signed certificate (AWS does not register the K8s API
    # server certificate with a public CA).
    try:
        ssl_ca_cert_data = base64.b64decode(
            cluster["certificate-authority-data"])
        cmd = user["exec"]["command"]
        args = user["exec"]["args"]
        env_kubeconf = user["exec"].get("env", [])
    except KeyError:
        logit.debug(f"Context {context} in <{fname}> is not an EKS config")
        return (K8sConfig(), True)

    # Convert a None value (valid value in YAML) to an empty list of env vars.
    env_kubeconf = env_kubeconf if env_kubeconf else []

    # Save the certificate to a temporary file. This is only necessary because
    # the Requests library will need a path to the CA file - unfortunately, we
    # cannot just pass it the content.
    _, tmp = tempfile.mkstemp(text=False)
    ssl_ca_cert = Filepath(tmp)
    ssl_ca_cert.write_bytes(ssl_ca_cert_data)

    # Compile the name, arguments and env vars for the command specified in kubeconf.
    cmd_args = [cmd] + args
    env_kubeconf = {_["name"]: _["value"] for _ in env_kubeconf}
    env.update(env_kubeconf)
    logit.debug(
        f"Requesting EKS certificate: {cmd_args} with envs: {env_kubeconf}")

    # Pre-format the command for the log message.
    log_cmd = (f"kubeconf={fname} kubectx={context} "
               f"cmd={cmd_args}  env={env_kubeconf}")

    # Run the specified command to produce the access token. That program must
    # produce a YAML document on stdout that specifies the bearer token.
    try:
        out = subprocess.run(cmd_args, stdout=subprocess.PIPE, env=env)
        token = yaml.safe_load(out.stdout.decode("utf8"))["status"]["token"]
    except FileNotFoundError:
        logit.error(
            f"Could not find {cmd} application to get token ({log_cmd})")
        return (K8sConfig(), True)
    except (KeyError, yaml.YAMLError):
        logit.error(
            f"Token manifest produce by {cmd_args} is corrupt ({log_cmd})")
        return (K8sConfig(), True)
    except TypeError:
        logit.error(
            f"The YAML token produced by {cmd_args} is corrupt ({log_cmd})")
        return (K8sConfig(), True)

    # Return the config data.
    logit.info("Assuming EKS cluster.")
    return K8sConfig(
        url=cluster["server"],
        token=token,
        ca_cert=ssl_ca_cert,
        client_cert=None,
        version="",
        name=cluster["name"],
    ), False