from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import KubeResourceKind
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import GetPodLogs
from airflow_kubernetes_job_operator.kube_api import kube_logger
KubeResourceKind.register_global_kind(
KubeResourceKind("HCjob",
"hc.dto.cbsinteractive.com/v1alpha1",
parse_kind_state=KubeResourceKind.parse_state_job))
kube_logger.level = logging.DEBUG
client = KubeApiRestClient()
logger = GetPodLogs(
name="logs-tester",
namespace=client.get_default_namespace(),
follow=True,
)
logger.pipe_to_logger()
rslt = client.query_async(logger)
label = f"{logger.namespace}/{logger.name}"
logging.info(f"Waiting for logs @ {label}...")
logger.wait_until_running()
logging.info(f"Starting logs watch @ {label}...")
logger.join()
logging.info(f"Watch for logs ended @ {label}")
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import Event
from airflow_kubernetes_job_operator.kube_api import KubeResourceKind
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import GetNamespaceResources
KubeResourceKind.register_global_kind(
KubeResourceKind("HCjob",
"hc.dto.cbsinteractive.com/v1alpha1",
parse_kind_state=KubeResourceKind.parse_state_job))
client = KubeApiRestClient()
query = GetNamespaceResources("Pod", namespace=client.get_default_namespace())
logging.info(f"Looking for {query.resource_path}...")
ev: Event = None
for ev in client.stream(query):
if ev.name == query.data_event_name:
logging.info(ev.args[0]["metadata"]["name"])
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import KubeResourceKind
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import NamespaceWatchQuery
KubeResourceKind.register_global_kind(
KubeResourceKind("HCjob", "hc.dto.cbsinteractive.com/v1alpha1", parse_kind_state=KubeResourceKind.parse_state_job)
)
client = KubeApiRestClient()
watcher = NamespaceWatchQuery(watch_pod_logs=True)
watcher.pipe_to_logger()
rslt = client.query_async(watcher)
logging.info(f"Waiting for watch @ {client.get_default_namespace()}...")
watcher.wait_until_running()
logging.info(f"Starting watch @ {client.get_default_namespace()}...")
watcher.join()
KubeResourceKind("HCjob", "hc.dto.cbsinteractive.com/v1alpha1", parse_kind_state=KubeResourceKind.parse_state_job)
)
def load_yaml_obj_configs(fpath: str):
if fpath.startswith("./") or fpath.startswith("../"):
fpath = os.path.join(os.path.dirname(__file__), fpath)
fpath = os.path.abspath(fpath)
assert os.path.isfile(fpath), Exception(f"{fpath} is not a file or dose not exist")
col = []
with open(fpath, "r") as raw:
col = list(yaml.safe_load_all(raw.read()))
return col
client = KubeApiRestClient()
namespace = client.get_default_namespace()
task_id = str(uuid1())
label_selector = None
label_selector = f"task-id={task_id}"
obj_definitions = load_yaml_obj_configs("../../.local/test_custom.yaml")
task = KubeResourceDescriptor(obj_definitions[0])
obj: dict = None
kinds = set()
def apply_on_objects(operator):
queries = []
for obj in obj_definitions:
update_metadata_labels(obj, {"task-id": task_id})
from tests.utils import logging, load_yaml_objects
from time import sleep
from typing import Type
from airflow_kubernetes_job_operator.kube_api import (
KubeApiRestClient,
CreateNamespaceResource,
DeleteNamespaceResource,
ConfigureNamespaceResource,
NamespaceWatchQuery,
)
objs = load_yaml_objects("./client/test_service.yaml")
client = KubeApiRestClient()
watcher = NamespaceWatchQuery()
def create_queries(action: Type[ConfigureNamespaceResource]):
queries = [
action(o, namespace=client.get_default_namespace()) for o in objs
]
for q in queries:
q.pipe_to_logger(logging)
return queries
client.query_async(watcher)
watcher.wait_until_running()
logging.info("Watcher is running, creating")
client.query(create_queries(CreateNamespaceResource))
logging.info("All created.")
import os
import yaml
import json
import kubernetes
import kubernetes.config.kube_config
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient, GetAPIResources, GetAPIVersions
client = KubeApiRestClient()
kubernetes.config.kube_config.Configuration.set_default(client.kube_config)
versions = kubernetes.client.CoreApi().get_api_versions()
kubernetes.client.AppsV1Api().patch_namespaced_deployment
rsp = client.query(GetAPIVersions())
rsp = client.query(GetAPIResources())
with open(os.path.join(os.path.dirname(__file__), "api_resources.json"), "w") as writer:
resources = rsp["resources"]
writer.write(json.dumps(resources))
def __init__(
self,
body: Union[str, dict, List[dict]],
namespace: str = None,
logger: Logger = kube_logger,
show_pod_logs: bool = True,
show_operation_logs: bool = True,
show_watcher_logs: bool = True,
show_executor_logs: bool = True,
show_error_logs: bool = True,
delete_policy: JobRunnerDeletePolicy = JobRunnerDeletePolicy.
IfSucceeded,
auto_load_kube_config=True,
random_name_postfix_length: int = 8,
name_prefix: str = None,
name_postfix: str = None,
):
"""A kubernetes job runner. Creates a collection of resources and waits for the
first resource to reach one of KubeResourceState.Succeeded/KubeResourceState.Failed/KubeResourceState.Deleted
Args:
body (Union[str, dict, List[dict]]): The execution body (kubernetes definition). If string then yaml is
expected.
namespace (str, optional): The default namespace to execute in. Defaults to None.
logger (Logger, optional): The logger to write to. Defaults to kube_logger.
show_pod_logs (bool, optional): If true, follow all pod logs. Defaults to True.
show_operation_logs (bool, optional): If true shows the operation logs (runner). Defaults to True.
show_watcher_logs (bool, optional): If true, shows the watcher events. Defaults to True.
show_executor_logs (bool, optional): If true, shows the create event events. Defaults to True.
show_error_logs (bool, optional): If true, shows errors in the log. Defaults to True.
delete_policy (JobRunnerDeletePolicy, optional): Determines when to delete the job resources upon finishing
. Defaults to JobRunnerDeletePolicy.IfSucceeded.
auto_load_kube_config (bool, optional): If true, and kube_config is None, load it. Defaults to True.
random_name_postfix_length (int, optional): Add a random string to all resource names if > 0. Defaults to 8.
name_prefix (str, optional): The prefix for the all resource names. Defaults to None.
name_postfix (str, optional): The postfix for all resource name. Defaults to None.
"""
body = body if not isinstance(body, str) else list(
yaml.safe_load_all(body))
body = body if isinstance(body, list) else [body]
assert all(isinstance(r, dict) for r in body), ValueError(
"body must be a dictionary, a list of dictionaries with at least one value, or string yaml"
)
self._client = KubeApiRestClient(
auto_load_kube_config=auto_load_kube_config)
self._is_body_ready = False
self._body = body
self.logger: Logger = logger
self._id = str(uuid4())
self.name_postfix = (name_postfix or ""
if random_name_postfix_length <= 0 else
random_string((random_name_postfix_length)))
self.name_prefix = name_prefix
self.namespace = namespace
self.show_pod_logs = show_pod_logs
self.show_operation_logs = show_operation_logs
self.show_watcher_logs = show_watcher_logs
self.show_executor_logs = show_executor_logs
self.show_error_logs = show_error_logs
self.delete_policy = delete_policy
if self.show_runner_id_on_logs is None:
self.show_runner_id_on_logs = SHOW_RUNNER_ID_IN_LOGS
super().__init__()