def _exec_test(fpath):
fpath = resolve_relative_path(fpath)
logging.info(style.CYAN(f"Testing {fpath}:"))
body = None
with open(fpath, "r") as raw:
body = list(yaml.safe_load_all(raw.read()))
test_name = re.sub(r"[^a-zA-Z0-9]", "-",
os.path.splitext(os.path.basename(fpath))[0])
idx = 0
for resource in body:
if resource.get("metadata", {}).get("name") is None:
resource["metadata"] = resource.get("metadata", {})
resource["metadata"][
"name"] = test_name if idx == 0 else f"{test_name}-{str(idx)}"
idx += 1
runner = JobRunner(
body,
auto_load_kube_config=True,
logger=logging,
delete_policy=JobRunnerDeletePolicy.Always) # type:ignore
runner.execute_job()
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api.client import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api.queries import GetPodLogs
client = KubeApiRestClient()
query = GetPodLogs("tester", follow=True)
query.pipe_to_logger()
client.query_async(query)
query.wait_until_running()
logging.info("Starting watch...")
query.join()
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import Event
from airflow_kubernetes_job_operator.kube_api import KubeResourceKind
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import GetNamespaceResources
KubeResourceKind.register_global_kind(
KubeResourceKind("HCjob",
"hc.dto.cbsinteractive.com/v1alpha1",
parse_kind_state=KubeResourceKind.parse_state_job))
client = KubeApiRestClient()
query = GetNamespaceResources("Pod", namespace=client.get_default_namespace())
logging.info(f"Looking for {query.resource_path}...")
ev: Event = None
for ev in client.stream(query):
if ev.name == query.data_event_name:
logging.info(ev.args[0]["metadata"]["name"])
from zthreading.tasks import Task
from airflow_kubernetes_job_operator.kube_api.client import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import CreateNamespaceResource, DeleteNamespaceResource
def load_yaml_obj_configs(fpath: str):
if fpath.startswith("./"):
fpath = os.path.join(os.path.dirname(__file__), fpath)
fpath = os.path.abspath(fpath)
assert os.path.isfile(fpath), Exception(
f"{fpath} is not a file or dose not exist")
col = []
with open(fpath, "r") as raw:
col = list(yaml.safe_load_all(raw.read()))
return col
client = KubeApiRestClient()
obj_definitions = load_yaml_obj_configs("./test_deployment.yaml")
obj: dict = None
queries = []
for obj in obj_definitions:
q = DeleteNamespaceResource(obj, namespace=client.get_default_namespace())
q.pipe_to_logger(logging)
queries.append(q)
logging.info("Sending to server...")
rslt = client.query(queries)
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import KubeResourceKind
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import GetPodLogs
from airflow_kubernetes_job_operator.kube_api import kube_logger
KubeResourceKind.register_global_kind(
KubeResourceKind("HCjob",
"hc.dto.cbsinteractive.com/v1alpha1",
parse_kind_state=KubeResourceKind.parse_state_job))
kube_logger.level = logging.DEBUG
client = KubeApiRestClient()
logger = GetPodLogs(
name="logs-tester",
namespace=client.get_default_namespace(),
follow=True,
)
logger.pipe_to_logger()
rslt = client.query_async(logger)
label = f"{logger.namespace}/{logger.name}"
logging.info(f"Waiting for logs @ {label}...")
logger.wait_until_running()
logging.info(f"Starting logs watch @ {label}...")
logger.join()
logging.info(f"Watch for logs ended @ {label}")
from tests.utils import logging
from airflow_kubernetes_job_operator.kube_api import KubeResourceKind
from airflow_kubernetes_job_operator.kube_api import KubeApiRestClient
from airflow_kubernetes_job_operator.kube_api import NamespaceWatchQuery
KubeResourceKind.register_global_kind(
KubeResourceKind("HCjob", "hc.dto.cbsinteractive.com/v1alpha1", parse_kind_state=KubeResourceKind.parse_state_job)
)
client = KubeApiRestClient()
watcher = NamespaceWatchQuery(watch_pod_logs=True)
watcher.pipe_to_logger()
rslt = client.query_async(watcher)
logging.info(f"Waiting for watch @ {client.get_default_namespace()}...")
watcher.wait_until_running()
logging.info(f"Starting watch @ {client.get_default_namespace()}...")
watcher.join()
def update_metadata_labels(obj: dict, labels: dict):
for v in list(obj.values()):
if isinstance(v, dict):
update_metadata_labels(v, labels)
if "spec" in obj:
obj["metadata"] = obj.get("metadata", {})
obj["metadata"]["labels"] = obj["metadata"].get("labels", {})
obj["metadata"]["labels"].update(labels)
watcher = NamespaceWatchQuery(namespace=namespace, label_selector=label_selector)
watcher.pipe_to_logger(logging)
client.query_async(watcher)
watcher.wait_until_running()
logging.info(f"Watcher started for {task_id}, watch kinds: " + ", ".join(watcher.kinds.keys()))
logging.info(f"Sending {task} to server with dependencies...")
client.query_async(apply_on_objects(CreateNamespaceResource))
logging.info(f"Waiting for {task} of kind {task.kind.name} to succeed or fail ...")
final_state = watcher.wait_for_state(
[KubeResourceState.Failed, KubeResourceState.Succeeded],
kind=task.kind,
namespace=task.namespace,
name=task.name,
)
logging.info(f"Finalized state with: {final_state}")
logging.info("Deleting leftovers..")
client.query(apply_on_objects(DeleteNamespaceResource))
KubeApiRestClient,
CreateNamespaceResource,
DeleteNamespaceResource,
ConfigureNamespaceResource,
NamespaceWatchQuery,
)
objs = load_yaml_objects("./client/test_service.yaml")
client = KubeApiRestClient()
watcher = NamespaceWatchQuery()
def create_queries(action: Type[ConfigureNamespaceResource]):
queries = [
action(o, namespace=client.get_default_namespace()) for o in objs
]
for q in queries:
q.pipe_to_logger(logging)
return queries
client.query_async(watcher)
watcher.wait_until_running()
logging.info("Watcher is running, creating")
client.query(create_queries(CreateNamespaceResource))
logging.info("All created.")
client.query(create_queries(DeleteNamespaceResource))
logging.info("All deleted")
watcher.stop()