def create_kpo_task(current_uuid):
#LOGGER = logging.getLogger(__name__)
#LOGGER.info("__name__ >>> 2 - INFO UUID test")
logging.info("This is the current_UUID:", current_uuid)
return KubernetesPodOperator(
namespace='kube-node-lease',
image="airflowacrcontainer.azurecr.io/argspython",
image_pull_secrets='acrsecret',
cmds=["python", "name.py"],
arguments=["Pudota", "Raja", "Sekhar"],
tolerations=[
k8s.V1Toleration(key='key1',
operator='Equal',
value='value1',
effect='NoSchedule')
],
labels={"foo": "bar"},
image_pull_policy="Always",
resources=resource1,
name="python_task_name",
task_id=
"kb_task", #eval({current_uuid}), #"kb_task", #"checktask",#'{{ dag_run.conf.uuid }}',
is_delete_operator_pod=False,
get_logs=True,
dag=dag)
def test_tolerations(self):
k8s_api_tolerations = [
k8s.V1Toleration(key="key", operator="Equal", value="value")
]
tolerations = [{'key': "key", 'operator': 'Equal', 'value': 'value'}]
k = KubernetesPodOperator(
namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo 10"],
labels={"foo": "bar"},
name="name",
task_id="task",
in_cluster=False,
do_xcom_push=False,
cluster_context='default',
tolerations=tolerations,
)
result = k.create_pod_request_obj()
client = ApiClient()
self.assertEqual(type(result.spec.tolerations[0]), k8s.V1Toleration)
self.assertEqual(
client.sanitize_for_serialization(result)['spec']['tolerations'],
tolerations)
k = KubernetesPodOperator(
namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo 10"],
labels={"foo": "bar"},
name="name",
task_id="task",
in_cluster=False,
do_xcom_push=False,
cluster_context='default',
tolerations=k8s_api_tolerations,
)
result = k.create_pod_request_obj()
self.assertEqual(type(result.spec.tolerations[0]), k8s.V1Toleration)
self.assertEqual(
client.sanitize_for_serialization(result)['spec']['tolerations'],
tolerations)
def create_tolerations(
tolerations: Optional[List[Dict[str, str]]] = None
) -> List[k8s.V1Toleration]:
"""
Creates k8s tolerations
:param tolerations:
:return:
"""
if not tolerations:
return []
return [
k8s.V1Toleration(
effect=toleration.get("effect"),
key=toleration.get("key"),
operator=toleration.get("operator"),
value=toleration.get("value"),
) for toleration in tolerations
]
def test_tolerations(self):
k8s_api_tolerations = [
k8s.V1Toleration(key="key", operator="Equal", value="value")
]
tolerations = [{"key": "key", "operator": "Equal", "value": "value"}]
k = KubernetesPodOperator(
namespace="default",
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo 10"],
labels={"foo": "bar"},
name="name",
task_id="task",
in_cluster=False,
do_xcom_push=False,
cluster_context="default",
tolerations=tolerations,
)
pod = k.create_pod_request_obj()
sanitized_pod = self.sanitize_for_serialization(pod)
assert isinstance(pod.spec.tolerations[0], k8s.V1Toleration)
assert sanitized_pod["spec"]["tolerations"] == tolerations
k = KubernetesPodOperator(
namespace="default",
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo 10"],
labels={"foo": "bar"},
name="name",
task_id="task",
in_cluster=False,
do_xcom_push=False,
cluster_context="default",
tolerations=k8s_api_tolerations,
)
pod = k.create_pod_request_obj()
sanitized_pod = self.sanitize_for_serialization(pod)
assert isinstance(pod.spec.tolerations[0], k8s.V1Toleration)
assert sanitized_pod["spec"]["tolerations"] == tolerations
pod_affinity=k8s.V1PodAffinity(
required_during_scheduling_ignored_during_execution=[
k8s.V1WeightedPodAffinityTerm(
weight=1,
pod_affinity_term=k8s.V1PodAffinityTerm(
label_selector=k8s.V1LabelSelector(match_expressions=[
k8s.V1LabelSelectorRequirement(
key="security", operator="In", values="S1")
]),
topology_key="failure-domain.beta.kubernetes.io/zone",
),
)
]),
)
tolerations = [k8s.V1Toleration(key="key", operator="Equal", value="value")]
# [END howto_operator_k8s_cluster_resources]
default_args = {
"owner": "airflow",
}
with DAG(
dag_id="example_kubernetes_operator",
default_args=default_args,
schedule_interval=None,
start_date=days_ago(2),
tags=["example"],
) as dag:
k = KubernetesPodOperator(
# Use k8s_client.V1Affinity to define node affinity
k8s_affinity = k8s.V1Affinity(pod_anti_affinity=k8s.V1PodAntiAffinity(
required_during_scheduling_ignored_during_execution=[
k8s.V1PodAffinityTerm(
label_selector=k8s.V1LabelSelector(match_expressions=[
k8s.V1LabelSelectorRequirement(
key='app', operator='In', values=['airflow'])
]),
topology_key='kubernetes.io/hostname',
)
]))
# Use k8s_client.V1Toleration to define node tolerations
k8s_tolerations = [
k8s.V1Toleration(key='dedicated', operator='Equal', value='airflow')
]
# Use k8s_client.V1ResourceRequirements to define resource limits
k8s_resource_requirements = k8s.V1ResourceRequirements(
requests={'memory': '512Mi'}, limits={'memory': '512Mi'})
kube_exec_config_resource_limits = {
"pod_override":
k8s.V1Pod(spec=k8s.V1PodSpec(
containers=[
k8s.V1Container(
name="base",
resources=k8s_resource_requirements,
)
],
