def create_kpo_task(current_uuid): #LOGGER = logging.getLogger(__name__) #LOGGER.info("__name__ >>> 2 - INFO UUID test") logging.info("This is the current_UUID:", current_uuid) return KubernetesPodOperator( namespace='kube-node-lease', image="airflowacrcontainer.azurecr.io/argspython", image_pull_secrets='acrsecret', cmds=["python", "name.py"], arguments=["Pudota", "Raja", "Sekhar"], tolerations=[ k8s.V1Toleration(key='key1', operator='Equal', value='value1', effect='NoSchedule') ], labels={"foo": "bar"}, image_pull_policy="Always", resources=resource1, name="python_task_name", task_id= "kb_task", #eval({current_uuid}), #"kb_task", #"checktask",#'{{ dag_run.conf.uuid }}', is_delete_operator_pod=False, get_logs=True, dag=dag)
def test_tolerations(self): k8s_api_tolerations = [ k8s.V1Toleration(key="key", operator="Equal", value="value") ] tolerations = [{'key': "key", 'operator': 'Equal', 'value': 'value'}] k = KubernetesPodOperator( namespace='default', image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels={"foo": "bar"}, name="name", task_id="task", in_cluster=False, do_xcom_push=False, cluster_context='default', tolerations=tolerations, ) result = k.create_pod_request_obj() client = ApiClient() self.assertEqual(type(result.spec.tolerations[0]), k8s.V1Toleration) self.assertEqual( client.sanitize_for_serialization(result)['spec']['tolerations'], tolerations) k = KubernetesPodOperator( namespace='default', image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels={"foo": "bar"}, name="name", task_id="task", in_cluster=False, do_xcom_push=False, cluster_context='default', tolerations=k8s_api_tolerations, ) result = k.create_pod_request_obj() self.assertEqual(type(result.spec.tolerations[0]), k8s.V1Toleration) self.assertEqual( client.sanitize_for_serialization(result)['spec']['tolerations'], tolerations)
def create_tolerations( tolerations: Optional[List[Dict[str, str]]] = None ) -> List[k8s.V1Toleration]: """ Creates k8s tolerations :param tolerations: :return: """ if not tolerations: return [] return [ k8s.V1Toleration( effect=toleration.get("effect"), key=toleration.get("key"), operator=toleration.get("operator"), value=toleration.get("value"), ) for toleration in tolerations ]
def test_tolerations(self): k8s_api_tolerations = [ k8s.V1Toleration(key="key", operator="Equal", value="value") ] tolerations = [{"key": "key", "operator": "Equal", "value": "value"}] k = KubernetesPodOperator( namespace="default", image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels={"foo": "bar"}, name="name", task_id="task", in_cluster=False, do_xcom_push=False, cluster_context="default", tolerations=tolerations, ) pod = k.create_pod_request_obj() sanitized_pod = self.sanitize_for_serialization(pod) assert isinstance(pod.spec.tolerations[0], k8s.V1Toleration) assert sanitized_pod["spec"]["tolerations"] == tolerations k = KubernetesPodOperator( namespace="default", image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["echo 10"], labels={"foo": "bar"}, name="name", task_id="task", in_cluster=False, do_xcom_push=False, cluster_context="default", tolerations=k8s_api_tolerations, ) pod = k.create_pod_request_obj() sanitized_pod = self.sanitize_for_serialization(pod) assert isinstance(pod.spec.tolerations[0], k8s.V1Toleration) assert sanitized_pod["spec"]["tolerations"] == tolerations
pod_affinity=k8s.V1PodAffinity( required_during_scheduling_ignored_during_execution=[ k8s.V1WeightedPodAffinityTerm( weight=1, pod_affinity_term=k8s.V1PodAffinityTerm( label_selector=k8s.V1LabelSelector(match_expressions=[ k8s.V1LabelSelectorRequirement( key="security", operator="In", values="S1") ]), topology_key="failure-domain.beta.kubernetes.io/zone", ), ) ]), ) tolerations = [k8s.V1Toleration(key="key", operator="Equal", value="value")] # [END howto_operator_k8s_cluster_resources] default_args = { "owner": "airflow", } with DAG( dag_id="example_kubernetes_operator", default_args=default_args, schedule_interval=None, start_date=days_ago(2), tags=["example"], ) as dag: k = KubernetesPodOperator(
# Use k8s_client.V1Affinity to define node affinity k8s_affinity = k8s.V1Affinity(pod_anti_affinity=k8s.V1PodAntiAffinity( required_during_scheduling_ignored_during_execution=[ k8s.V1PodAffinityTerm( label_selector=k8s.V1LabelSelector(match_expressions=[ k8s.V1LabelSelectorRequirement( key='app', operator='In', values=['airflow']) ]), topology_key='kubernetes.io/hostname', ) ])) # Use k8s_client.V1Toleration to define node tolerations k8s_tolerations = [ k8s.V1Toleration(key='dedicated', operator='Equal', value='airflow') ] # Use k8s_client.V1ResourceRequirements to define resource limits k8s_resource_requirements = k8s.V1ResourceRequirements( requests={'memory': '512Mi'}, limits={'memory': '512Mi'}) kube_exec_config_resource_limits = { "pod_override": k8s.V1Pod(spec=k8s.V1PodSpec( containers=[ k8s.V1Container( name="base", resources=k8s_resource_requirements, ) ],