def execute(self, context):
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file)
gen = pod_generator.PodGenerator()
for port in self.ports:
gen.add_port(port)
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.arguments,
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.image_pull_secrets = self.image_pull_secrets
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.hostnetwork = self.hostnetwork
pod.tolerations = self.tolerations
pod.configmaps = self.configmaps
pod.security_context = self.security_context
pod.pod_runtime_info_envs = self.pod_runtime_info_envs
pod.dnspolicy = self.dnspolicy
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.xcom_push)
try:
(final_state, result) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
finally:
if self.is_delete_operator_pod:
launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
if self.xcom_push:
return result
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
client = kube_client.get_kube_client(in_cluster=self.in_cluster)
gen = pod_generator.PodGenerator()
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.arguments,
labels=self.labels
)
pod.secrets = self.secrets
launcher = pod_launcher.PodLauncher(client)
final_state = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
if final_state != State.SUCCESS:
raise AirflowException('Pod returned a failure')
except AirflowException as ex:
raise AirflowException('Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file)
gen = pod_generator.PodGenerator()
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
if self.in_cluster:
worker_pod_name = os.environ.get('WORKER_POD_NAME')
worker_pod_uid = os.environ.get('WORKER_POD_UID')
gen.add_ownerreference(worker_pod_name, "v1", "Pod",
worker_pod_uid)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.arguments,
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.image_pull_secrets = self.image_pull_secrets
pod.hostnetwork = self.hostnetwork
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.xcom_push)
(final_state, result) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
# if self.is_delete_operator_pod:
# launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
if self.xcom_push:
return result
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file)
gen = pod_generator.PodGenerator()
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.arguments,
labels=self.labels,
)
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.xcom_push)
(final_state, result) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
if self.xcom_push:
return result
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
from dagster_graphql.client.mutations import (
DagsterGraphQLClientError,
handle_execution_errors,
handle_execute_plan_result_raw,
)
except ImportError:
raise AirflowException(
'To use the DagsterKubernetesPodOperator, dagster and dagster_graphql must be'
' installed in your Airflow environment.'
)
if 'run_id' in self.params:
self._run_id = self.params['run_id']
elif 'dag_run' in context and context['dag_run'] is not None:
self._run_id = context['dag_run'].run_id
# return to original execute code:
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file,
)
gen = pod_generator.PodGenerator()
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.query(context.get('ts')),
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.image_pull_secrets = self.image_pull_secrets
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.hostnetwork = self.hostnetwork
pod.tolerations = self.tolerations
pod.configmaps = self.configmaps
pod.security_context = self.security_context
launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push)
try:
if self.instance:
tags = (
{AIRFLOW_EXECUTION_DATE_STR: context.get('ts')} if 'ts' in context else {}
)
run = self.instance.register_managed_run(
pipeline_name=self.pipeline_name,
run_id=self.run_id,
run_config=self.run_config,
mode=self.mode,
solids_to_execute=None,
step_keys_to_execute=None,
tags=tags,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=self.pipeline_snapshot,
execution_plan_snapshot=self.execution_plan_snapshot,
parent_pipeline_snapshot=self.parent_pipeline_snapshot,
)
# we won't use the "result", which is the pod's xcom json file
(final_state, _) = launcher.run_pod(
pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs
)
# fetch the last line independently of whether logs were read
# unbelievably, if you set tail_lines=1, the returned json has its double quotes
# turned into unparseable single quotes
res = None
num_attempts = 0
while not res and num_attempts < LOG_RETRIEVAL_MAX_ATTEMPTS:
raw_res = client.read_namespaced_pod_log(
name=pod.name, namespace=pod.namespace, container='base'
)
res = parse_raw_log_lines(raw_res.split('\n'))
time.sleep(LOG_RETRIEVAL_WAITS_BETWEEN_ATTEMPTS_SEC)
num_attempts += 1
try:
handle_execution_errors(res, 'executePlan')
except DagsterGraphQLClientError as err:
self.instance.report_engine_event(
str(err),
run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())
),
self.__class__,
)
raise
events = handle_execute_plan_result_raw(res)
if self.instance:
for event in events:
self.instance.handle_new_event(event)
events = [e.dagster_event for e in events]
check_events_for_failures(events)
check_events_for_skips(events)
return events
finally:
self._run_id = None
if self.is_delete_operator_pod:
launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException('Pod returned a failure: {state}'.format(state=final_state))
# note the lack of returning the default xcom
except AirflowException as ex:
raise AirflowException('Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
from dagster_graphql.implementation.pipeline_execution_manager import (
build_synthetic_pipeline_error_record, )
from dagster_graphql.client.mutations import (
DagsterGraphQLClientError,
handle_execution_errors,
handle_execute_plan_result_raw,
)
except ImportError:
raise AirflowException(
'To use the DagsterKubernetesPodOperator, dagster and dagster_graphql must be'
' installed in your Airflow environment.')
if 'run_id' in self.params:
self._run_id = self.params['run_id']
elif 'dag_run' in context and context['dag_run'] is not None:
self._run_id = context['dag_run'].run_id
# return to original execute code:
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file,
)
gen = pod_generator.PodGenerator()
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.query,
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.image_pull_secrets = self.image_pull_secrets
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.hostnetwork = self.hostnetwork
pod.tolerations = self.tolerations
pod.configmaps = self.configmaps
pod.security_context = self.security_context
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.xcom_push)
try:
if self.instance:
self.instance.get_or_create_run(
PipelineRun(
pipeline_name=self.pipeline_name,
run_id=self.run_id,
environment_dict=self.environment_dict,
mode=self.mode,
selector=ExecutionSelector(self.pipeline_name),
reexecution_config=None,
step_keys_to_execute=None,
tags=None,
status=PipelineRunStatus.MANAGED,
))
# we won't use the "result", which is the pod's xcom json file
(final_state, _) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
# fetch the last line independently of whether logs were read
# unbelievably, if you set tail_lines=1, the returned json has its double quotes
# turned into unparseable single quotes
# TODO: add retries - k8s log servers are _extremely_ flaky
raw_res = client.read_namespaced_pod_log(
name=pod.name,
namespace=pod.namespace,
container='base',
tail_lines=5)
res = parse_raw_res(raw_res.split('\n'))
try:
handle_execution_errors(res, 'executePlan')
except DagsterGraphQLClientError:
event = build_synthetic_pipeline_error_record(
self.run_id,
serializable_error_info_from_exc_info(sys.exc_info()),
self.pipeline_name,
)
if self.instance:
self.instance.handle_new_event(event)
raise
events = handle_execute_plan_result_raw(res)
if self.instance:
for event in events:
self.instance.handle_new_event(event)
check_raw_events_for_skips(events)
return events
finally:
self._run_id = None
if self.is_delete_operator_pod:
launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
# note the lack of returning the default xcom
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
from dagster_graphql.client.mutations import (
handle_start_pipeline_execution_errors,
handle_start_pipeline_execution_result,
)
except ImportError:
raise AirflowException(
'To use the DagsterKubernetesPodOperator, dagster and dagster_graphql must be'
' installed in your Airflow environment.')
if 'run_id' in self.params:
self._run_id = self.params['run_id']
elif 'dag_run' in context and context['dag_run'] is not None:
self._run_id = context['dag_run'].run_id
# return to original execute code:
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file,
)
gen = pod_generator.PodGenerator()
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.query,
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.image_pull_secrets = self.image_pull_secrets
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.hostnetwork = self.hostnetwork
pod.tolerations = self.tolerations
pod.configmaps = self.configmaps
pod.security_context = self.security_context
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.xcom_push)
try:
# we won't use the "result", which is the pod's xcom json file
(final_state, _) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
# fetch the last line independently of whether logs were read
# unbelievably, if you set tail_lines=1, the returned json has its double quotes
# turned into unparseable single quotes
# TODO: add retries - k8s log servers are _extremely_ flaky
raw_res = client.read_namespaced_pod_log(
name=pod.name,
namespace=pod.namespace,
container='base',
tail_lines=5)
# find the relevant line
# TODO: raise sensible exception on garbage API string responses
res = parse_raw_res(raw_res)
handle_start_pipeline_execution_errors(res)
events = handle_start_pipeline_execution_result(res)
check_events_for_skips(events)
return events
finally:
self._run_id = None
if self.is_delete_operator_pod:
launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
# note the lack of returning the default xcom
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
(
pod_template,
deployment,
) = get_pod_template_from_deployment_labels_and_namespace(
namespace=self.deployment_namespace or self.namespace,
config_file=self.config_file,
cluster_context=self.cluster_context,
in_cluster=self.in_cluster,
fields=self.deployment_fields,
labels=self.deployment_labels,
)
pod_spec: V1PodSpec = pod_template.spec
container: V1Container = pod_spec.containers[0]
metadata: V1ObjectMeta = pod_template.metadata
(
plain_env_vars,
container_secrets,
container_config_maps,
runtime_info_envs,
) = handle_container_environment_variables(container.env)
self.image = self.image or container.image
self.cmds = self.cmds or container.command
self.arguments = self.arguments or container.args or []
self.labels = self.labels or metadata.labels or {}
self.name = self._set_name(self.name or deployment.metadata.name)
self.env_vars = self.env_vars or plain_env_vars
self.ports = self.ports or convert_ports(container)
self.volume_mounts = self.volume_mounts or convert_volume_mounts(
container)
self.volumes = self.volumes or convert_volumes(pod_spec)
self.secrets = self.secrets or container_secrets
self.image_pull_policy = (self.image_pull_policy
or container.image_pull_policy
or "IfNotPresent")
self.node_selectors = self.node_selectors or pod_spec.node_selector or {}
self.annotations = self.annotations or metadata.annotations or {}
self.affinity = self.affinity or convert_affinity(pod_spec)
self.resources = (self.resources if (self.resources.has_limits()
or self.resources.has_requests())
else convert_resources(container))
self.image_pull_secrets = self.image_pull_secrets or convert_image_pull_secrets(
pod_spec)
self.service_account_name = (self.service_account_name
or pod_spec.service_account_name
or pod_spec.service_account or "default")
self.hostnetwork = (pod_spec.host_network or False
if self.hostnetwork is None else self.hostnetwork)
self.tolerations = self.tolerations or convert_tolerations(pod_spec)
self.configmaps = self.configmaps or container_config_maps
self.security_context = self.security_context or convert_security_context(
pod_spec)
self.pod_runtime_info_envs = self.pod_runtime_info_envs or runtime_info_envs
self.dnspolicy = self.dnspolicy or pod_spec.dns_policy
self.log.info("volumes %s", self.volumes)
try:
if self.in_cluster is not None:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file)
else:
client = kube_client.get_kube_client(
cluster_context=self.cluster_context,
config_file=self.config_file)
# Add Airflow Version to the label
# And a label to identify that pod is launched by KubernetesPodOperator
self.labels.update({
'airflow_version':
airflow_version.replace('+', '-'),
'kubernetes_pod_operator':
'True',
})
gen = pod_generator.PodGenerator()
for port in self.ports:
gen.add_port(port)
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.arguments,
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.image_pull_secrets = self.image_pull_secrets
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.hostnetwork = self.hostnetwork
pod.tolerations = self.tolerations
pod.configmaps = self.configmaps
pod.security_context = self.security_context
pod.pod_runtime_info_envs = self.pod_runtime_info_envs
pod.dnspolicy = self.dnspolicy
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.do_xcom_push)
# monkey patch to avoid https://github.com/apache/airflow/issues/8275
launcher.kube_req_factory.extract_env_and_secrets = extract_env_and_secrets
try:
(final_state, result) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
finally:
if self.is_delete_operator_pod:
launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
if self.do_xcom_push:
return result
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context):
try:
conf = context['dag_run'].conf
self.image = conf.get('docker_image_name')
if conf.get("commands"):
self.cmds = conf.get("commands")
if conf.get("arguments"):
self.arguments = conf.get("arguments")
if conf.get("env_vars"):
self.env_vars = conf.get("env_vars")
except Exception as e:
raise XKubernetesPodOperatorException(
"Could not start off with Dag Run Configuration", e)
try:
client = kube_client.get_kube_client(
in_cluster=self.in_cluster,
cluster_context=self.cluster_context,
config_file=self.config_file)
gen = pod_generator.PodGenerator()
for mount in self.volume_mounts:
gen.add_mount(mount)
for volume in self.volumes:
gen.add_volume(volume)
pod = gen.make_pod(
namespace=self.namespace,
image=self.image,
pod_id=self.name,
cmds=self.cmds,
arguments=self.arguments,
labels=self.labels,
)
pod.service_account_name = self.service_account_name
pod.secrets = self.secrets
pod.envs = self.env_vars
pod.image_pull_policy = self.image_pull_policy
pod.annotations = self.annotations
pod.resources = self.resources
pod.affinity = self.affinity
pod.node_selectors = self.node_selectors
pod.hostnetwork = self.hostnetwork
pod.tolerations = self.tolerations
launcher = pod_launcher.PodLauncher(kube_client=client,
extract_xcom=self.xcom_push)
(final_state, result) = launcher.run_pod(
pod,
startup_timeout=self.startup_timeout_seconds,
get_logs=self.get_logs)
if self.is_delete_operator_pod:
launcher.delete_pod(pod)
if final_state != State.SUCCESS:
raise AirflowException(
'Pod returned a failure: {state}'.format(
state=final_state))
if self.xcom_push:
return result
except AirflowException as ex:
raise AirflowException(
'Pod Launching failed: {error}'.format(error=ex))