def execute(self, context): try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file) gen = pod_generator.PodGenerator() for port in self.ports: gen.add_port(port) for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.image_pull_secrets = self.image_pull_secrets pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations pod.configmaps = self.configmaps pod.security_context = self.security_context pod.pod_runtime_info_envs = self.pod_runtime_info_envs pod.dnspolicy = self.dnspolicy launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) try: (final_state, result) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) finally: if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) if self.xcom_push: return result except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context): try: client = kube_client.get_kube_client(in_cluster=self.in_cluster) gen = pod_generator.PodGenerator() pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels ) pod.secrets = self.secrets pod.envs = self.env_vars launcher = pod_launcher.PodLauncher(client) final_state = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) if final_state != State.SUCCESS: raise AirflowException('Pod returned a failure') except AirflowException as ex: raise AirflowException('Pod Launching failed: {error}'.format(error=ex))
def infrastructure(self): kube_client = get_kube_client() namespace = configuration.get("kubernetes", "namespace") headers = { "Authorization": kube_client.api_client.configuration.get_api_key_with_prefix( 'authorization') } url = "{0}/oapi/v1/namespaces/{1}/deploymentconfigs".format( kube_client.api_client.configuration.host, namespace) response = requests.get( url, headers=headers, params={"labelSelector": self.AIRFLOW_LABEL}, verify=kube_client.api_client.configuration.ssl_ca_cert) if response.status_code == 200: deployment_configs = response.json() else: return abort(response.status_code) try: pods = kube_client.list_namespaced_pod( namespace=namespace, label_selector=self.AIRFLOW_LABEL) pods = pods.to_dict() except ApiException: return abort(401) return jsonify( self.to_graph(namespace=namespace, oc_deployment_configs=deployment_configs, oc_pods=pods))
def pod(self, pod): task_instances = self.get_task_instances(pod=pod) if not task_instances: abort(404) kube_client = get_kube_client() namespace = configuration.get("kubernetes", "namespace") client = elasticsearch.Elasticsearch([ELASTICSEARCH_HOST]) count = Search(using=client) \ .query('match', **{"beat.hostname": pod}) \ .sort('offset') \ .count() task_instance = task_instances try: pod = kube_client.read_namespaced_pod(name=pod, namespace=namespace) pod = self.pod_info(oc_pod=pod.to_dict(), task_instance=task_instance, namespace=namespace, host=self.HOST) except ApiException: pod = self.pod_info( oc_pod=self.get_pod_from_log_info(task_instance=task_instance), task_instance=task_instance, namespace=namespace, host=self.HOST) pod['log'] = {"count": count} return jsonify(pod)
def execute(self, context): try: client = kube_client.get_kube_client(in_cluster=self.in_cluster) gen = pod_generator.PodGenerator() pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels ) pod.secrets = self.secrets launcher = pod_launcher.PodLauncher(client) final_state = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) if final_state != State.SUCCESS: raise AirflowException('Pod returned a failure') except AirflowException as ex: raise AirflowException('Pod Launching failed: {error}'.format(error=ex))
def on_kill(self): self.log.debug("Kill Command is being called") if self._should_track_driver_status: if self._driver_id: self.log.info("Killing driver {} on cluster".format( self._driver_id)) kill_cmd = self._build_spark_driver_kill_command() driver_kill = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.log.info( "Spark driver {} killed with return code: {}".format( self._driver_id, driver_kill.wait())) if self._submit_sp and self._submit_sp.poll() is None: self.log.info("Sending kill signal to %s", self._connection["spark_binary"]) self._submit_sp.kill() if self._yarn_application_id: self.log.info("Killing application {} on YARN".format( self._yarn_application_id)) kill_cmd = "yarn application -kill {}".format( self._yarn_application_id).split() yarn_kill = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.log.info("YARN killed with return code: %s", yarn_kill.wait()) if self._kubernetes_driver_pod: self.log.info("Killing pod %s on Kubernetes", self._kubernetes_driver_pod) # Currently only instantiate Kubernetes client for killing a spark pod. try: import kubernetes client = kube_client.get_kube_client() api_response = client.delete_namespaced_pod( self._kubernetes_driver_pod, self._connection["namespace"], body=kubernetes.client.V1DeleteOptions(), pretty=True, ) self.log.info("Spark on K8s killed with response: %s", api_response) except kube_client.ApiException as e: self.log.info( "Exception when attempting to kill Spark on K8s:") self.log.exception(e)
def execute(self, context): try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) if self.in_cluster: worker_pod_name = os.environ.get('WORKER_POD_NAME') worker_pod_uid = os.environ.get('WORKER_POD_UID') gen.add_ownerreference(worker_pod_name, "v1", "Pod", worker_pod_uid) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.image_pull_secrets = self.image_pull_secrets pod.hostnetwork = self.hostnetwork launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) (final_state, result) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) # if self.is_delete_operator_pod: # launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) if self.xcom_push: return result except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def cluster_view(self): kube_client = get_kube_client() namespace = configuration.get("kubernetes", "namespace") pods = kube_client.list_namespaced_pod( namespace=namespace, label_selector=self.AIRFLOW_LABEL) return self.render("openshift_cluster_view.html", namespace=namespace, pods=pods)
def log_pod_creation(self, pod: Pod, resp, session=None): from openshift_plugin.executor.airflow_openshift_scheduler import AirflowOpenShiftScheduler execution_date = AirflowOpenShiftScheduler.label_safe_datestring_to_datetime(pod.labels['execution_date']) task_instance = session.query(TaskInstance) \ .filter(TaskInstance.dag_id == pod.labels['dag_id']) \ .filter(TaskInstance.task_id == pod.labels['task_id']) \ .filter(TaskInstance.execution_date == execution_date).first() if not task_instance: self.log.error( "Could not find task instance based on the pod labels" " ({dag_id} {task_id} {execution_date} {try_number})".format( **pod.labels)) self.log.error("Log information will be incomplete. This is a BUG please report!!!") def default(o): if isinstance(o, (datetime.date, datetime.datetime)): return o.isoformat() kube_client = get_kube_client() headers = {"Authorization": kube_client.api_client.configuration.get_api_key_with_prefix('authorization')} url = "{0}/apis/image.openshift.io/v1/namespaces/{1}/imagestreamtags/{2}".format( kube_client.api_client.configuration.host, pod.image.split("/")[-2], quote_plus(pod.image.split("/")[-1])) response = requests.get(url, headers=headers, verify=kube_client.api_client.configuration.ssl_ca_cert) resp = resp.to_dict() if response.status_code == 200: image_reference = response.json() resp['spec']['containers'][0]['image'] = image_reference["tag"]["from"]["name"] else: image_reference = None log = Log( event=OpenShiftPodLauncer.EVENT_POD_CREATION, dag_id=task_instance.dag_id, task_instance=None, task_id=task_instance.task_id, execution_date=task_instance.execution_date, extra=json.dumps( { "request": self.kube_req_factory.create(pod), "response": resp, "image": image_reference }, default=default) ) session.add(log) session.commit() pass
def execute(self, context): try: client = kube_client.get_kube_client(in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.image_pull_secrets = self.image_pull_secrets pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations pod.configmaps = self.configmaps pod.security_context = self.security_context launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.do_xcom_push) try: (final_state, result) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) finally: if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format(state=final_state) ) return result except AirflowException as ex: raise AirflowException('Pod Launching failed: {error}'.format(error=ex))
def run(self): kube_client = get_kube_client() while True: try: self.resource_version = self._run(kube_client, self.resource_version, self.worker_uuid) except Exception: self.log.exception('Unknown error in KubernetesJobWatcher. Failing') raise else: self.log.warn('Watch died gracefully, starting back up with: ' 'last resource_version: %s', self.resource_version)
def run(self): kube_client = get_kube_client() while True: try: self.resource_version = self._run(kube_client, self.resource_version, self.worker_uuid) except Exception: self.log.exception('Unknown error in KubernetesJobWatcher. Failing') raise else: self.log.warn('Watch died gracefully, starting back up with: ' 'last resource_version: %s', self.resource_version)
def get_image_url(self, host, orig_namespace, image_reference: str): if "kibana" in image_reference: return "https://www.docker.elastic.co" if "postgresql" in image_reference: return None kube_client = get_kube_client() image_reference_split = image_reference.split("/") image_hash = image_reference_split[-1] if len(image_reference_split) >= 2: namespace = image_reference_split[-2] else: namespace = orig_namespace if "@" in image_hash: image_stream, image_sha256 = image_hash.split("@") headers = { "Authorization": kube_client.api_client.configuration.get_api_key_with_prefix( 'authorization') } url = "{0}/oapi/v1/namespaces/{1}/imagestreams/{2}".format( kube_client.api_client.configuration.host, namespace, image_stream) response = requests.get( url, headers=headers, params={"labelSelector": self.AIRFLOW_LABEL}, verify=kube_client.api_client.configuration.ssl_ca_cert) if response.status_code == 200: image = response.json() else: return None image_tag_name = None for image_tag in image['spec']['tags']: if image_tag['from']['name'] == image_hash: image_tag_name = image_tag['name'] break else: image_stream, image_tag_name = image_hash.split(":") if image_tag_name: return "{0}/console/project/{1}/browse/images/{2}/{3}?tab=body".format( host, namespace, image_stream, image_tag_name) else: return None
def on_kill(self): self.log.debug("Kill Command is being called") if self._should_track_driver_status: if self._driver_id: self.log.info('Killing driver {} on cluster' .format(self._driver_id)) kill_cmd = self._build_spark_driver_kill_command() driver_kill = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.log.info("Spark driver {} killed with return code: {}" .format(self._driver_id, driver_kill.wait())) if self._submit_sp and self._submit_sp.poll() is None: self.log.info('Sending kill signal to %s', self._connection['spark_binary']) self._submit_sp.kill() if self._yarn_application_id: self.log.info('Killing application {} on YARN' .format(self._yarn_application_id)) kill_cmd = "yarn application -kill {}" \ .format(self._yarn_application_id).split() yarn_kill = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.log.info("YARN killed with return code: %s", yarn_kill.wait()) if self._kubernetes_driver_pod: self.log.info('Killing pod %s on Kubernetes', self._kubernetes_driver_pod) # Currently only instantiate Kubernetes client for killing a spark pod. try: import kubernetes client = kube_client.get_kube_client() api_response = client.delete_namespaced_pod( self._kubernetes_driver_pod, self._connection['namespace'], body=kubernetes.client.V1DeleteOptions(), pretty=True) self.log.info("Spark on K8s killed with response: %s", api_response) except kube_client.ApiException as e: self.log.info("Exception when attempting to kill Spark on K8s:") self.log.exception(e)
def get_image_dag_info(self): client = self.kube_client or get_kube_client() launcher = PodLauncher(kube_client=client) pod = self.create_sync_pod() status, result = launcher.run_pod(pod, get_logs=False) logs = client.read_namespaced_pod_log( name=pod.name, namespace=pod.namespace, container='base', follow=True, _preload_content=False) launcher.delete_pod(pod) return status, logs.data, pod
def extract_env_and_secrets(pod, req): KubernetesRequestFactory.extract_env_and_secrets(pod, req) env = req['spec']['containers'][0]['env'].copy() env = [i for i in env if not i["name"].startswith("AIRFLOW")] kube_client = get_kube_client() if configuration.conf.getboolean("kubernetes", "in_cluster"): pod_config = kube_client.read_namespaced_pod( name=os.getenv("HOSTNAME"), namespace=configuration.conf.get("kubernetes", "namespace")) else: pods = kube_client.list_namespaced_pod( namespace=configuration.conf.get("kubernetes", "namespace"), label_selector="component=airflow-scheduler") pod_config = pods.items[0] self_env = [ convert_dict_key_case(e.to_dict()) for e in pod_config.spec.containers[0].env if e.name not in ["AIRFLOW_COMMAND", "AIRFLOW_EXECUTOR"] ] self_env.append({"name": "AIRFLOW_EXECUTOR", "value": "LocalExecutor"}) if configuration.conf.has_option("core", "worker_logging_level"): self_env.append({ "name": "AIRFLOW__CORE__LOGGING_LEVEL", "value": configuration.conf.get("core", "worker_logging_level") }) self_env.append({ "name": "AIRFLOW__CORE__FAB_LOGGING_LEVEL", "value": configuration.conf.get("core", "worker_logging_level") }) req['spec']['containers'][0]['env'] = self_env + env self_env_from = [ convert_dict_key_case(e.to_dict()) for e in pod_config.spec.containers[0].env_from ] req['spec']['containers'][0]['envFrom'] = self_env_from
def authorize(oauth_app, authorized_response, user_info): with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: namespace = file.read() kube_client = get_kube_client() url = "{0}/apis/rbac.authorization.k8s.io/v1beta1/namespaces/{1}/rolebindings".format( kube_client.api_client.configuration.host, namespace) response = requests.get( url, headers={ "Authorization": "Bearer {0}".format(oauth_app.consumer_secret) }, verify=kube_client.api_client.configuration.ssl_ca_cert if kube_client.api_client.configuration.ssl_ca_cert else False) if response.status_code != 200: LoggingMixin().log.error( "The service account providing OAuth is not allowed to list rolebindings. Deniyng " "access to everyone!!!") return False, False role_binding_list = response.json() allowed_roles = [] for role in role_binding_list['items']: def predicate(subject): if subject['kind'] in ['ServiceAccount', 'User']: return subject['name'] == user_info['metadata']['name'] elif subject['kind'] is 'Group': return subject['name'] in user_info['groups'] name = role['roleRef']['name'] if next((x for x in role['subjects'] if predicate(x)), None): allowed_roles.append(name) allowed_roles = set(allowed_roles) access_roles = set( configuration.conf.get('openshift_plugin', 'access_roles').split(',')) superuser_roles = set( configuration.conf.get('openshift_plugin', 'superuser_roles').split(',')) return bool(allowed_roles & access_roles), \ bool(allowed_roles & superuser_roles)
def start(self): self.log.info('Start Kubernetes executor') self.worker_uuid = KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid( ) self.log.debug('Start with worker_uuid: %s', self.worker_uuid) # always need to reset resource version since we don't know # when we last started, note for behavior below # https://github.com/kubernetes-client/python/blob/master/kubernetes/docs # /CoreV1Api.md#list_namespaced_pod KubeResourceVersion.reset_resource_version() self.task_queue = Queue() self.result_queue = Queue() self.kube_client = get_kube_client() self.kube_scheduler = AirflowKubernetesScheduler( self.kube_config, self.task_queue, self.result_queue, self.kube_client, self.worker_uuid) self._inject_secrets() self.clear_not_launched_queued_tasks()
def start(self): self.log.info('Start Kubernetes executor') self.worker_uuid = KubeWorkerIdentifier.get_or_create_current_kube_worker_uuid() self.log.debug('Start with worker_uuid: %s', self.worker_uuid) # always need to reset resource version since we don't know # when we last started, note for behavior below # https://github.com/kubernetes-client/python/blob/master/kubernetes/docs # /CoreV1Api.md#list_namespaced_pod KubeResourceVersion.reset_resource_version() self.task_queue = Queue() self.result_queue = Queue() self.kube_client = get_kube_client() self.kube_scheduler = AirflowKubernetesScheduler( self.kube_config, self.task_queue, self.result_queue, self.kube_client, self.worker_uuid ) self._inject_secrets() self.clear_not_launched_queued_tasks()
def execute(self, context): try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) (final_state, result) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) if self.xcom_push: return result except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def post_sync(self): dag_tar, pod = self.get_dag_tag() kube_client = get_kube_client() headers = { "Authorization": kube_client.api_client.configuration.get_api_key_with_prefix( 'authorization') } url = "{0}/apis/image.openshift.io/v1/namespaces/{1}/imagestreamtags/{2}".format( kube_client.api_client.configuration.host, pod.image.split("/")[-2], quote_plus(pod.image.split("/")[-1])) response = requests.get( url, headers=headers, verify=kube_client.api_client.configuration.ssl_ca_cert) if response.status_code == 200: image = response.json() else: image = None echo = subprocess.Popen(("echo", dag_tar.decode("utf-8")), stdout=subprocess.PIPE) base64 = subprocess.Popen(("base64", "-d"), stdin=echo.stdout, stdout=subprocess.PIPE) output = subprocess.check_output(("tar", "-tzv"), stdin=base64.stdout) echo.wait() return self.render( "openshift_worker_image_sync_view.html", files=self.parse_tar_list(output), image=image, image_url= "{0}/console/project/dsi-test/browse/images/{1}/{2}?tab=body". format(OpenshiftClusterView.HOST, image['metadata']['name'].split(":")[0], image['tag']['name']))
def execute(self, context): try: client = kube_client.get_kube_client(in_cluster=self.in_cluster, cluster_context=self.cluster_context) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity launcher = pod_launcher.PodLauncher(kube_client=client) final_state = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format(state=final_state) ) except AirflowException as ex: raise AirflowException('Pod Launching failed: {error}'.format(error=ex))
def run(self): """Performs watching""" kube_client = get_kube_client() while True: try: self.resource_version = self._run(kube_client, self.resource_version, self.worker_uuid, self.kube_config) except ReadTimeoutError: self.log.warning( "There was a timeout error accessing the Kube API. " "Retrying request.", exc_info=True) time.sleep(1) except Exception: self.log.exception( 'Unknown error in KubernetesJobWatcher. Failing') raise else: self.log.warning( 'Watch died gracefully, starting back up with: ' 'last resource_version: %s', self.resource_version)
def on_kill(self): self.log.info("Kill Command is being called") if self._is_ssh: if self._dataeng_spark: SSHOperator( task_id='_kill_task', command=f'kill -TERM $(cat {self.pidfile})', ssh_conn_id=self._ssh_conn_id).execute(context=None) self.log.info("on_kill is finished") elif self._is_yarn: self.log.info('Killing application {} on YARN'.format( self._yarn_application_id)) kill_cmd = "yarn application -kill {}" \ .format(self._yarn_application_id) self.log.info('Killing via ssh command: {}'.format(kill_cmd)) SSHOperator(task_id='_kill_spark', ssh_conn_id=self._ssh_conn_id, command=kill_cmd).execute(None) self.log.info("YARN killed") if self._should_track_driver_status: if self._driver_id: self.log.info('Killing driver {} on cluster'.format( self._driver_id)) kill_cmd = self._build_spark_driver_kill_command() if self._is_ssh: ssh_kill_command = " ".join(kill_cmd) self.log.info( 'Killing via ssh command: {}'.format(ssh_kill_command)) SSHOperator(task_id='_kill_spark', ssh_conn_id=self._ssh_conn_id, command=ssh_kill_command).execute(None) self.log.info("Spark driver {} killed".format( self._driver_id)) else: driver_kill = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.log.info( "Spark driver {} killed with return code: {}".format( self._driver_id, driver_kill.wait())) if self._submit_sp and self._submit_sp.poll() is None: self.log.info('Sending kill signal to %s', self._connection['spark_binary']) self._submit_sp.kill() if self._yarn_application_id: self.log.info('Killing application {} on YARN'.format( self._yarn_application_id)) kill_cmd = "yarn application -kill {}" \ .format(self._yarn_application_id).split() self.log.info('Killing via ssh command: {}'.format(kill_cmd)) yarn_kill = subprocess.Popen(kill_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.log.info("YARN killed with return code: %s", yarn_kill.wait()) if self._kubernetes_driver_pod: self.log.info('Killing pod %s on Kubernetes', self._kubernetes_driver_pod) # Currently only instantiate Kubernetes client for killing a spark pod. try: client = kube_client.get_kube_client() api_response = client.delete_namespaced_pod( self._kubernetes_driver_pod, self._connection['namespace'], body=client.V1DeleteOptions(), pretty=True) self.log.info("Spark on K8s killed with response: %s", api_response) except kube_client.ApiException as e: self.log.info( "Exception when attempting to kill Spark on K8s:") self.log.exception(e)
def execute(self, context): ( pod_template, deployment, ) = get_pod_template_from_deployment_labels_and_namespace( namespace=self.deployment_namespace or self.namespace, config_file=self.config_file, cluster_context=self.cluster_context, in_cluster=self.in_cluster, fields=self.deployment_fields, labels=self.deployment_labels, ) pod_spec: V1PodSpec = pod_template.spec container: V1Container = pod_spec.containers[0] metadata: V1ObjectMeta = pod_template.metadata ( plain_env_vars, container_secrets, container_config_maps, runtime_info_envs, ) = handle_container_environment_variables(container.env) self.image = self.image or container.image self.cmds = self.cmds or container.command self.arguments = self.arguments or container.args or [] self.labels = self.labels or metadata.labels or {} self.name = self._set_name(self.name or deployment.metadata.name) self.env_vars = self.env_vars or plain_env_vars self.ports = self.ports or convert_ports(container) self.volume_mounts = self.volume_mounts or convert_volume_mounts( container) self.volumes = self.volumes or convert_volumes(pod_spec) self.secrets = self.secrets or container_secrets self.image_pull_policy = (self.image_pull_policy or container.image_pull_policy or "IfNotPresent") self.node_selectors = self.node_selectors or pod_spec.node_selector or {} self.annotations = self.annotations or metadata.annotations or {} self.affinity = self.affinity or convert_affinity(pod_spec) self.resources = (self.resources if (self.resources.has_limits() or self.resources.has_requests()) else convert_resources(container)) self.image_pull_secrets = self.image_pull_secrets or convert_image_pull_secrets( pod_spec) self.service_account_name = (self.service_account_name or pod_spec.service_account_name or pod_spec.service_account or "default") self.hostnetwork = (pod_spec.host_network or False if self.hostnetwork is None else self.hostnetwork) self.tolerations = self.tolerations or convert_tolerations(pod_spec) self.configmaps = self.configmaps or container_config_maps self.security_context = self.security_context or convert_security_context( pod_spec) self.pod_runtime_info_envs = self.pod_runtime_info_envs or runtime_info_envs self.dnspolicy = self.dnspolicy or pod_spec.dns_policy self.log.info("volumes %s", self.volumes) try: if self.in_cluster is not None: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file) else: client = kube_client.get_kube_client( cluster_context=self.cluster_context, config_file=self.config_file) # Add Airflow Version to the label # And a label to identify that pod is launched by KubernetesPodOperator self.labels.update({ 'airflow_version': airflow_version.replace('+', '-'), 'kubernetes_pod_operator': 'True', }) gen = pod_generator.PodGenerator() for port in self.ports: gen.add_port(port) for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.image_pull_secrets = self.image_pull_secrets pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations pod.configmaps = self.configmaps pod.security_context = self.security_context pod.pod_runtime_info_envs = self.pod_runtime_info_envs pod.dnspolicy = self.dnspolicy launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.do_xcom_push) # monkey patch to avoid https://github.com/apache/airflow/issues/8275 launcher.kube_req_factory.extract_env_and_secrets = extract_env_and_secrets try: (final_state, result) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) finally: if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) if self.do_xcom_push: return result except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context): try: from dagster_graphql.client.mutations import ( handle_start_pipeline_execution_errors, handle_start_pipeline_execution_result, ) except ImportError: raise AirflowException( 'To use the DagsterKubernetesPodOperator, dagster and dagster_graphql must be' ' installed in your Airflow environment.') if 'run_id' in self.params: self._run_id = self.params['run_id'] elif 'dag_run' in context and context['dag_run'] is not None: self._run_id = context['dag_run'].run_id # return to original execute code: try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file, ) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.query, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.image_pull_secrets = self.image_pull_secrets pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations pod.configmaps = self.configmaps pod.security_context = self.security_context launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) try: # we won't use the "result", which is the pod's xcom json file (final_state, _) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) # fetch the last line independently of whether logs were read # unbelievably, if you set tail_lines=1, the returned json has its double quotes # turned into unparseable single quotes # TODO: add retries - k8s log servers are _extremely_ flaky raw_res = client.read_namespaced_pod_log( name=pod.name, namespace=pod.namespace, container='base', tail_lines=5) # find the relevant line # TODO: raise sensible exception on garbage API string responses res = parse_raw_res(raw_res) handle_start_pipeline_execution_errors(res) events = handle_start_pipeline_execution_result(res) check_events_for_skips(events) return events finally: self._run_id = None if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) # note the lack of returning the default xcom except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context): try: conf = context['dag_run'].conf self.image = conf.get('docker_image_name') if conf.get("commands"): self.cmds = conf.get("commands") if conf.get("arguments"): self.arguments = conf.get("arguments") if conf.get("env_vars"): self.env_vars = conf.get("env_vars") except Exception as e: raise XKubernetesPodOperatorException( "Could not start off with Dag Run Configuration", e) try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.arguments, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) (final_state, result) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) if self.xcom_push: return result except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context): try: from dagster_graphql.implementation.pipeline_execution_manager import ( build_synthetic_pipeline_error_record, ) from dagster_graphql.client.mutations import ( DagsterGraphQLClientError, handle_execution_errors, handle_execute_plan_result_raw, ) except ImportError: raise AirflowException( 'To use the DagsterKubernetesPodOperator, dagster and dagster_graphql must be' ' installed in your Airflow environment.') if 'run_id' in self.params: self._run_id = self.params['run_id'] elif 'dag_run' in context and context['dag_run'] is not None: self._run_id = context['dag_run'].run_id # return to original execute code: try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file, ) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.query, labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.image_pull_secrets = self.image_pull_secrets pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations pod.configmaps = self.configmaps pod.security_context = self.security_context launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) try: if self.instance: self.instance.get_or_create_run( PipelineRun( pipeline_name=self.pipeline_name, run_id=self.run_id, environment_dict=self.environment_dict, mode=self.mode, selector=ExecutionSelector(self.pipeline_name), reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.MANAGED, )) # we won't use the "result", which is the pod's xcom json file (final_state, _) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs) # fetch the last line independently of whether logs were read # unbelievably, if you set tail_lines=1, the returned json has its double quotes # turned into unparseable single quotes # TODO: add retries - k8s log servers are _extremely_ flaky raw_res = client.read_namespaced_pod_log( name=pod.name, namespace=pod.namespace, container='base', tail_lines=5) res = parse_raw_res(raw_res.split('\n')) try: handle_execution_errors(res, 'executePlan') except DagsterGraphQLClientError: event = build_synthetic_pipeline_error_record( self.run_id, serializable_error_info_from_exc_info(sys.exc_info()), self.pipeline_name, ) if self.instance: self.instance.handle_new_event(event) raise events = handle_execute_plan_result_raw(res) if self.instance: for event in events: self.instance.handle_new_event(event) check_raw_events_for_skips(events) return events finally: self._run_id = None if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException( 'Pod returned a failure: {state}'.format( state=final_state)) # note the lack of returning the default xcom except AirflowException as ex: raise AirflowException( 'Pod Launching failed: {error}'.format(error=ex))
def execute(self, context): try: from dagster_graphql.client.mutations import ( DagsterGraphQLClientError, handle_execution_errors, handle_execute_plan_result_raw, ) except ImportError: raise AirflowException( 'To use the DagsterKubernetesPodOperator, dagster and dagster_graphql must be' ' installed in your Airflow environment.' ) if 'run_id' in self.params: self._run_id = self.params['run_id'] elif 'dag_run' in context and context['dag_run'] is not None: self._run_id = context['dag_run'].run_id # return to original execute code: try: client = kube_client.get_kube_client( in_cluster=self.in_cluster, cluster_context=self.cluster_context, config_file=self.config_file, ) gen = pod_generator.PodGenerator() for mount in self.volume_mounts: gen.add_mount(mount) for volume in self.volumes: gen.add_volume(volume) pod = gen.make_pod( namespace=self.namespace, image=self.image, pod_id=self.name, cmds=self.cmds, arguments=self.query(context.get('ts')), labels=self.labels, ) pod.service_account_name = self.service_account_name pod.secrets = self.secrets pod.envs = self.env_vars pod.image_pull_policy = self.image_pull_policy pod.image_pull_secrets = self.image_pull_secrets pod.annotations = self.annotations pod.resources = self.resources pod.affinity = self.affinity pod.node_selectors = self.node_selectors pod.hostnetwork = self.hostnetwork pod.tolerations = self.tolerations pod.configmaps = self.configmaps pod.security_context = self.security_context launcher = pod_launcher.PodLauncher(kube_client=client, extract_xcom=self.xcom_push) try: if self.instance: tags = ( {AIRFLOW_EXECUTION_DATE_STR: context.get('ts')} if 'ts' in context else {} ) run = self.instance.register_managed_run( pipeline_name=self.pipeline_name, run_id=self.run_id, run_config=self.run_config, mode=self.mode, solids_to_execute=None, step_keys_to_execute=None, tags=tags, root_run_id=None, parent_run_id=None, pipeline_snapshot=self.pipeline_snapshot, execution_plan_snapshot=self.execution_plan_snapshot, parent_pipeline_snapshot=self.parent_pipeline_snapshot, ) # we won't use the "result", which is the pod's xcom json file (final_state, _) = launcher.run_pod( pod, startup_timeout=self.startup_timeout_seconds, get_logs=self.get_logs ) # fetch the last line independently of whether logs were read # unbelievably, if you set tail_lines=1, the returned json has its double quotes # turned into unparseable single quotes res = None num_attempts = 0 while not res and num_attempts < LOG_RETRIEVAL_MAX_ATTEMPTS: raw_res = client.read_namespaced_pod_log( name=pod.name, namespace=pod.namespace, container='base' ) res = parse_raw_log_lines(raw_res.split('\n')) time.sleep(LOG_RETRIEVAL_WAITS_BETWEEN_ATTEMPTS_SEC) num_attempts += 1 try: handle_execution_errors(res, 'executePlan') except DagsterGraphQLClientError as err: self.instance.report_engine_event( str(err), run, EngineEventData.engine_error( serializable_error_info_from_exc_info(sys.exc_info()) ), self.__class__, ) raise events = handle_execute_plan_result_raw(res) if self.instance: for event in events: self.instance.handle_new_event(event) events = [e.dagster_event for e in events] check_events_for_failures(events) check_events_for_skips(events) return events finally: self._run_id = None if self.is_delete_operator_pod: launcher.delete_pod(pod) if final_state != State.SUCCESS: raise AirflowException('Pod returned a failure: {state}'.format(state=final_state)) # note the lack of returning the default xcom except AirflowException as ex: raise AirflowException('Pod Launching failed: {error}'.format(error=ex))