def __kubernetes_kwargs(self): kubernetes_kwargs = { 'namespace': get_variable('kubernetes_namespace', default_val='default'), 'name': self.task_name.replace('_', '-'), 'in_cluster': get_variable('in_kubernetes_cluster', default_val=False), 'image_pull_policy': get_variable('image_pull_policy', default_val='IfNotPresent'), 'get_logs': True, 'is_delete_operator_pod': True, 'startup_timeout_seconds': 300, 'image_pull_secrets': 'regcred', 'resources': self.resources, 'dag': self.dag, 'volumes': self.volumes, 'volume_mounts': [ VolumeMount(mount['volume'], mount['path'], mount.get('sub_path'), mount.get('read_only', False)) for mount in self.mounts ] } return kubernetes_kwargs
def __env_vars(self, env) -> Dict: env_vars = dict(self.task_config['env_vars']) if 'env_vars' in self.task_config else {} env_vars.update(self.__get_local_env_params_from_env_file()) airflow_configuration_variable = get_variable( f'''{self.pipeline_config['pipeline']}_dag_configuration''', default_val=None ) if airflow_configuration_variable: airflow_configs = json.loads(airflow_configuration_variable) environment_variables_key = f"{self.pipeline_config['pipeline']}_environment_variables" if environment_variables_key in airflow_configs: env_vars = airflow_configs[environment_variables_key] if ENV not in env_vars: env_vars[ENV] = env env_vars[OUTPUT_PATH] = self.task_config[ OUTPUT_PATH ] if OUTPUT_PATH in self.task_config else '/tmp/s3_mount' if 'output_destination_path' in self.task_config: env_vars[OUTPUT_DESTINATION_PATH] = self.task_config[ 'output_destination_path' ] return dict([(k, str(v)) for k, v in env_vars.items()])
def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config): super().__init__(task_id, dag, parent, trigger_rule, liminal_config, pipeline_config, task_config) env = standalone_variable_backend.get_variable(ENV, DEFAULT) self.env_vars = self.__env_vars(env) self.image = self.task_config['image'] self.mounts = self.task_config.get('mounts', []) self.cmds, self.arguments = self._kubernetes_cmds_and_arguments( self.env_vars.get(OUTPUT_PATH), self.env_vars.get(OUTPUT_DESTINATION_PATH) )
def __kubernetes_kwargs(self, task: ContainerTask): config = copy.deepcopy(self.executor_config) kubernetes_kwargs = { 'task_id': task.task_id, 'image': task.image, 'arguments': task.arguments, 'namespace': os.environ.get('AIRFLOW__KUBERNETES__NAMESPACE', 'default'), 'name': task.task_id.replace('_', '-'), 'in_cluster': os.environ.get('AIRFLOW__KUBERNETES__IN_CLUSTER', False), 'image_pull_policy': get_variable('image_pull_policy', default_val='IfNotPresent'), 'get_logs': config.pop('get_logs', True), 'is_delete_operator_pod': config.pop('is_delete_operator_pod', True), 'startup_timeout_seconds': config.pop('startup_timeout_seconds', 1200), 'env_vars': task.env_vars, 'do_xcom_push': task.task_config.get('do_xcom_push', False), 'image_pull_secrets': config.pop('image_pull_secrets', 'regcred'), 'volumes': self.volumes, 'config_file': os.environ.get('AIRFLOW__KUBERNETES__CONFIG_FILE'), 'cluster_context': os.environ.get('AIRFLOW__KUBERNETES__CLUSTER_CONTEXT', None), 'cmds': task.cmds, 'volume_mounts': [ VolumeMount(mount['volume'], mount['path'], mount.get('sub_path'), mount.get('read_only', False)) for mount in task.mounts ] } config.pop('in_cluster', None) config.pop('volumes', None) config.pop('volume_mounts', None) config.pop('executor', None) config.pop('type', None) kubernetes_kwargs.update(config) if env_util.is_running_on_jenkins(): kubernetes_kwargs['affinity'] = self.__jenkins_kubernetes_affinity() kubernetes_kwargs['namespace'] = 'jenkins' if not task.dag: kubernetes_kwargs.update({ 'start_date': datetime.datetime(1970, 1, 1), }) return kubernetes_kwargs
def __try_backend_variables(key, default): return standalone_variable_backend.get_variable(key, default)