def __kubernetes_kwargs(self):
kubernetes_kwargs = {
'namespace':
get_variable('kubernetes_namespace', default_val='default'),
'name':
self.task_name.replace('_', '-'),
'in_cluster':
get_variable('in_kubernetes_cluster', default_val=False),
'image_pull_policy':
get_variable('image_pull_policy', default_val='IfNotPresent'),
'get_logs':
True,
'is_delete_operator_pod':
True,
'startup_timeout_seconds':
300,
'image_pull_secrets':
'regcred',
'resources':
self.resources,
'dag':
self.dag,
'volumes':
self.volumes,
'volume_mounts': [
VolumeMount(mount['volume'], mount['path'],
mount.get('sub_path'),
mount.get('read_only', False))
for mount in self.mounts
]
}
return kubernetes_kwargs
def __env_vars(self, env) -> Dict:
env_vars = dict(self.task_config['env_vars']) if 'env_vars' in self.task_config else {}
env_vars.update(self.__get_local_env_params_from_env_file())
airflow_configuration_variable = get_variable(
f'''{self.pipeline_config['pipeline']}_dag_configuration''',
default_val=None
)
if airflow_configuration_variable:
airflow_configs = json.loads(airflow_configuration_variable)
environment_variables_key = f"{self.pipeline_config['pipeline']}_environment_variables"
if environment_variables_key in airflow_configs:
env_vars = airflow_configs[environment_variables_key]
if ENV not in env_vars:
env_vars[ENV] = env
env_vars[OUTPUT_PATH] = self.task_config[
OUTPUT_PATH
] if OUTPUT_PATH in self.task_config else '/tmp/s3_mount'
if 'output_destination_path' in self.task_config:
env_vars[OUTPUT_DESTINATION_PATH] = self.task_config[
'output_destination_path'
]
return dict([(k, str(v)) for k, v in env_vars.items()])
def __init__(self, task_id, dag, parent, trigger_rule, liminal_config, pipeline_config,
task_config):
super().__init__(task_id, dag, parent, trigger_rule, liminal_config,
pipeline_config, task_config)
env = standalone_variable_backend.get_variable(ENV, DEFAULT)
self.env_vars = self.__env_vars(env)
self.image = self.task_config['image']
self.mounts = self.task_config.get('mounts', [])
self.cmds, self.arguments = self._kubernetes_cmds_and_arguments(
self.env_vars.get(OUTPUT_PATH),
self.env_vars.get(OUTPUT_DESTINATION_PATH)
)
def __kubernetes_kwargs(self, task: ContainerTask):
config = copy.deepcopy(self.executor_config)
kubernetes_kwargs = {
'task_id': task.task_id,
'image': task.image,
'arguments': task.arguments,
'namespace': os.environ.get('AIRFLOW__KUBERNETES__NAMESPACE', 'default'),
'name': task.task_id.replace('_', '-'),
'in_cluster': os.environ.get('AIRFLOW__KUBERNETES__IN_CLUSTER', False),
'image_pull_policy': get_variable('image_pull_policy', default_val='IfNotPresent'),
'get_logs': config.pop('get_logs', True),
'is_delete_operator_pod': config.pop('is_delete_operator_pod', True),
'startup_timeout_seconds': config.pop('startup_timeout_seconds', 1200),
'env_vars': task.env_vars,
'do_xcom_push': task.task_config.get('do_xcom_push', False),
'image_pull_secrets': config.pop('image_pull_secrets', 'regcred'),
'volumes': self.volumes,
'config_file': os.environ.get('AIRFLOW__KUBERNETES__CONFIG_FILE'),
'cluster_context': os.environ.get('AIRFLOW__KUBERNETES__CLUSTER_CONTEXT', None),
'cmds': task.cmds,
'volume_mounts': [
VolumeMount(mount['volume'],
mount['path'],
mount.get('sub_path'),
mount.get('read_only', False))
for mount
in task.mounts
]
}
config.pop('in_cluster', None)
config.pop('volumes', None)
config.pop('volume_mounts', None)
config.pop('executor', None)
config.pop('type', None)
kubernetes_kwargs.update(config)
if env_util.is_running_on_jenkins():
kubernetes_kwargs['affinity'] = self.__jenkins_kubernetes_affinity()
kubernetes_kwargs['namespace'] = 'jenkins'
if not task.dag:
kubernetes_kwargs.update({
'start_date': datetime.datetime(1970, 1, 1),
})
return kubernetes_kwargs
def __try_backend_variables(key, default):
return standalone_variable_backend.get_variable(key, default)