def __create_python_task(self,
dag,
task_id,
parent,
image,
cmd,
env_vars=None,
executors=None):
task_config = {
'task':
task_id,
'cmd':
cmd,
'image':
image,
'env_vars':
env_vars if env_vars is not None else {},
'mounts': [{
'mount': 'mymount',
'volume': self._VOLUME_NAME,
'path': '/mnt/vol1'
}]
}
if executors:
task_config['executors'] = executors
return python.PythonTask(dag=dag,
liminal_config=self.liminal_config,
pipeline_config={'pipeline': 'my_pipeline'},
task_config=task_config,
parent=parent,
trigger_rule='all_success')
def __create_python_task(self,
dag,
task_id,
parent,
image,
cmd,
env_vars=None,
executors=None):
self.liminal_config['volumes'] = [{
'volume': self._VOLUME_NAME,
'local': {
'path':
self.temp_dir.replace("/var/folders", "/private/var/folders")
}
}]
self.liminal_config['executors'] = [{
'executor': 'k8s',
'type': 'kubernetes',
}]
task_config = {
'task':
task_id,
'cmd':
cmd,
'image':
image,
'env_vars':
env_vars if env_vars is not None else {},
'mounts': [{
'mount': 'mymount',
'volume': self._VOLUME_NAME,
'path': '/mnt/vol1'
}]
}
if executors:
task_config['executors'] = executors
return python.PythonTask(task_id=task_id,
dag=dag,
liminal_config=self.liminal_config,
pipeline_config={'pipeline': 'my_pipeline'},
task_config=task_config,
parent=parent,
trigger_rule='all_success',
executor=KubernetesPodExecutor(
task_id='k8s',
liminal_config=self.liminal_config,
executor_config={
'executor': 'k8s',
'name': 'mypod'
}))
def test_apply_task_to_dag(self):
# TODO: elaborate tests
dag = dag_test_utils.create_dag()
task_id = 'my_task'
config = self.__create_conf(task_id)
task0 = python.PythonTask(dag, 'my_pipeline', None, config,
'all_success')
task0.apply_task_to_dag()
self.assertEqual(len(dag.tasks), 1)
dag_task0 = dag.tasks[0]
self.assertIsInstance(dag_task0,
KubernetesPodOperatorWithInputAndOutput)
self.assertEqual(dag_task0.task_id, task_id)