def test_call_dotted_path_unexpected_kwargs(tmp_directory,
add_current_to_sys_path,
no_sys_modules_cache):
Path('my_module.py').write_text("""
def function():
pass
""")
with pytest.raises(TypeError) as excinfo:
dotted_path.call_dotted_path('my_module.function', kwargs=dict(a=1))
expected = ("function() got an unexpected keyword argument 'a' "
"(Loaded from:")
assert expected in str(excinfo.value)
def _to_dag(self):
"""
Internal method to manage the different cases to convert to a DAG
object
"""
if 'location' in self:
return dotted_path.call_dotted_path(self['location'])
dag = DAG()
if 'config' in self:
dag._params = DAGConfiguration.from_dict(self['config'])
clients = self.get('clients')
if clients:
for class_name, dotted_path_spec in clients.items():
dag.clients[class_name] = dotted_path.call_spec(
dotted_path_spec)
# FIXME: this violates lazy_import, we must change DAG's implementation
# to accept strings as attribute and load them until they are called
for attr in ['serializer', 'unserializer']:
if attr in self:
setattr(dag, attr, dotted_path.load_dotted_path(self[attr]))
process_tasks(dag, self, root_path=self._parent_path)
return dag
def test_call_dotted_path(tmp_directory, add_current_to_sys_path,
no_sys_modules_cache, kwargs, expected):
Path('my_module.py').write_text("""
def function(a=42):
return a
""")
assert dotted_path.call_dotted_path('my_module.function',
kwargs=kwargs) == expected
def _to_dag(self):
"""
Internal method to manage the different cases to convert to a DAG
object
"""
if 'location' in self:
return dotted_path.call_dotted_path(self['location'])
dag = DAG()
if 'config' in self:
dag._params = DAGConfiguration.from_dict(self['config'])
if 'executor' in self:
executor = self['executor']
if isinstance(executor,
str) and executor in {'serial', 'parallel'}:
if executor == 'parallel':
dag.executor = Parallel()
elif isinstance(executor, Mapping):
dag.executor = dotted_path.DottedPath(
executor, lazy_load=False, allow_return_none=False)()
else:
raise DAGSpecInitializationError(
'"executor" must be '
'"serial", "parallel", or a dotted path'
f', got: {executor!r}')
clients = self.get('clients')
if clients:
for class_name, dotted_path_spec in clients.items():
dps = dotted_path.DottedPath(dotted_path_spec,
lazy_load=self._lazy_import,
allow_return_none=False)
if self._lazy_import:
dag.clients[class_name] = dps
else:
dag.clients[class_name] = dps()
for attr in ('serializer', 'unserializer', 'on_finish', 'on_render',
'on_failure'):
if attr in self:
setattr(
dag, attr,
dotted_path.DottedPath(self[attr],
lazy_load=self._lazy_import))
process_tasks(dag, self, root_path=self._parent_path)
return dag
def _to_dag(self):
"""
Internal method to manage the different cases to convert to a DAG
object
"""
if 'location' in self:
return dotted_path.call_dotted_path(self['location'])
dag = DAG()
if 'config' in self:
dag._params = DAGConfiguration.from_dict(self['config'])
if 'executor' in self:
valid = {'serial', 'parallel'}
executor = self['executor']
if executor not in valid:
raise ValueError('executor must be one '
f'of {valid}, got: {executor}')
if executor == 'parallel':
dag.executor = Parallel()
clients = self.get('clients')
if clients:
for class_name, dotted_path_spec in clients.items():
dps = dotted_path.DottedPathSpec(dotted_path_spec)
if self._lazy_import:
dag.clients[class_name] = dps
else:
dag.clients[class_name] = dps()
for attr in ['serializer', 'unserializer']:
if attr in self:
setattr(
dag, attr,
dotted_path.DottedPath(self[attr],
lazy_load=self._lazy_import))
process_tasks(dag, self, root_path=self._parent_path)
return dag