def test_call_spec(monkeypatch, spec):
mock = Mock()
monkeypatch.setattr(functions, 'some_function', mock)
dotted_path.call_spec(spec)
mock.assert_called_once_with()
def test_call_spec_without_dotted_path_key():
spec = {'a': 1}
with pytest.raises(SpecValidationError) as excinfo:
dotted_path.call_spec(spec)
assert excinfo.value.errors == [{
'loc': ('dotted_path', ),
'msg': 'field required',
'type': 'value_error.missing'
}]
def test_call_spec_with_kwargs(monkeypatch):
mock = Mock()
monkeypatch.setattr(functions, 'some_function', mock)
spec = {
'dotted_path': 'test_pkg.functions.some_function',
'a': 1,
'b': 2,
}
dotted_path.call_spec(spec)
mock.assert_called_once_with(a=1, b=2)
def _to_dag(self):
"""
Internal method to manage the different cases to convert to a DAG
object
"""
if 'location' in self:
return dotted_path.call_dotted_path(self['location'])
dag = DAG()
if 'config' in self:
dag._params = DAGConfiguration.from_dict(self['config'])
clients = self.get('clients')
if clients:
for class_name, dotted_path_spec in clients.items():
dag.clients[class_name] = dotted_path.call_spec(
dotted_path_spec)
# FIXME: this violates lazy_import, we must change DAG's implementation
# to accept strings as attribute and load them until they are called
for attr in ['serializer', 'unserializer']:
if attr in self:
setattr(dag, attr, dotted_path.load_dotted_path(self[attr]))
process_tasks(dag, self, root_path=self._parent_path)
return dag
def init_product(task_dict, meta, task_class, root_path):
"""
Initialize product.
Resolution logic order:
task.product_class
meta.{task_class}.product_default_class
Current limitation: When there is more than one product, they all must
be from the same class.
"""
product_raw = task_dict.pop('product')
# if the product is not yet initialized (e.g. scripts extract products
# as dictionaries, lists or strings)
if isinstance(product_raw, products.Product):
return product_raw
key = 'product_default_class.' + task_class.__name__
meta_product_default_class = get_value_at(meta, key)
if 'product_class' in task_dict:
CLASS = validate_product_class_name(task_dict.pop('product_class'))
elif meta_product_default_class:
CLASS = validate_product_class_name(meta_product_default_class)
else:
raise ValueError('Could not determine a product class for task: '
'"{}". Add an explicit value in the '
'"product_class" key or provide a default value in '
'meta.product_default_class by setting the '
'key to the applicable task class'.format(task_dict))
if 'product_client' in task_dict:
kwargs = {
'client': dotted_path.call_spec(task_dict.pop('product_client'))
}
else:
kwargs = {}
relative_to = (Path(task_dict['source']).parent
if meta['product_relative_to_source'] else root_path)
if isinstance(product_raw, Mapping):
return {
key: try_product_init(CLASS,
resolve_if_file(value, relative_to, CLASS),
kwargs)
for key, value in product_raw.items()
}
else:
source = resolve_if_file(product_raw, relative_to, CLASS)
return try_product_init(CLASS, source, kwargs)
def _init_client(task_dict):
if 'client' in task_dict:
task_dict['client'] = dotted_path.call_spec(task_dict.pop('client'))