def test_initialization(spec, expected, tmp_sample_tasks, tmp_imports):
meta = Meta.default_meta({
'extract_product': False,
'extract_upstream': True
})
spec = TaskSpec(spec, meta=meta, project_root='.')
# check values after initialization
assert spec['class'] == expected
assert isinstance(spec['source'], Path)
# check we can convert it to a Task
spec.to_task(dag=DAG())
def test_error_on_invalid_value_for_file_product(backup_online, tmp_imports):
meta = Meta.default_meta()
meta['extract_product'] = False
spec = TaskSpec({
'source': 'online_tasks.square',
'product': 1,
},
meta=meta,
project_root='.')
with pytest.raises(TypeError) as excinfo:
spec.to_task(dag=DAG())
expected = ('Error initializing File with argument 1 '
'(expected str, bytes or os.PathLike object, not int)')
assert expected == str(excinfo.value)
def test_loads_serializer_and_unserializer(backup_online, tmp_imports):
meta = Meta.default_meta()
meta['extract_product'] = False
spec = TaskSpec(
{
'source': 'online_tasks.square',
'product': 'output/square.parquet',
'serializer': 'online_io.serialize',
'unserializer': 'online_io.unserialize',
},
meta=meta,
project_root='.')
dag = DAG()
task, _ = spec.to_task(dag=dag)
from online_io import serialize, unserialize
assert task._serializer.callable is serialize
assert task._unserializer.callable is unserialize
def test_lazy_load(tmp_directory, tmp_imports):
Path('my_module.py').write_text("""
def fn():
pass
""")
meta = Meta.default_meta()
spec = TaskSpec(
{
'source': 'my_module.fn',
'product': 'report.ipynb',
'on_finish': 'not_a_module.not_a_function',
'on_render': 'not_a_module.not_a_function',
'on_failure': 'not_a_module.not_a_function',
'serializer': 'not_a_module.not_a_function',
'unserializer': 'not_a_module.not_a_function',
},
meta,
'.',
lazy_import=True)
assert spec.to_task(dag=DAG())