def pipline_that_has_airflow():
t1 = TTask()
t2 = BashOperator(task_id="sleep",
bash_command="sleep 0.1",
retries=3)
t1.set_upstream(t2)
return t1
def _test_run(self, databand_test_context):
task = TTask()
run = task.dbnd_run()
self.task = task
self.task_af_id = run.get_task_run_by_id(task.task_id).task_af_id
self.dag_id = task.task_name
self.task_execution_date_str = airflow_datetime_str(run.execution_date)
def test_dbnd_run(self, tmpdir):
t = targets.target(tmpdir.join("task_output"))
args = [
TTask.get_task_family(), "-r", "t_param=10", "-r",
"t_output=" + t.path
]
run_dbnd_subprocess_test(args)
assert t.exists()
def test_sign_by_task_code_build(self):
with new_dbnd_context(
conf={"task_build": {
"sign_with_full_qualified_name": "True"
}}):
task = TTask()
assert str(
TTask.__module__) in task.task_meta.task_signature_source
def test_wrong_config_validation(self):
# raise exception
with pytest.raises(UnknownParameterError) as e:
with config({
"TTask": {
"t_parammm": 2,
"validate_no_extra_params": ParamValidation.error,
}
}):
TTask()
assert "Did you mean: t_param" in e.value.help_msg
# log warning to log
with config({
"TTask": {
"t_parammm": 2,
"validate_no_extra_params": ParamValidation.warn,
}
}):
TTask()
# tried to add a capsys assert here but couldn't get it to work
# do nothing
with config({
"TTask": {
"t_parammm": 2,
"validate_no_extra_params": ParamValidation.disabled,
}
}):
TTask()
# handle core config sections too
with pytest.raises(
DatabandError
): # might be other extra params in the config in which case a DatabandBuildError will be raised
with config({
"config": {
"validate_no_extra_params": ParamValidation.error
},
"core": {
"blabla": "bla"
},
}):
CoreConfig()
def test_exception(self):
s = TTask(t_param="my_param")
try:
raise Exception("MyException")
except Exception:
actual = TaskVisualiser(s).banner("Runinng task",
exc_info=sys.exc_info())
assert actual
assert "MyException" in actual
def test_inconsistent_output(self, monkeypatch):
from dbnd._core.task_run.task_run_runner import TaskRunRunner
task = TTask()
with initialized_run(task):
validator = task.ctrl.validator
with monkeypatch.context() as m:
m.setattr(FileTarget, "exist_after_write_consistent",
lambda a: False)
m.setattr(FileTarget, "exists", lambda a: False)
m.setattr(
dbnd._core.task_ctrl.task_validator,
"EVENTUAL_CONSISTENCY_MAX_SLEEPS",
1,
)
assert not validator.wait_for_consistency()
def band(self):
self.t_output = TTask(t_param=SomeObject(1)).t_output
def test_cmdline_main_task_cls(self):
dbnd_run_cmd([TTask.get_task_family(), "-r", "t_param=100"])
def ret_dict():
v = TTask(t_param=1)
return v
def test_task_version_parse(self):
target = TTask(task_version="now")
assert target.task_version != "now"
def band(self):
t_inputs = {t: TTask(t_param=t).t_output for t in self.t_types}
self.t_output = TTaskCombineInputs(t_inputs=t_inputs).t_output
def test_input_task(self):
t = TTaskWithInput(t_input=TTask())
assert_run_task(t)
def band(self):
self.t_output = {
t: TTask(t_param=t).t_output
for t in self.t_types
}
def test_dbnd_help(self):
stdout = run_dbnd_subprocess_test([TTask.get_task_family(), "--help"])
assert "-r", "t_param" in stdout
def test_task_call_source_class(self):
task = TTask()
logger.info(task.task_meta.task_call_source)
assert task.task_meta.task_call_source
assert task.task_meta.task_call_source[0].filename in __file__
def test_verbose_build(self):
with new_dbnd_context(conf={"task_build": {"verbose": "True"}}):
task = TTask(override={TTask.t_param: "test_driver"})
assert task.t_param == "test_driver"
def test_object_parameters_immutable(self):
assert (TTask(t_param=SomeObject(1)).task_id !=
TTask(t_param=SomeObject(2)).task_id)
def test_save_time_1(self, benchmark):
s = TTask()
self._benchmark_pipeline_save(benchmark=benchmark, pipeline=s)
def band(self):
return TTask(t_param=1)
def test_save_databand_run(self):
s = TTask()
r = self._save_graph(s)
actual = DatabandRun.load_run(r.driver_dump, False)
assert actual
def test_simple_dump(self):
s = TTask(t_param="my_param")
actual = TaskVisualiser(s).banner("Runinng task")
assert "my_param" in actual
