def pipline_that_has_airflow():
t1 = TTask()
t2 = BashOperator(task_id="sleep",
bash_command="sleep 0.1",
retries=3)
t1.set_upstream(t2)
return t1
def _test_run(self, databand_test_context):
task = TTask()
run = task.dbnd_run()
self.task = task
self.task_af_id = run.get_task_run_by_id(task.task_id).task_af_id
self.dag_id = task.task_name
self.task_execution_date_str = airflow_datetime_str(run.execution_date)
def _test_run(self, databand_test_context, logging_config_for_log_view):
task = TTask()
run = task.dbnd_run()
self.task = task
self.task_af_id = run.get_task_run_by_id(task.task_id).task_af_id
self.dag_id = run.dag_id
self.task_execution_date_str = airflow_datetime_str(run.execution_date)
def test_custom_partition_from_ctor(self):
task = TTask(
task_output_path_format=
"{root}/{env_label}/{task_family}{task_class_version}_custom/"
"{output_name}{output_ext}/date={task_target_date}")
assert_run_task(task)
assert "TTask_custom/t_output.csv/" in str(task.t_output)
def test_sign_by_task_code_build(self):
with new_dbnd_context(
conf={"task_build": {
"sign_with_full_qualified_name": "True"
}}):
task = TTask()
assert str(
TTask.__module__) in task.task_meta.task_signature_source
def test_dbnd_run(self, tmpdir):
t = targets.target(tmpdir.join("task_output"))
args = [
TTask.get_task_family(), "-r", "t_param=10", "-r",
"t_output=" + t.path
]
run_dbnd_subprocess_test(args)
assert t.exists()
def test_wrong_config_validation(self):
# raise exception
with pytest.raises(UnknownParameterError) as e:
with config({
"TTask": {
"t_parammm": 2,
"validate_no_extra_params": ParamValidation.error,
}
}):
TTask()
assert "Did you mean: t_param" in e.value.help_msg
# log warning to log
with config({
"TTask": {
"t_parammm": 2,
"validate_no_extra_params": ParamValidation.warn,
}
}):
TTask()
# tried to add a capsys assert here but couldn't get it to work
# do nothing
with config({
"TTask": {
"t_parammm": 2,
"validate_no_extra_params": ParamValidation.disabled,
}
}):
TTask()
# handle core config sections too
with pytest.raises(
DatabandError
): # might be other extra params in the config in which case a DatabandBuildError will be raised
with config({
"config": {
"validate_no_extra_params": ParamValidation.error
},
"core": {
"blabla": "bla"
},
}):
CoreConfig()
def test_exception(self):
s = TTask(t_param="my_param")
try:
raise Exception("MyException")
except Exception:
actual = TaskVisualiser(s).banner("Runinng task",
exc_info=sys.exc_info())
assert actual
assert "MyException" in actual
def test_custom_partition_from_config(self):
with dbnd_config(
config_values=
{
"task": {
"task_output_path_format":
"{root}/{env_label}/{task_family}{task_class_version}_custom/"
"{output_name}{output_ext}/date={task_target_date}"
}
}):
task = TTask()
assert_run_task(task)
assert "TTask_custom/t_output.csv/" in str(task.t_output)
def test_inconsistent_output(self, monkeypatch):
task = TTask()
validator = task.ctrl.validator
with monkeypatch.context() as m:
m.setattr(FileTarget, "exist_after_write_consistent",
lambda a: False)
m.setattr(FileTarget, "exists", lambda a: False)
m.setattr(
dbnd._core.task_ctrl.task_validator,
"EVENTUAL_CONSISTENCY_MAX_SLEEPS",
1,
)
assert not validator.wait_for_consistency()
def test_task_version_parse(self):
target = TTask(task_version="now")
assert target.task_version != "now"
def band(self):
self.t_output = TTask(t_param=SomeObject(1)).t_output
def test_dbnd_help(self):
stdout = run_dbnd_subprocess_test([TTask.get_task_family(), "--help"])
assert "-r", "t_param" in stdout
def test_task_call_source_class(self):
task = TTask()
logger.info(task.task_call_source)
assert task.task_call_source
assert task.task_call_source[0].filename in __file__
def test_object_parameters_immutable(self):
assert (TTask(t_param=SomeObject(1)).task_id !=
TTask(t_param=SomeObject(2)).task_id)
def test_dbnd_task_version(self):
args = [
TTask.get_task_family(), "-r", "t_param=100", "--task-version", "5"
]
run = dbnd_run_cmd(args)
assert run.root_task.task_version == "5"
def test_verbose_build(self):
with new_dbnd_context(conf={"task_build": {"verbose": "True"}}):
task = TTask(override={TTask.t_param: "test_driver"})
assert task.t_param == "test_driver"
def test_save_databand_run(self):
s = TTask()
r = self._save_graph(s)
actual = DatabandRun.load_run(r.driver_dump, False)
assert actual
def test_ttask(self):
task = TTask()
assert_run_task(task)
def test_thread_task_run(self):
t = TTask()
_run_in_thread(t.dbnd_run)
def test_simple_dump(self):
s = TTask(t_param="my_param")
actual = TaskVisualiser(s).banner("Runinng task")
assert "my_param" in actual
def test_save_time_1(self, benchmark):
s = TTask()
self._benchmark_pipeline_save(benchmark=benchmark, pipeline=s)
def ret_dict():
v = TTask(t_param=1)
return v
def band(self):
return TTask(t_param=1)
def test_input_task(self):
t = TTaskWithInput(t_input=TTask())
assert_run_task(t)
def test_simple_build(self):
with new_dbnd_context():
task = TTask(t_param="test_driver")
assert task.t_param == "test_driver"
assert task.t_output
def band(self):
t_inputs = {t: TTask(t_param=t).t_output for t in self.t_types}
self.t_output = TTaskCombineInputs(t_inputs=t_inputs).t_output
def band(self):
self.t_output = {
t: TTask(t_param=t).t_output
for t in self.t_types
}
def test_save_databand_run(self):
s = TTask()
r = self._save_graph(s)
actual = RunExecutor.load_run(r.run_executor.driver_dump, False)
assert actual
def test_cmdline_main_task_cls(self):
dbnd_run_cmd([TTask.get_task_family(), "-r", "t_param=100"])