def pipline_that_has_airflow(): t1 = TTask() t2 = BashOperator(task_id="sleep", bash_command="sleep 0.1", retries=3) t1.set_upstream(t2) return t1
def _test_run(self, databand_test_context): task = TTask() run = task.dbnd_run() self.task = task self.task_af_id = run.get_task_run_by_id(task.task_id).task_af_id self.dag_id = task.task_name self.task_execution_date_str = airflow_datetime_str(run.execution_date)
def test_dbnd_run(self, tmpdir): t = targets.target(tmpdir.join("task_output")) args = [ TTask.get_task_family(), "-r", "t_param=10", "-r", "t_output=" + t.path ] run_dbnd_subprocess_test(args) assert t.exists()
def test_sign_by_task_code_build(self): with new_dbnd_context( conf={"task_build": { "sign_with_full_qualified_name": "True" }}): task = TTask() assert str( TTask.__module__) in task.task_meta.task_signature_source
def test_wrong_config_validation(self): # raise exception with pytest.raises(UnknownParameterError) as e: with config({ "TTask": { "t_parammm": 2, "validate_no_extra_params": ParamValidation.error, } }): TTask() assert "Did you mean: t_param" in e.value.help_msg # log warning to log with config({ "TTask": { "t_parammm": 2, "validate_no_extra_params": ParamValidation.warn, } }): TTask() # tried to add a capsys assert here but couldn't get it to work # do nothing with config({ "TTask": { "t_parammm": 2, "validate_no_extra_params": ParamValidation.disabled, } }): TTask() # handle core config sections too with pytest.raises( DatabandError ): # might be other extra params in the config in which case a DatabandBuildError will be raised with config({ "config": { "validate_no_extra_params": ParamValidation.error }, "core": { "blabla": "bla" }, }): CoreConfig()
def test_exception(self): s = TTask(t_param="my_param") try: raise Exception("MyException") except Exception: actual = TaskVisualiser(s).banner("Runinng task", exc_info=sys.exc_info()) assert actual assert "MyException" in actual
def test_inconsistent_output(self, monkeypatch): from dbnd._core.task_run.task_run_runner import TaskRunRunner task = TTask() with initialized_run(task): validator = task.ctrl.validator with monkeypatch.context() as m: m.setattr(FileTarget, "exist_after_write_consistent", lambda a: False) m.setattr(FileTarget, "exists", lambda a: False) m.setattr( dbnd._core.task_ctrl.task_validator, "EVENTUAL_CONSISTENCY_MAX_SLEEPS", 1, ) assert not validator.wait_for_consistency()
def band(self): self.t_output = TTask(t_param=SomeObject(1)).t_output
def test_cmdline_main_task_cls(self): dbnd_run_cmd([TTask.get_task_family(), "-r", "t_param=100"])
def ret_dict(): v = TTask(t_param=1) return v
def test_task_version_parse(self): target = TTask(task_version="now") assert target.task_version != "now"
def band(self): t_inputs = {t: TTask(t_param=t).t_output for t in self.t_types} self.t_output = TTaskCombineInputs(t_inputs=t_inputs).t_output
def test_input_task(self): t = TTaskWithInput(t_input=TTask()) assert_run_task(t)
def band(self): self.t_output = { t: TTask(t_param=t).t_output for t in self.t_types }
def test_dbnd_help(self): stdout = run_dbnd_subprocess_test([TTask.get_task_family(), "--help"]) assert "-r", "t_param" in stdout
def test_task_call_source_class(self): task = TTask() logger.info(task.task_meta.task_call_source) assert task.task_meta.task_call_source assert task.task_meta.task_call_source[0].filename in __file__
def test_verbose_build(self): with new_dbnd_context(conf={"task_build": {"verbose": "True"}}): task = TTask(override={TTask.t_param: "test_driver"}) assert task.t_param == "test_driver"
def test_object_parameters_immutable(self): assert (TTask(t_param=SomeObject(1)).task_id != TTask(t_param=SomeObject(2)).task_id)
def test_save_time_1(self, benchmark): s = TTask() self._benchmark_pipeline_save(benchmark=benchmark, pipeline=s)
def band(self): return TTask(t_param=1)
def test_save_databand_run(self): s = TTask() r = self._save_graph(s) actual = DatabandRun.load_run(r.driver_dump, False) assert actual
def test_simple_dump(self): s = TTask(t_param="my_param") actual = TaskVisualiser(s).banner("Runinng task") assert "my_param" in actual