def test_manual_instance(): defn = PipelineDefinition([the_solid], "test") with pytest.raises( DagsterInvariantViolationError, match="Reconstructable target should be a function or definition produced by a decorated function", ): reconstructable(defn)
def test_inner_scope_2(): def get_the_pipeline_inner(): return the_pipeline with pytest.raises( DagsterInvariantViolationError, match="not found at module scope in file", ): reconstructable(get_the_pipeline_inner)
def test_bad_target(): with pytest.raises( DagsterInvariantViolationError, match=re.escape( "Loadable attributes must be either a JobDefinition, GraphDefinition, PipelineDefinition, or a " "RepositoryDefinition. Got None."), ): reconstructable(not_the_pipeline)
def test_not_defined_in_module(mocker): mocker.patch("inspect.getmodule", return_value=types.ModuleType("__main__")) with pytest.raises( DagsterInvariantViolationError, match=re.escape( "reconstructable() can not reconstruct pipelines defined in interactive environments" ), ): reconstructable(get_the_pipeline)
def test_inner_scope(): def get_the_pipeline_inner(): return the_pipeline with pytest.raises( DagsterInvariantViolationError, match="Use a function or decorated function defined at module scope", ): reconstructable(get_the_pipeline_inner)
def test_inner_decorator_2(): @pipeline def pipe(): the_solid() with pytest.raises( DagsterInvariantViolationError, match="not found at module scope in file", ): reconstructable(pipe)
def test_inner_decorator(): @pipeline def pipe(): the_solid() with pytest.raises( DagsterInvariantViolationError, match="Use a function or decorated function defined at module scope", ): reconstructable(pipe)
def test_reconstructable_module(): original_sys_path = sys.path try: sys.path.insert(0, file_relative_path(__file__, ".")) from foo import bar_pipeline # pylint: disable=import-error reconstructable(bar_pipeline) finally: sys.path = original_sys_path
def test_step_handler(kubeconfig_file): mock_k8s_client_batch_api = mock.MagicMock() handler = K8sStepHandler( job_config=DagsterK8sJobConfig(instance_config_map="foobar", job_image="bizbuz"), job_namespace="foo", load_incluster_config=False, kubeconfig_file=kubeconfig_file, k8s_client_batch_api=mock_k8s_client_batch_api, ) with instance_for_test() as instance: run = create_run_for_test( instance, pipeline_name="bar", ) handler.launch_step( StepHandlerContext( instance, ExecuteStepArgs( reconstructable(bar).get_python_origin(), run.run_id, ["foo_solid"] ), {"foo_solid": {}}, ) ) # Check that user defined k8s config was passed down to the k8s job. mock_method_calls = mock_k8s_client_batch_api.method_calls assert len(mock_method_calls) > 0 method_name, _args, kwargs = mock_method_calls[0] assert method_name == "create_namespaced_job" assert kwargs["body"].spec.template.spec.containers[0].image == "bizbuz"
def test_requires_k8s_launcher_fail(): with instance_for_test() as instance: with pytest.raises( DagsterUnmetExecutorRequirementsError, match="This engine is only compatible with a K8sRunLauncher", ): execute_pipeline(reconstructable(bar), instance=instance)
def test_solid_selection(): recon_pipe = reconstructable(get_the_pipeline) sub_pipe_full = recon_pipe.subset_for_execution(["the_solid"]) assert sub_pipe_full.solids_to_execute == {"the_solid"} sub_pipe_unresolved = recon_pipe.subset_for_execution(["the_solid+"]) assert sub_pipe_unresolved.solids_to_execute == {"the_solid"}
def test_step_handler_user_defined_config(kubeconfig_file): mock_k8s_client_batch_api = mock.MagicMock() handler = K8sStepHandler( job_config=DagsterK8sJobConfig(instance_config_map="foobar", job_image="bizbuz"), job_namespace="foo", load_incluster_config=False, kubeconfig_file=kubeconfig_file, k8s_client_batch_api=mock_k8s_client_batch_api, ) # Construct Dagster solid tags with user defined k8s config. expected_resources = { "requests": { "cpu": "250m", "memory": "64Mi" }, "limits": { "cpu": "500m", "memory": "2560Mi" }, } user_defined_k8s_config = UserDefinedDagsterK8sConfig( container_config={"resources": expected_resources}, ) user_defined_k8s_config_json = json.dumps( user_defined_k8s_config.to_dict()) tags = {"dagster-k8s/config": user_defined_k8s_config_json} with instance_for_test() as instance: run = create_run_for_test( instance, pipeline_name="bar", ) handler.launch_step( StepHandlerContext( instance, ExecuteStepArgs( reconstructable(bar).get_python_origin(), run.run_id, ["foo_solid"]), {"foo_solid": tags}, )) # Check that user defined k8s config was passed down to the k8s job. mock_method_calls = mock_k8s_client_batch_api.method_calls assert len(mock_method_calls) > 0 method_name, _args, kwargs = mock_method_calls[0] assert method_name == "create_namespaced_job" assert kwargs["body"].spec.template.spec.containers[ 0].image == "bizbuz" job_resources = kwargs["body"].spec.template.spec.containers[ 0].resources assert job_resources == expected_resources
def test_step_handler_context(): recon_pipeline = reconstructable(foo_pipline) with instance_for_test() as instance: run = create_run_for_test(instance) args = ExecuteStepArgs( pipeline_origin=recon_pipeline.get_python_origin(), pipeline_run_id=run.run_id, step_keys_to_execute=run.step_keys_to_execute, instance_ref=None, ) ctx = StepHandlerContext( instance=instance, execute_step_args=args, step_tags={}, pipeline_run=run, ) assert ctx.execute_step_args == args assert ctx.pipeline_run == run
def test_args_fails(): with pytest.raises( DagsterInvariantViolationError, match="Reconstructable target must be callable with no arguments", ): reconstructable(get_with_args)
def test_lambda(): with pytest.raises(DagsterInvariantViolationError, match="Reconstructable target can not be a lambda"): reconstructable(lambda_version)
def test_decorator(): recon_pipe = reconstructable(the_pipeline) assert pid(recon_pipe.get_definition()) == pid(the_pipeline)
def test_function(): recon_pipe = reconstructable(get_the_pipeline) assert pid(recon_pipe.get_definition()) == pid(the_pipeline)
def test_bad_target(): with pytest.raises( DagsterInvariantViolationError, match='must resolve to a PipelineDefinition', ): reconstructable(not_the_pipeline)