def test_manual_instance():
defn = PipelineDefinition([the_solid], "test")
with pytest.raises(
DagsterInvariantViolationError,
match="Reconstructable target should be a function or definition produced by a decorated function",
):
reconstructable(defn)
def test_inner_scope_2():
def get_the_pipeline_inner():
return the_pipeline
with pytest.raises(
DagsterInvariantViolationError, match="not found at module scope in file",
):
reconstructable(get_the_pipeline_inner)
def test_bad_target():
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
"Loadable attributes must be either a JobDefinition, GraphDefinition, PipelineDefinition, or a "
"RepositoryDefinition. Got None."),
):
reconstructable(not_the_pipeline)
def test_not_defined_in_module(mocker):
mocker.patch("inspect.getmodule", return_value=types.ModuleType("__main__"))
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
"reconstructable() can not reconstruct pipelines defined in interactive environments"
),
):
reconstructable(get_the_pipeline)
def test_inner_scope():
def get_the_pipeline_inner():
return the_pipeline
with pytest.raises(
DagsterInvariantViolationError,
match="Use a function or decorated function defined at module scope",
):
reconstructable(get_the_pipeline_inner)
def test_inner_decorator_2():
@pipeline
def pipe():
the_solid()
with pytest.raises(
DagsterInvariantViolationError, match="not found at module scope in file",
):
reconstructable(pipe)
def test_inner_decorator():
@pipeline
def pipe():
the_solid()
with pytest.raises(
DagsterInvariantViolationError,
match="Use a function or decorated function defined at module scope",
):
reconstructable(pipe)
def test_reconstructable_module():
original_sys_path = sys.path
try:
sys.path.insert(0, file_relative_path(__file__, "."))
from foo import bar_pipeline # pylint: disable=import-error
reconstructable(bar_pipeline)
finally:
sys.path = original_sys_path
def test_step_handler(kubeconfig_file):
mock_k8s_client_batch_api = mock.MagicMock()
handler = K8sStepHandler(
job_config=DagsterK8sJobConfig(instance_config_map="foobar", job_image="bizbuz"),
job_namespace="foo",
load_incluster_config=False,
kubeconfig_file=kubeconfig_file,
k8s_client_batch_api=mock_k8s_client_batch_api,
)
with instance_for_test() as instance:
run = create_run_for_test(
instance,
pipeline_name="bar",
)
handler.launch_step(
StepHandlerContext(
instance,
ExecuteStepArgs(
reconstructable(bar).get_python_origin(), run.run_id, ["foo_solid"]
),
{"foo_solid": {}},
)
)
# Check that user defined k8s config was passed down to the k8s job.
mock_method_calls = mock_k8s_client_batch_api.method_calls
assert len(mock_method_calls) > 0
method_name, _args, kwargs = mock_method_calls[0]
assert method_name == "create_namespaced_job"
assert kwargs["body"].spec.template.spec.containers[0].image == "bizbuz"
def test_requires_k8s_launcher_fail():
with instance_for_test() as instance:
with pytest.raises(
DagsterUnmetExecutorRequirementsError,
match="This engine is only compatible with a K8sRunLauncher",
):
execute_pipeline(reconstructable(bar), instance=instance)
def test_solid_selection():
recon_pipe = reconstructable(get_the_pipeline)
sub_pipe_full = recon_pipe.subset_for_execution(["the_solid"])
assert sub_pipe_full.solids_to_execute == {"the_solid"}
sub_pipe_unresolved = recon_pipe.subset_for_execution(["the_solid+"])
assert sub_pipe_unresolved.solids_to_execute == {"the_solid"}
def test_step_handler_user_defined_config(kubeconfig_file):
mock_k8s_client_batch_api = mock.MagicMock()
handler = K8sStepHandler(
job_config=DagsterK8sJobConfig(instance_config_map="foobar",
job_image="bizbuz"),
job_namespace="foo",
load_incluster_config=False,
kubeconfig_file=kubeconfig_file,
k8s_client_batch_api=mock_k8s_client_batch_api,
)
# Construct Dagster solid tags with user defined k8s config.
expected_resources = {
"requests": {
"cpu": "250m",
"memory": "64Mi"
},
"limits": {
"cpu": "500m",
"memory": "2560Mi"
},
}
user_defined_k8s_config = UserDefinedDagsterK8sConfig(
container_config={"resources": expected_resources}, )
user_defined_k8s_config_json = json.dumps(
user_defined_k8s_config.to_dict())
tags = {"dagster-k8s/config": user_defined_k8s_config_json}
with instance_for_test() as instance:
run = create_run_for_test(
instance,
pipeline_name="bar",
)
handler.launch_step(
StepHandlerContext(
instance,
ExecuteStepArgs(
reconstructable(bar).get_python_origin(), run.run_id,
["foo_solid"]),
{"foo_solid": tags},
))
# Check that user defined k8s config was passed down to the k8s job.
mock_method_calls = mock_k8s_client_batch_api.method_calls
assert len(mock_method_calls) > 0
method_name, _args, kwargs = mock_method_calls[0]
assert method_name == "create_namespaced_job"
assert kwargs["body"].spec.template.spec.containers[
0].image == "bizbuz"
job_resources = kwargs["body"].spec.template.spec.containers[
0].resources
assert job_resources == expected_resources
def test_step_handler_context():
recon_pipeline = reconstructable(foo_pipline)
with instance_for_test() as instance:
run = create_run_for_test(instance)
args = ExecuteStepArgs(
pipeline_origin=recon_pipeline.get_python_origin(),
pipeline_run_id=run.run_id,
step_keys_to_execute=run.step_keys_to_execute,
instance_ref=None,
)
ctx = StepHandlerContext(
instance=instance,
execute_step_args=args,
step_tags={},
pipeline_run=run,
)
assert ctx.execute_step_args == args
assert ctx.pipeline_run == run
def test_args_fails():
with pytest.raises(
DagsterInvariantViolationError,
match="Reconstructable target must be callable with no arguments",
):
reconstructable(get_with_args)
def test_lambda():
with pytest.raises(DagsterInvariantViolationError,
match="Reconstructable target can not be a lambda"):
reconstructable(lambda_version)
def test_decorator():
recon_pipe = reconstructable(the_pipeline)
assert pid(recon_pipe.get_definition()) == pid(the_pipeline)
def test_function():
recon_pipe = reconstructable(get_the_pipeline)
assert pid(recon_pipe.get_definition()) == pid(the_pipeline)
def test_bad_target():
with pytest.raises(
DagsterInvariantViolationError,
match='must resolve to a PipelineDefinition',
):
reconstructable(not_the_pipeline)