def test_no_outputs_no_inputs_config_schema(): assert ResolvedRunConfig.build(no_input_no_output_pipeline()) with pytest.raises(DagsterInvalidConfigError) as exc_context: ResolvedRunConfig.build(no_input_no_output_pipeline(), {"solids": {"return_one": {}}}) assert len(exc_context.value.errors) == 1 assert ( 'Error 1: Received unexpected config entry "return_one" at path root:solids' in exc_context.value.message )
def test_basic_json_misnamed_output_config_schema(): with pytest.raises(DagsterInvalidConfigError) as exc_context: ResolvedRunConfig.build( single_int_named_output_pipeline(), { "solids": { "return_named_one": {"outputs": [{"wrong_name": {"json": {"path": "foo"}}}]} } }, ) assert len(exc_context.value.errors) == 1 assert 'Error 1: Received unexpected config entry "wrong_name"' in exc_context.value.message assert "at path root:solids:return_named_one:outputs[0]" in exc_context.value.message
def test_basic_json_default_output_config_schema(): env = ResolvedRunConfig.build( single_int_output_pipeline(), { "solids": { "return_one": { "outputs": [{ "result": { "json": { "path": "foo" } } }] } } }, ) assert env.solids["return_one"] assert env.solids["return_one"].outputs.type_materializer_specs == [{ "result": { "json": { "path": "foo" } } }]
def test_execution_plan_reexecution_with_in_memory(): pipeline_def = define_addy_pipeline() instance = DagsterInstance.ephemeral() run_config = {"solids": {"add_one": {"inputs": {"num": {"value": 3}}}}} result = execute_pipeline(pipeline_def, run_config=run_config, instance=instance) assert result.success ## re-execute add_two resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config=run_config) execution_plan = ExecutionPlan.build(InMemoryPipeline(pipeline_def), resolved_run_config) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, execution_plan=execution_plan, run_config=run_config, parent_run_id=result.run_id, root_run_id=result.run_id, ) with pytest.raises(DagsterInvariantViolationError): execute_plan( execution_plan.build_subset_plan(["add_two"], pipeline_def, resolved_run_config), InMemoryPipeline(pipeline_def), run_config=run_config, pipeline_run=pipeline_run, instance=instance, )
def test_compile(): resolved_run_config = ResolvedRunConfig.build( composition, {"solids": { "add_four": { "inputs": { "num": { "value": 1 } } } }}, ) plan = ExecutionPlan.build(InMemoryPipeline(composition), resolved_run_config) res = coalesce_execution_steps(plan) assert set(res.keys()) == { "add_four.add", "div_four.div_two", "div_four.div_two_2", "add_four.emit_two.emit_one_2", "add_four.emit_two_2.add", "int_to_float", "add_four.emit_two_2.emit_one_2", "add_four.emit_two.add", "add_four.emit_two_2.emit_one", "add_four.emit_two.emit_one", }
def create_execution_plan( pipeline: Union[IPipeline, PipelineDefinition], run_config: Optional[dict] = None, mode: Optional[str] = None, step_keys_to_execute: Optional[List[str]] = None, known_state: KnownExecutionState = None, ) -> ExecutionPlan: pipeline = _check_pipeline(pipeline) pipeline_def = pipeline.get_definition() check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config = check.opt_dict_param(run_config, "run_config", key_type=str) mode = check.opt_str_param(mode, "mode", default=pipeline_def.get_default_mode_name()) check.opt_nullable_list_param(step_keys_to_execute, "step_keys_to_execute", of_type=str) resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config, mode=mode) return ExecutionPlan.build( pipeline, resolved_run_config, step_keys_to_execute=step_keys_to_execute, known_state=known_state, )
def test_memoized_plan_memoized_results(): with instance_for_test() as instance: manager = VersionedInMemoryIOManager() versioned_pipeline = versioned_pipeline_factory(manager) plan = create_execution_plan(versioned_pipeline, instance_ref=instance.get_ref()) resolved_run_config = ResolvedRunConfig.build(versioned_pipeline) # Affix a memoized value to the output step_output_handle = StepOutputHandle("versioned_solid_no_input", "result") step_output_version = plan.get_version_for_step_output_handle( step_output_handle) manager.values[(step_output_handle.step_key, step_output_handle.output_name, step_output_version)] = 4 memoized_plan = plan.build_memoized_plan(versioned_pipeline, resolved_run_config, instance=None, selected_step_keys=None) assert memoized_plan.step_keys_to_execute == [ "versioned_solid_takes_input" ]
def test_resolve_memoized_execution_plan_partial_versioning(): manager = VersionedInMemoryIOManager() partially_versioned_pipeline = partially_versioned_pipeline_factory( manager) speculative_execution_plan = create_execution_plan( partially_versioned_pipeline) resolved_run_config = ResolvedRunConfig.build(partially_versioned_pipeline) step_output_handle = StepOutputHandle("versioned_solid_no_input", "result") step_output_version = resolve_step_output_versions( partially_versioned_pipeline, speculative_execution_plan, resolved_run_config)[step_output_handle] manager.values[(step_output_handle.step_key, step_output_handle.output_name, step_output_version)] = 4 with DagsterInstance.ephemeral() as instance: assert (resolve_memoized_execution_plan( speculative_execution_plan, partially_versioned_pipeline, {}, instance, resolved_run_config, ).step_keys_to_execute == ["solid_takes_input"])
def validate_run_config( pipeline_def: PipelineDefinition, run_config: Optional[Dict[str, Any]] = None, mode: Optional[str] = None, ) -> Dict[str, Any]: """Function to validate a provided run config blob against a given pipeline and mode. If validation is successful, this function will return a dictionary representation of the validated config actually used during execution. Args: pipeline_def (PipelineDefinition): The pipeline definition to validate run config against run_config (Optional[Dict[str, Any]]): The run config to validate mode (str): The mode of the pipeline to validate against (different modes may require different config) Returns: Dict[str, Any]: A dictionary representation of the validated config. """ pipeline_def = check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config = check.opt_dict_param(run_config, "run_config", key_type=str) mode = check.opt_str_param(mode, "mode", default=pipeline_def.get_default_mode_name()) return ResolvedRunConfig.build(pipeline_def, run_config, mode=mode).to_dict()
def test_optional_solid_with_optional_scalar_config(): def _assert_config_none(context, value): assert context.solid_config is value pipeline_def = PipelineDefinition( name="some_pipeline", solid_defs=[ SolidDefinition( name="int_config_solid", config_schema=Field(Int, is_required=False), input_defs=[], output_defs=[], compute_fn=lambda context, _inputs: _assert_config_none( context, 234), ) ], ) env_type = create_run_config_schema_type(pipeline_def) assert env_type.fields["solids"].is_required is False solids_type = env_type.fields["solids"].config_type assert solids_type.fields["int_config_solid"].is_required is False env_obj = ResolvedRunConfig.build(pipeline_def, {}) assert env_obj.solids["int_config_solid"].config is None
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() env_obj = ResolvedRunConfig.build( pipeline_def, { "solids": { "int_config_solid": { "config": 1 }, "string_config_solid": { "config": "bar" } }, }, ) value = env_obj.solids assert set(["int_config_solid", "string_config_solid"]) == set(value.keys()) assert value == { "int_config_solid": SolidConfig.from_dict({"config": 1}), "string_config_solid": SolidConfig.from_dict({"config": "bar"}), }
def test_solid_dictionary_some_no_config(): @solid(name="int_config_solid", config_schema=Int, input_defs=[], output_defs=[]) def int_config_solid(_): return None @solid(name="no_config_solid", input_defs=[], output_defs=[]) def no_config_solid(_): return None @pipeline def pipeline_def(): int_config_solid() no_config_solid() env = ResolvedRunConfig.build( pipeline_def, {"solids": { "int_config_solid": { "config": 1 } }}) assert {"int_config_solid", "no_config_solid"} == set(env.solids.keys()) assert env.solids == { "int_config_solid": SolidConfig.from_dict({"config": 1}), "no_config_solid": SolidConfig.from_dict({}), }
def test_using_file_system_for_subplan_invalid_step(): pipeline = define_inty_pipeline() run_config = {"storage": {"filesystem": {}}} instance = DagsterInstance.ephemeral() resolved_run_config = ResolvedRunConfig.build( pipeline, run_config=run_config, ) execution_plan = ExecutionPlan.build( InMemoryPipeline(pipeline), resolved_run_config, ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline, execution_plan=execution_plan) with pytest.raises(DagsterExecutionStepNotFoundError): execute_plan( execution_plan.build_subset_plan(["nope.compute"], pipeline, resolved_run_config), InMemoryPipeline(pipeline), instance, run_config=run_config, pipeline_run=pipeline_run, )
def _io_manager(init_context): if not init_context.pipeline_run: raise DagsterInvariantViolationError( "Attempted to construct intermediate storage outside of execution context. " "Intermediate storage can only be constructed within the context of an execution." ) pipeline_run = init_context.pipeline_run instance = init_context.instance pipeline_def = init_context.pipeline_def_for_backwards_compat # depend on InitResourceContext.instance and pipeline_def_for_backwards_compat resolved_run_config = ResolvedRunConfig.build(pipeline_def, pipeline_run.run_config, mode=pipeline_run.mode) mode_def = pipeline_def.get_mode_definition(pipeline_run.mode) intermediate_storage_context = InitIntermediateStorageContext( pipeline_def=pipeline_def, mode_def=mode_def, intermediate_storage_def=intermediate_storage_def, pipeline_run=pipeline_run, instance=instance, resolved_run_config=resolved_run_config, type_storage_plugin_registry=construct_type_storage_plugin_registry( pipeline_def, intermediate_storage_def), resources=init_context.resources, intermediate_storage_config=resolved_run_config. intermediate_storage.intermediate_storage_config, ) intermediate_storage = intermediate_storage_def.intermediate_storage_creation_fn( intermediate_storage_context) return IntermediateStorageAdapter(intermediate_storage)
def basic_resource_versions(): run_config = { "resources": { "basic_resource": { "config": { "input_str": "apple" }, }, "resource_no_version": { "config": { "input_str": "banana" } }, } } resolved_run_config = ResolvedRunConfig.build(modes_pipeline, run_config, mode="fakemode") resource_versions_by_key = resolve_resource_versions( resolved_run_config, modes_pipeline) assert resource_versions_by_key["basic_resource"] == join_and_hash( resolve_config_version({"input_str": "apple"}), basic_resource.version) assert resource_versions_by_key["resource_no_version"] == None assert resource_versions_by_key["resource_no_config"] == join_and_hash( join_and_hash(), "42")
def test_using_file_system_for_subplan_missing_input(): pipeline = define_inty_pipeline() run_config = {"storage": {"filesystem": {}}} instance = DagsterInstance.ephemeral() resolved_run_config = ResolvedRunConfig.build( pipeline, run_config=run_config, ) execution_plan = ExecutionPlan.build( InMemoryPipeline(pipeline), resolved_run_config, ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline, execution_plan=execution_plan) events = execute_plan( execution_plan.build_subset_plan(["add_one"], pipeline, resolved_run_config), InMemoryPipeline(pipeline), instance, run_config=run_config, pipeline_run=pipeline_run, ) failures = [ event for event in events if event.event_type_value == "STEP_FAILURE" ] assert len(failures) == 1 assert failures[0].step_key == "add_one" assert "DagsterStepOutputNotFoundError" in failures[ 0].event_specific_data.error.message
def test_get_output_context_with_resources(): @solid def basic_solid(): pass @pipeline def basic_pipeline(): basic_solid() with pytest.raises( CheckError, match="Expected either resources or step context to be set, but " "received both. If step context is provided, resources for IO manager will be " "retrieved off of that.", ): get_output_context( execution_plan=create_execution_plan(basic_pipeline), pipeline_def=basic_pipeline, resolved_run_config=ResolvedRunConfig.build(basic_pipeline), step_output_handle=StepOutputHandle("basic_solid", "result"), run_id=None, log_manager=None, step_context=mock.MagicMock(), resources=mock.MagicMock(), )
def create_context_creation_data( pipeline: IPipeline, execution_plan: ExecutionPlan, run_config: Dict[str, Any], pipeline_run: PipelineRun, instance: DagsterInstance, ) -> "ContextCreationData": pipeline_def = pipeline.get_definition() resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config, mode=pipeline_run.mode) mode_def = pipeline_def.get_mode_definition(pipeline_run.mode) intermediate_storage_def = resolved_run_config.intermediate_storage_def_for_mode( mode_def) executor_def = executor_def_from_config(mode_def, resolved_run_config) return ContextCreationData( pipeline=pipeline, resolved_run_config=resolved_run_config, pipeline_run=pipeline_run, mode_def=mode_def, intermediate_storage_def=intermediate_storage_def, executor_def=executor_def, instance=instance, resource_keys_to_init=get_required_resource_keys_to_init( execution_plan, pipeline_def, resolved_run_config, intermediate_storage_def), execution_plan=execution_plan, )
def test_resolve_memoized_execution_plan_yes_stored_results(): manager = VersionedInMemoryIOManager() versioned_pipeline = versioned_pipeline_factory(manager) speculative_execution_plan = create_execution_plan(versioned_pipeline) resolved_run_config = ResolvedRunConfig.build(versioned_pipeline) step_output_handle = StepOutputHandle("versioned_solid_no_input", "result") step_output_version = resolve_step_output_versions( versioned_pipeline, speculative_execution_plan, resolved_run_config)[step_output_handle] manager.values[(step_output_handle.step_key, step_output_handle.output_name, step_output_version)] = 4 with DagsterInstance.ephemeral() as dagster_instance: memoized_execution_plan = resolve_memoized_execution_plan( speculative_execution_plan, versioned_pipeline, {}, dagster_instance, resolved_run_config, ) assert memoized_execution_plan.step_keys_to_execute == [ "versioned_solid_takes_input" ] expected_handle = StepOutputHandle(step_key="versioned_solid_no_input", output_name="result") assert (memoized_execution_plan.get_step_by_key( "versioned_solid_takes_input").step_input_dict["intput"].source. step_output_handle == expected_handle)
def test_clean_event_generator_exit(): """Testing for generator cleanup (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/) """ from dagster.core.execution.context.init import InitResourceContext from dagster.core.definitions.resource import ScopedResourcesBuilder pipeline_def = gen_basic_resource_pipeline() instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline_def) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, execution_plan=execution_plan) log_manager = DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]) resolved_run_config = ResolvedRunConfig.build(pipeline_def) execution_plan = create_execution_plan(pipeline_def) resource_name, resource_def = next( iter(pipeline_def.get_default_mode().resource_defs.items())) resource_context = InitResourceContext( resource_def=resource_def, resources=ScopedResourcesBuilder().build(None), resource_config=None, pipeline_run=pipeline_run, instance=instance, ) generator = single_resource_event_generator(resource_context, resource_name, resource_def) next(generator) generator.close() resource_defs = pipeline_def.get_mode_definition(resolved_run_config.mode) generator = resource_initialization_event_generator( resource_defs=resource_defs, resource_configs=resolved_run_config.resources, log_manager=log_manager, execution_plan=execution_plan, pipeline_run=pipeline_run, resource_keys_to_init={"a"}, instance=instance, emit_persistent_events=True, pipeline_def_for_backwards_compat=pipeline_def, ) next(generator) generator.close() generator = PlanExecutionContextManager( # pylint: disable=protected-access pipeline=InMemoryPipeline(pipeline_def), execution_plan=execution_plan, run_config={}, pipeline_run=pipeline_run, instance=instance, retry_mode=RetryMode.DISABLED, scoped_resources_builder_cm=resource_initialization_manager, ).get_generator() next(generator) generator.close()
def get_step_keys_to_execute(pipeline, run_config, mode, instance): memoized_execution_plan = resolve_memoized_execution_plan( create_execution_plan(pipeline, run_config=run_config, mode=mode), pipeline, run_config, instance, ResolvedRunConfig.build(pipeline, run_config=run_config, mode=mode), ) return memoized_execution_plan.step_keys_to_execute
def test_using_intermediate_file_system_for_subplan_multiprocessing(): with instance_for_test() as instance: run_config = {"intermediate_storage": {"filesystem": {}}} pipeline = reconstructable(define_inty_pipeline) resolved_run_config = ResolvedRunConfig.build( pipeline.get_definition(), run_config=run_config, ) execution_plan = ExecutionPlan.build( pipeline, resolved_run_config, ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), execution_plan=execution_plan) assert execution_plan.get_step_by_key("return_one") return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["return_one"], pipeline.get_definition(), resolved_run_config), pipeline, instance, run_config=dict(run_config, execution={"multiprocess": {}}), pipeline_run=pipeline_run, )) intermediate_storage = build_fs_intermediate_storage( instance.intermediates_directory, pipeline_run.run_id) assert get_step_output(return_one_step_events, "return_one") assert intermediate_storage.has_intermediate( None, StepOutputHandle("return_one")) assert (intermediate_storage.get_intermediate( None, Int, StepOutputHandle("return_one")).obj == 1) add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["add_one"], pipeline.get_definition(), resolved_run_config), pipeline, instance, run_config=dict(run_config, execution={"multiprocess": {}}), pipeline_run=pipeline_run, )) assert get_step_output(add_one_step_events, "add_one") assert intermediate_storage.has_intermediate( None, StepOutputHandle("add_one")) assert (intermediate_storage.get_intermediate( None, Int, StepOutputHandle("add_one")).obj == 2)
def test_default_environment(): @solid(name="some_solid", input_defs=[], output_defs=[]) def some_solid(_): return None @pipeline def pipeline_def(): some_solid() assert ResolvedRunConfig.build(pipeline_def, {})
def run_test_with_builtin_type(type_to_test, loader_version, type_value): @solid(version="42", input_defs=[InputDefinition("_builtin_type", type_to_test)]) def versioned_solid_ext_input_builtin_type(_, _builtin_type): pass @pipeline def versioned_pipeline_ext_input_builtin_type(): versioned_solid_takes_input(versioned_solid_ext_input_builtin_type()) run_config = { "solids": { "versioned_solid_ext_input_builtin_type": { "inputs": { "_builtin_type": type_value } } } } speculative_execution_plan = create_execution_plan( versioned_pipeline_ext_input_builtin_type, run_config=run_config, ) resolved_run_config = ResolvedRunConfig.build( versioned_pipeline_ext_input_builtin_type, run_config=run_config) versions = resolve_step_versions(versioned_pipeline_ext_input_builtin_type, speculative_execution_plan, resolved_run_config) ext_input_version = join_and_hash(str(type_value)) input_version = join_and_hash(loader_version + ext_input_version) solid1_def_version = versioned_solid_ext_input_builtin_type.version solid1_config_version = resolve_config_version(None) solid1_resources_version = join_and_hash() solid1_version = join_and_hash(solid1_def_version, solid1_config_version, solid1_resources_version) step1_version = join_and_hash(input_version, solid1_version) assert versions["versioned_solid_ext_input_builtin_type"] == step1_version output_version = join_and_hash(step1_version, "result") hashed_input2 = output_version solid2_def_version = versioned_solid_takes_input.version solid2_config_version = resolve_config_version(None) solid2_resources_version = join_and_hash() solid2_version = join_and_hash(solid2_def_version, solid2_config_version, solid2_resources_version) step2_version = join_and_hash(hashed_input2, solid2_version) assert versions["versioned_solid_takes_input"] == step2_version
def test_whole_environment(): pipeline_def = PipelineDefinition( name="some_pipeline", mode_defs=[ ModeDefinition( name="test_mode", resource_defs={ "test_resource": ResourceDefinition(resource_fn=lambda _: None, config_schema=Any) }, ) ], solid_defs=[ SolidDefinition( name="int_config_solid", config_schema=Int, input_defs=[], output_defs=[OutputDefinition()], required_resource_keys={"test_resource"}, compute_fn=lambda *args: None, ), SolidDefinition(name="no_config_solid", input_defs=[], output_defs=[], compute_fn=lambda *args: None), ], ) env = ResolvedRunConfig.build( pipeline_def, { "resources": { "test_resource": { "config": 1 } }, "solids": { "int_config_solid": { "config": 123 } }, }, ) assert isinstance(env, ResolvedRunConfig) assert env.solids == { "int_config_solid": SolidConfig.from_dict({"config": 123}), "no_config_solid": SolidConfig.from_dict({}), } assert env.resources == { "test_resource": ResourceConfig(1), "io_manager": ResourceConfig(None) }
def test_step_versions_with_resources(): run_config = { "resources": { "basic_resource": { "config": { "input_str": "apple" } } } } speculative_execution_plan = create_execution_plan( versioned_modes_pipeline, run_config=run_config, mode="fakemode") resolved_run_config = ResolvedRunConfig.build(versioned_modes_pipeline, run_config=run_config, mode="fakemode") versions = resolve_step_versions(versioned_modes_pipeline, speculative_execution_plan, resolved_run_config) solid_def_version = fake_solid_resources_versioned.version solid_config_version = resolve_config_version(None) resolved_run_config = ResolvedRunConfig.build(versioned_modes_pipeline, run_config, mode="fakemode") resource_versions_by_key = resolve_resource_versions( resolved_run_config, versioned_modes_pipeline, ) solid_resources_version = join_and_hash(*[ resource_versions_by_key[resource_key] for resource_key in fake_solid_resources_versioned.required_resource_keys ]) solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version) step_version = join_and_hash(solid_version) assert versions["fake_solid_resources_versioned"] == step_version
def test_using_file_system_for_subplan_multiprocessing(): with instance_for_test() as instance: pipeline = reconstructable(define_reconstructable_inty_pipeline) resolved_run_config = ResolvedRunConfig.build( pipeline.get_definition(), ) execution_plan = ExecutionPlan.build( pipeline, resolved_run_config, ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline.get_definition(), execution_plan=execution_plan) assert execution_plan.get_step_by_key("return_one") return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["return_one"], pipeline.get_definition(), resolved_run_config), pipeline, instance, run_config=dict(execution={"multiprocess": {}}), pipeline_run=pipeline_run, )) assert get_step_output(return_one_step_events, "return_one") with open( os.path.join(instance.storage_directory(), pipeline_run.run_id, "return_one", "result"), "rb", ) as read_obj: assert pickle.load(read_obj) == 1 add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["add_one"], pipeline.get_definition(), resolved_run_config), pipeline, instance, run_config=dict(execution={"multiprocess": {}}), pipeline_run=pipeline_run, )) assert get_step_output(add_one_step_events, "add_one") with open( os.path.join(instance.storage_directory(), pipeline_run.run_id, "add_one", "result"), "rb", ) as read_obj: assert pickle.load(read_obj) == 2
def test_using_intermediates_file_system_for_subplan(): pipeline = define_inty_pipeline() run_config = {"intermediate_storage": {"filesystem": {}}} instance = DagsterInstance.ephemeral() resolved_run_config = ResolvedRunConfig.build( pipeline, run_config=run_config, ) execution_plan = ExecutionPlan.build( InMemoryPipeline(pipeline), resolved_run_config, ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline, execution_plan=execution_plan) assert execution_plan.get_step_by_key("return_one") return_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["return_one"], pipeline, resolved_run_config), InMemoryPipeline(pipeline), instance, run_config=run_config, pipeline_run=pipeline_run, )) intermediate_storage = build_fs_intermediate_storage( instance.intermediates_directory, pipeline_run.run_id) assert get_step_output(return_one_step_events, "return_one") assert intermediate_storage.has_intermediate( None, StepOutputHandle("return_one")) assert intermediate_storage.get_intermediate( None, Int, StepOutputHandle("return_one")).obj == 1 add_one_step_events = list( execute_plan( execution_plan.build_subset_plan(["add_one"], pipeline, resolved_run_config), InMemoryPipeline(pipeline), instance, run_config=run_config, pipeline_run=pipeline_run, )) assert get_step_output(add_one_step_events, "add_one") assert intermediate_storage.has_intermediate(None, StepOutputHandle("add_one")) assert intermediate_storage.get_intermediate( None, Int, StepOutputHandle("add_one")).obj == 2
def test_no_outputs_one_input_config_schema(): assert ResolvedRunConfig.build( one_input_no_output_pipeline(), {"solids": {"take_input_return_nothing": {"inputs": {"dummy": {"value": "value"}}}}}, ) with pytest.raises(DagsterInvalidConfigError) as exc_context: ResolvedRunConfig.build( one_input_no_output_pipeline(), { "solids": { "take_input_return_nothing": { "inputs": {"dummy": {"value": "value"}}, "outputs": {}, } } }, ) assert len(exc_context.value.errors) == 1 exp_msg = 'Error 1: Received unexpected config entry "outputs" at path root:solids:take_input_return_nothing' assert exp_msg in exc_context.value.message
def test_intermediate_storage_definition_run_config_required(): """Run config required for intermediate storage definition, none provided to pipeline def.""" intermediate_storage_requires_config = IntermediateStorageDefinition( name="test_intermediate_requires_config", is_persistent=False, required_resource_keys=set(), config_schema={"field": Field(StringSource)}, ) run_config = { "intermediate_storage": { "test_intermediate_requires_config": { "config": { "field": "value" } } } } fake_mode = ModeDefinition( name="fakemode", intermediate_storage_defs=[intermediate_storage_requires_config]) pipeline_def = PipelineDefinition([fake_solid], name="fakename", mode_defs=[fake_mode]) resolved_run_config = ResolvedRunConfig.build(pipeline_def, run_config, mode="fakemode") assert (resolved_run_config.intermediate_storage.intermediate_storage_name == "test_intermediate_requires_config") with pytest.raises(DagsterInvalidConfigError): resolved_run_config = ResolvedRunConfig.build(pipeline_def, {}, mode="fakemode")