def resolve_step_versions_for_test(execution_plan, run_config=None, mode=None): return resolve_step_versions( execution_plan=execution_plan, environment_config=EnvironmentConfig.build(execution_plan.pipeline_def, run_config, mode), mode_def=execution_plan.pipeline_def.get_mode_definition(mode), )
def test_step_versions_composite_solid(): @solid(config_schema=Field(String, is_required=False)) def scalar_config_solid(context): yield Output(context.solid_config) @composite_solid( config_schema={"override_str": Field(String)}, config_fn=lambda cfg: {"scalar_config_solid": {"config": cfg["override_str"]}}, ) def wrap(): return scalar_config_solid() @pipeline def wrap_pipeline(): wrap.alias("do_stuff")() run_config = { "solids": {"do_stuff": {"config": {"override_str": "override"}}}, "loggers": {"console": {"config": {"log_level": "ERROR"}}}, } speculative_execution_plan = create_execution_plan(wrap_pipeline, run_config=run_config) environment_config = EnvironmentConfig.build(wrap_pipeline, run_config=run_config) versions = resolve_step_versions(wrap_pipeline, speculative_execution_plan, environment_config) assert versions["do_stuff.scalar_config_solid"] == None
def test_step_versions_with_resources(): run_config = {"resources": {"test_resource": {"config": {"input_str": "apple"}}}} speculative_execution_plan = create_execution_plan( versioned_modes_pipeline, run_config=run_config, mode="fakemode" ) environment_config = EnvironmentConfig.build( versioned_modes_pipeline, run_config=run_config, mode="fakemode" ) versions = resolve_step_versions( versioned_modes_pipeline, speculative_execution_plan, environment_config ) solid_def_version = fake_solid_resources_versioned.version solid_config_version = resolve_config_version(None) environment_config = EnvironmentConfig.build( versioned_modes_pipeline, run_config, mode="fakemode" ) resource_versions_by_key = resolve_resource_versions( environment_config, versioned_modes_pipeline, ) solid_resources_version = join_and_hash( *[ resource_versions_by_key[resource_key] for resource_key in fake_solid_resources_versioned.required_resource_keys ] ) solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version) step_version = join_and_hash(solid_version) assert versions["fake_solid_resources_versioned"] == step_version
def test_resolve_step_versions_no_external_dependencies(): speculative_execution_plan = create_execution_plan(versioned_pipeline) versions = resolve_step_versions(speculative_execution_plan) assert (versions["versioned_solid_no_input.compute"] == versioned_pipeline_expected_step1_version()) assert (versions["versioned_solid_takes_input.compute"] == versioned_pipeline_expected_step2_version())
def run_test_with_builtin_type(type_to_test, loader_version, type_value): @solid(version="42", input_defs=[InputDefinition("_builtin_type", type_to_test)]) def versioned_solid_ext_input_builtin_type(_, _builtin_type): pass @pipeline def versioned_pipeline_ext_input_builtin_type(): versioned_solid_takes_input(versioned_solid_ext_input_builtin_type()) run_config = { "solids": { "versioned_solid_ext_input_builtin_type": { "inputs": { "_builtin_type": type_value } } } } speculative_execution_plan = create_execution_plan( versioned_pipeline_ext_input_builtin_type, run_config=run_config, ) resolved_run_config = ResolvedRunConfig.build( versioned_pipeline_ext_input_builtin_type, run_config=run_config) versions = resolve_step_versions(versioned_pipeline_ext_input_builtin_type, speculative_execution_plan, resolved_run_config) ext_input_version = join_and_hash(str(type_value)) input_version = join_and_hash(loader_version + ext_input_version) solid1_def_version = versioned_solid_ext_input_builtin_type.version solid1_config_version = resolve_config_version(None) solid1_resources_version = join_and_hash() solid1_version = join_and_hash(solid1_def_version, solid1_config_version, solid1_resources_version) step1_version = join_and_hash(input_version, solid1_version) assert versions["versioned_solid_ext_input_builtin_type"] == step1_version output_version = join_and_hash(step1_version, "result") hashed_input2 = output_version solid2_def_version = versioned_solid_takes_input.version solid2_config_version = resolve_config_version(None) solid2_resources_version = join_and_hash() solid2_version = join_and_hash(solid2_def_version, solid2_config_version, solid2_resources_version) step2_version = join_and_hash(hashed_input2, solid2_version) assert versions["versioned_solid_takes_input"] == step2_version
def test_resolve_step_versions_no_external_dependencies(): speculative_execution_plan = create_execution_plan(versioned_pipeline) versions = resolve_step_versions(speculative_execution_plan) solid1_hash = join_and_hash([versioned_solid_no_input.version]) output1_hash = join_and_hash([solid1_hash + "result"]) outputs_hash = join_and_hash([output1_hash]) solid2_hash = join_and_hash([outputs_hash, versioned_solid_takes_input.version]) assert versions["versioned_solid_no_input.compute"] == solid1_hash assert versions["versioned_solid_takes_input.compute"] == solid2_hash
def test_resolve_step_versions_no_external_dependencies(): versioned_pipeline = versioned_pipeline_factory() speculative_execution_plan = create_execution_plan(versioned_pipeline) environment_config = EnvironmentConfig.build(versioned_pipeline) versions = resolve_step_versions( versioned_pipeline, speculative_execution_plan, environment_config ) assert versions["versioned_solid_no_input"] == versioned_pipeline_expected_step1_version() assert versions["versioned_solid_takes_input"] == versioned_pipeline_expected_step2_version()
def test_resolve_step_versions_default_value(): speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value) environment_config = EnvironmentConfig.build(versioned_pipeline_default_value) versions = resolve_step_versions( versioned_pipeline_default_value, speculative_execution_plan, environment_config ) input_version = join_and_hash(repr("DEFAULTVAL")) solid_def_version = versioned_solid_default_value.version solid_config_version = resolve_config_version(None) solid_resources_version = join_and_hash() solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version) step_version = join_and_hash(input_version, solid_version) assert versions["versioned_solid_default_value"] == step_version
def test_resolve_step_versions_external_dependencies(): speculative_execution_plan = create_execution_plan( versioned_pipeline_ext_input, run_config={"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}}, ) versions = resolve_step_versions(speculative_execution_plan) ext_input_version = join_and_hash(["a"]) input_version = join_and_hash([InputHydration.loader_version + ext_input_version]) step_version1 = join_and_hash([input_version, versioned_solid_ext_input.version]) assert versions["versioned_solid_ext_input.compute"] == step_version1 output_version = join_and_hash([step_version1, "result"]) hashed_input2 = join_and_hash([output_version]) step_version2 = join_and_hash([hashed_input2, versioned_solid_takes_input.version]) assert versions["versioned_solid_takes_input.compute"] == step_version2
def test_resolve_step_versions_default_value(): speculative_execution_plan = create_execution_plan( versioned_pipeline_default_value) versions = resolve_step_versions(speculative_execution_plan) default_val_version = join_and_hash("DEFAULTVAL") input_version = join_and_hash(InputHydration.loader_version + default_val_version) solid_def_version = versioned_solid_ext_input.version solid_config_version = resolve_config_version(None) solid_resources_version = join_and_hash() solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version) step_version = join_and_hash(input_version, solid_version) assert versions["versioned_solid_default_value.compute"] == step_version
def test_resolve_step_versions_external_dependencies(): run_config = { "solids": { "versioned_solid_ext_input": { "inputs": { "custom_type": "a" } } } } speculative_execution_plan = create_execution_plan( versioned_pipeline_ext_input, run_config=run_config, ) versions = resolve_step_versions( speculative_execution_plan, run_config=run_config, ) ext_input_version = join_and_hash("a") input_version = join_and_hash(InputHydration.loader_version + ext_input_version) solid1_def_version = versioned_solid_ext_input.version solid1_config_version = resolve_config_version(None) solid1_resources_version = join_and_hash() solid1_version = join_and_hash(solid1_def_version, solid1_config_version, solid1_resources_version) step1_version = join_and_hash(input_version, solid1_version) assert versions["versioned_solid_ext_input.compute"] == step1_version output_version = join_and_hash(step1_version, "result") hashed_input2 = join_and_hash(output_version) solid2_def_version = versioned_solid_takes_input.version solid2_config_version = resolve_config_version(None) solid2_resources_version = join_and_hash() solid2_version = join_and_hash(solid2_def_version, solid2_config_version, solid2_resources_version) step2_version = join_and_hash(hashed_input2, solid2_version) assert versions["versioned_solid_takes_input.compute"] == step2_version
def test_step_versions_separate_io_manager(): mgr = io_manager_factory(VersionedInMemoryIOManager()) @solid(version="39", output_defs=[OutputDefinition(io_manager_key="fake")]) def solid_requires_io_manager(): return Output(5) @pipeline(mode_defs=[ ModeDefinition( name="fakemode", resource_defs={ "fake": mgr, }, ), ]) def io_mgr_pipeline(): solid_requires_io_manager() speculative_execution_plan = create_execution_plan(io_mgr_pipeline, run_config={}, mode="fakemode") resolved_run_config = ResolvedRunConfig.build(io_mgr_pipeline, run_config={}, mode="fakemode") versions = resolve_step_versions(io_mgr_pipeline, speculative_execution_plan, resolved_run_config) solid_def_version = fake_solid_resources_versioned.version solid_config_version = resolve_config_version(None) solid_resources_version = join_and_hash(*[]) solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version) step_version = join_and_hash(solid_version) assert versions["solid_requires_io_manager"] == step_version
def create_run_for_pipeline( self, pipeline_def, execution_plan=None, run_id=None, run_config=None, mode=None, solids_to_execute=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, solid_selection=None, ): from dagster.core.execution.api import create_execution_plan from dagster.core.execution.plan.plan import ExecutionPlan from dagster.core.snap import snapshot_from_execution_plan check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan) # note that solids_to_execute is required to execute the solid subset, which is the # frozenset version of the previous solid_subset. # solid_selection is not required and will not be converted to solids_to_execute here. # i.e. this function doesn't handle solid queries. # solid_selection is only used to pass the user queries further down. check.opt_set_param(solids_to_execute, "solids_to_execute", of_type=str) check.opt_list_param(solid_selection, "solid_selection", of_type=str) if solids_to_execute: if isinstance(pipeline_def, PipelineSubsetDefinition): # for the case when pipeline_def is created by IPipeline or ExternalPipeline check.invariant( solids_to_execute == pipeline_def.solids_to_execute, "Cannot create a PipelineRun from pipeline subset {pipeline_solids_to_execute} " "that conflicts with solids_to_execute arg {solids_to_execute}".format( pipeline_solids_to_execute=str_format_list(pipeline_def.solids_to_execute), solids_to_execute=str_format_list(solids_to_execute), ), ) else: # for cases when `create_run_for_pipeline` is directly called pipeline_def = pipeline_def.get_pipeline_subset_def( solids_to_execute=solids_to_execute ) if _is_memoized_run(tags): if step_keys_to_execute: raise DagsterInvariantViolationError( "step_keys_to_execute parameter cannot be used in conjunction with memoized " "pipeline runs." ) speculative_execution_plan = create_execution_plan( pipeline_def, run_config=run_config, mode=mode, ) step_versions = resolve_step_versions( speculative_execution_plan=speculative_execution_plan, ) step_keys_to_execute = self.resolve_unmemoized_steps( speculative_execution_plan, step_versions ) # TODO: tighter integration with existing step_keys_to_execute functionality if execution_plan is None: execution_plan = create_execution_plan( pipeline_def, run_config=run_config, mode=mode, step_keys_to_execute=step_keys_to_execute, ) return self.create_run( pipeline_name=pipeline_def.name, run_id=run_id, run_config=run_config, mode=check.opt_str_param(mode, "mode", default=pipeline_def.get_default_mode_name()), solid_selection=solid_selection, solids_to_execute=solids_to_execute, step_keys_to_execute=step_keys_to_execute, status=status, tags=tags, root_run_id=root_run_id, parent_run_id=parent_run_id, pipeline_snapshot=pipeline_def.get_pipeline_snapshot(), execution_plan_snapshot=snapshot_from_execution_plan( execution_plan, pipeline_def.get_pipeline_snapshot_id() ), parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(), )