def resolve_step_versions_for_test(execution_plan, run_config=None, mode=None):
return resolve_step_versions(
execution_plan=execution_plan,
environment_config=EnvironmentConfig.build(execution_plan.pipeline_def,
run_config, mode),
mode_def=execution_plan.pipeline_def.get_mode_definition(mode),
)
def test_step_versions_composite_solid():
@solid(config_schema=Field(String, is_required=False))
def scalar_config_solid(context):
yield Output(context.solid_config)
@composite_solid(
config_schema={"override_str": Field(String)},
config_fn=lambda cfg: {"scalar_config_solid": {"config": cfg["override_str"]}},
)
def wrap():
return scalar_config_solid()
@pipeline
def wrap_pipeline():
wrap.alias("do_stuff")()
run_config = {
"solids": {"do_stuff": {"config": {"override_str": "override"}}},
"loggers": {"console": {"config": {"log_level": "ERROR"}}},
}
speculative_execution_plan = create_execution_plan(wrap_pipeline, run_config=run_config)
environment_config = EnvironmentConfig.build(wrap_pipeline, run_config=run_config)
versions = resolve_step_versions(wrap_pipeline, speculative_execution_plan, environment_config)
assert versions["do_stuff.scalar_config_solid"] == None
def test_step_versions_with_resources():
run_config = {"resources": {"test_resource": {"config": {"input_str": "apple"}}}}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config, mode="fakemode"
)
environment_config = EnvironmentConfig.build(
versioned_modes_pipeline, run_config=run_config, mode="fakemode"
)
versions = resolve_step_versions(
versioned_modes_pipeline, speculative_execution_plan, environment_config
)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
environment_config = EnvironmentConfig.build(
versioned_modes_pipeline, run_config, mode="fakemode"
)
resource_versions_by_key = resolve_resource_versions(
environment_config,
versioned_modes_pipeline,
)
solid_resources_version = join_and_hash(
*[
resource_versions_by_key[resource_key]
for resource_key in fake_solid_resources_versioned.required_resource_keys
]
)
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned"] == step_version
def test_resolve_step_versions_no_external_dependencies():
speculative_execution_plan = create_execution_plan(versioned_pipeline)
versions = resolve_step_versions(speculative_execution_plan)
assert (versions["versioned_solid_no_input.compute"] ==
versioned_pipeline_expected_step1_version())
assert (versions["versioned_solid_takes_input.compute"] ==
versioned_pipeline_expected_step2_version())
def run_test_with_builtin_type(type_to_test, loader_version, type_value):
@solid(version="42",
input_defs=[InputDefinition("_builtin_type", type_to_test)])
def versioned_solid_ext_input_builtin_type(_, _builtin_type):
pass
@pipeline
def versioned_pipeline_ext_input_builtin_type():
versioned_solid_takes_input(versioned_solid_ext_input_builtin_type())
run_config = {
"solids": {
"versioned_solid_ext_input_builtin_type": {
"inputs": {
"_builtin_type": type_value
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input_builtin_type,
run_config=run_config,
)
resolved_run_config = ResolvedRunConfig.build(
versioned_pipeline_ext_input_builtin_type, run_config=run_config)
versions = resolve_step_versions(versioned_pipeline_ext_input_builtin_type,
speculative_execution_plan,
resolved_run_config)
ext_input_version = join_and_hash(str(type_value))
input_version = join_and_hash(loader_version + ext_input_version)
solid1_def_version = versioned_solid_ext_input_builtin_type.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
step1_version = join_and_hash(input_version, solid1_version)
assert versions["versioned_solid_ext_input_builtin_type"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = output_version
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input"] == step2_version
def test_resolve_step_versions_no_external_dependencies():
speculative_execution_plan = create_execution_plan(versioned_pipeline)
versions = resolve_step_versions(speculative_execution_plan)
solid1_hash = join_and_hash([versioned_solid_no_input.version])
output1_hash = join_and_hash([solid1_hash + "result"])
outputs_hash = join_and_hash([output1_hash])
solid2_hash = join_and_hash([outputs_hash, versioned_solid_takes_input.version])
assert versions["versioned_solid_no_input.compute"] == solid1_hash
assert versions["versioned_solid_takes_input.compute"] == solid2_hash
def test_resolve_step_versions_no_external_dependencies():
versioned_pipeline = versioned_pipeline_factory()
speculative_execution_plan = create_execution_plan(versioned_pipeline)
environment_config = EnvironmentConfig.build(versioned_pipeline)
versions = resolve_step_versions(
versioned_pipeline, speculative_execution_plan, environment_config
)
assert versions["versioned_solid_no_input"] == versioned_pipeline_expected_step1_version()
assert versions["versioned_solid_takes_input"] == versioned_pipeline_expected_step2_version()
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value)
environment_config = EnvironmentConfig.build(versioned_pipeline_default_value)
versions = resolve_step_versions(
versioned_pipeline_default_value, speculative_execution_plan, environment_config
)
input_version = join_and_hash(repr("DEFAULTVAL"))
solid_def_version = versioned_solid_default_value.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value"] == step_version
def test_resolve_step_versions_external_dependencies():
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input,
run_config={"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}},
)
versions = resolve_step_versions(speculative_execution_plan)
ext_input_version = join_and_hash(["a"])
input_version = join_and_hash([InputHydration.loader_version + ext_input_version])
step_version1 = join_and_hash([input_version, versioned_solid_ext_input.version])
assert versions["versioned_solid_ext_input.compute"] == step_version1
output_version = join_and_hash([step_version1, "result"])
hashed_input2 = join_and_hash([output_version])
step_version2 = join_and_hash([hashed_input2, versioned_solid_takes_input.version])
assert versions["versioned_solid_takes_input.compute"] == step_version2
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(
versioned_pipeline_default_value)
versions = resolve_step_versions(speculative_execution_plan)
default_val_version = join_and_hash("DEFAULTVAL")
input_version = join_and_hash(InputHydration.loader_version +
default_val_version)
solid_def_version = versioned_solid_ext_input.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value.compute"] == step_version
def test_resolve_step_versions_external_dependencies():
run_config = {
"solids": {
"versioned_solid_ext_input": {
"inputs": {
"custom_type": "a"
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input,
run_config=run_config,
)
versions = resolve_step_versions(
speculative_execution_plan,
run_config=run_config,
)
ext_input_version = join_and_hash("a")
input_version = join_and_hash(InputHydration.loader_version +
ext_input_version)
solid1_def_version = versioned_solid_ext_input.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
step1_version = join_and_hash(input_version, solid1_version)
assert versions["versioned_solid_ext_input.compute"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = join_and_hash(output_version)
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input.compute"] == step2_version
def test_step_versions_separate_io_manager():
mgr = io_manager_factory(VersionedInMemoryIOManager())
@solid(version="39", output_defs=[OutputDefinition(io_manager_key="fake")])
def solid_requires_io_manager():
return Output(5)
@pipeline(mode_defs=[
ModeDefinition(
name="fakemode",
resource_defs={
"fake": mgr,
},
),
])
def io_mgr_pipeline():
solid_requires_io_manager()
speculative_execution_plan = create_execution_plan(io_mgr_pipeline,
run_config={},
mode="fakemode")
resolved_run_config = ResolvedRunConfig.build(io_mgr_pipeline,
run_config={},
mode="fakemode")
versions = resolve_step_versions(io_mgr_pipeline,
speculative_execution_plan,
resolved_run_config)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash(*[])
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["solid_requires_io_manager"] == step_version
def create_run_for_pipeline(
self,
pipeline_def,
execution_plan=None,
run_id=None,
run_config=None,
mode=None,
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
solid_selection=None,
):
from dagster.core.execution.api import create_execution_plan
from dagster.core.execution.plan.plan import ExecutionPlan
from dagster.core.snap import snapshot_from_execution_plan
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan)
# note that solids_to_execute is required to execute the solid subset, which is the
# frozenset version of the previous solid_subset.
# solid_selection is not required and will not be converted to solids_to_execute here.
# i.e. this function doesn't handle solid queries.
# solid_selection is only used to pass the user queries further down.
check.opt_set_param(solids_to_execute, "solids_to_execute", of_type=str)
check.opt_list_param(solid_selection, "solid_selection", of_type=str)
if solids_to_execute:
if isinstance(pipeline_def, PipelineSubsetDefinition):
# for the case when pipeline_def is created by IPipeline or ExternalPipeline
check.invariant(
solids_to_execute == pipeline_def.solids_to_execute,
"Cannot create a PipelineRun from pipeline subset {pipeline_solids_to_execute} "
"that conflicts with solids_to_execute arg {solids_to_execute}".format(
pipeline_solids_to_execute=str_format_list(pipeline_def.solids_to_execute),
solids_to_execute=str_format_list(solids_to_execute),
),
)
else:
# for cases when `create_run_for_pipeline` is directly called
pipeline_def = pipeline_def.get_pipeline_subset_def(
solids_to_execute=solids_to_execute
)
if _is_memoized_run(tags):
if step_keys_to_execute:
raise DagsterInvariantViolationError(
"step_keys_to_execute parameter cannot be used in conjunction with memoized "
"pipeline runs."
)
speculative_execution_plan = create_execution_plan(
pipeline_def, run_config=run_config, mode=mode,
)
step_versions = resolve_step_versions(
speculative_execution_plan=speculative_execution_plan,
)
step_keys_to_execute = self.resolve_unmemoized_steps(
speculative_execution_plan, step_versions
) # TODO: tighter integration with existing step_keys_to_execute functionality
if execution_plan is None:
execution_plan = create_execution_plan(
pipeline_def,
run_config=run_config,
mode=mode,
step_keys_to_execute=step_keys_to_execute,
)
return self.create_run(
pipeline_name=pipeline_def.name,
run_id=run_id,
run_config=run_config,
mode=check.opt_str_param(mode, "mode", default=pipeline_def.get_default_mode_name()),
solid_selection=solid_selection,
solids_to_execute=solids_to_execute,
step_keys_to_execute=step_keys_to_execute,
status=status,
tags=tags,
root_run_id=root_run_id,
parent_run_id=parent_run_id,
pipeline_snapshot=pipeline_def.get_pipeline_snapshot(),
execution_plan_snapshot=snapshot_from_execution_plan(
execution_plan, pipeline_def.get_pipeline_snapshot_id()
),
parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(),
)
