def test_resolve_step_versions_external_dependencies():
run_config = {"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input, run_config=run_config,
)
versions = resolve_step_versions_for_test(speculative_execution_plan, run_config=run_config)
ext_input_version = join_and_hash("a")
input_version = join_and_hash(InputHydration.loader_version + ext_input_version)
solid1_def_version = versioned_solid_ext_input.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(
solid1_def_version, solid1_config_version, solid1_resources_version
)
step1_version = join_and_hash(input_version, solid1_version)
assert versions["versioned_solid_ext_input.compute"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = join_and_hash(output_version)
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(
solid2_def_version, solid2_config_version, solid2_resources_version
)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input.compute"] == step2_version
def run_test_with_builtin_type(type_to_test, loader_version, type_value):
@solid(version="42",
input_defs=[InputDefinition("_builtin_type", type_to_test)])
def versioned_solid_ext_input_builtin_type(_, _builtin_type):
pass
@pipeline
def versioned_pipeline_ext_input_builtin_type():
versioned_solid_takes_input(versioned_solid_ext_input_builtin_type())
run_config = {
"solids": {
"versioned_solid_ext_input_builtin_type": {
"inputs": {
"_builtin_type": type_value
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input_builtin_type,
run_config=run_config,
)
resolved_run_config = ResolvedRunConfig.build(
versioned_pipeline_ext_input_builtin_type, run_config=run_config)
versions = resolve_step_versions(versioned_pipeline_ext_input_builtin_type,
speculative_execution_plan,
resolved_run_config)
ext_input_version = join_and_hash(str(type_value))
input_version = join_and_hash(loader_version + ext_input_version)
solid1_def_version = versioned_solid_ext_input_builtin_type.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
step1_version = join_and_hash(input_version, solid1_version)
assert versions["versioned_solid_ext_input_builtin_type"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = output_version
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input"] == step2_version
def test_resolve_config_version():
assert resolve_config_version({}) == join_and_hash()
assert resolve_config_version({"a": "b", "c": "d"}) == join_and_hash(
"a" + join_and_hash("b"), "c" + join_and_hash("d")
)
assert resolve_config_version({"a": "b", "c": "d"}) == resolve_config_version(
{"c": "d", "a": "b"}
)
assert resolve_config_version({"a": {"b": "c"}, "d": "e"}) == join_and_hash(
"a" + join_and_hash("b" + join_and_hash("c")), "d" + join_and_hash("e")
)
def test_resolve_config_version():
assert resolve_config_version({}) == join_and_hash([])
assert resolve_config_version({"a": "b", "c": "d"}) == join_and_hash(
["a" + join_and_hash(["b"]), "c" + join_and_hash(["d"])]
)
assert resolve_config_version({"a": "b", "c": "d"}) == resolve_config_version(
{"c": "d", "a": "b"}
)
assert resolve_config_version({"a": {"b": "c"}, "d": "e"}) == join_and_hash(
["a" + join_and_hash(["b" + join_and_hash(["c"])]), "d" + join_and_hash(["e"])]
)
def versioned_pipeline_expected_step1_version():
solid1_def_version = versioned_solid_no_input.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
return join_and_hash(solid1_version)
def test_resource_versions():
run_config = {
"resources": {
"test_resource": {
"config": {
"input_str": "apple"
},
},
"test_resource_no_version": {
"config": {
"input_str": "banana"
}
},
}
}
environment_config = EnvironmentConfig.build(
modes_pipeline,
run_config=run_config,
mode="fakemode",
)
resource_versions_by_key = resolve_resource_versions(
environment_config, modes_pipeline.get_mode_definition("fakemode"))
assert resource_versions_by_key["test_resource"] == join_and_hash(
resolve_config_version({"config": {
"input_str": "apple"
}}), test_resource.version)
assert resource_versions_by_key["test_resource_no_version"] == None
assert resource_versions_by_key[
"test_resource_no_config"] == join_and_hash(join_and_hash(), "42")
def test_step_versions_with_resources():
run_config = {"resources": {"test_resource": {"config": {"input_str": "apple"}}}}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config, mode="fakemode"
)
versions = speculative_execution_plan.resolve_step_versions()
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
resource_versions_by_key = resolve_resource_versions(
speculative_execution_plan.environment_config,
speculative_execution_plan.pipeline.get_definition(),
)
solid_resources_version = join_and_hash(
*[
resource_versions_by_key[resource_key]
for resource_key in fake_solid_resources_versioned.required_resource_keys
]
)
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned"] == step_version
def test_resource_versions():
run_config = {
"resources": {
"test_resource": {
"config": {"input_str": "apple"},
},
"test_resource_no_version": {"config": {"input_str": "banana"}},
}
}
execution_plan = create_execution_plan(modes_pipeline, run_config=run_config, mode="fakemode")
resource_versions_by_key = resolve_resource_versions(
execution_plan.environment_config, execution_plan.pipeline.get_definition()
)
assert resource_versions_by_key["test_resource"] == join_and_hash(
resolve_config_version({"input_str": "apple"}), test_resource.version
)
assert resource_versions_by_key["test_resource_no_version"] == None
assert resource_versions_by_key["test_resource_no_config"] == join_and_hash(
join_and_hash(), "42"
)
def test_step_versions_with_resources():
run_config = {"resources": {"test_resource": {"config": {"input_str": "apple"}}}}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config,
)
versions = resolve_step_versions_for_test(
speculative_execution_plan, run_config=run_config, mode="fakemode"
)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
environment_config = EnvironmentConfig.build(
versioned_modes_pipeline, mode="fakemode", run_config=run_config
)
resource_versions_by_key = resolve_resource_versions(
environment_config, versioned_modes_pipeline.get_mode_definition("fakemode")
)
solid_resources_version = join_and_hash(
*[
resource_versions_by_key[resource_key]
for resource_key in fake_solid_resources_versioned.required_resource_keys
]
)
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned.compute"] == step_version
def basic_resource_versions():
run_config = {
"resources": {
"basic_resource": {
"config": {
"input_str": "apple"
},
},
"resource_no_version": {
"config": {
"input_str": "banana"
}
},
}
}
resolved_run_config = ResolvedRunConfig.build(modes_pipeline,
run_config,
mode="fakemode")
resource_versions_by_key = resolve_resource_versions(
resolved_run_config, modes_pipeline)
assert resource_versions_by_key["basic_resource"] == join_and_hash(
resolve_config_version({"input_str": "apple"}), basic_resource.version)
assert resource_versions_by_key["resource_no_version"] == None
assert resource_versions_by_key["resource_no_config"] == join_and_hash(
join_and_hash(), "42")
def versioned_pipeline_expected_step2_version():
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step1_outputs_hash = versioned_pipeline_expected_step1_output_version()
step2_version = join_and_hash(step1_outputs_hash, solid2_version)
return step2_version
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value)
versions = speculative_execution_plan.resolve_step_versions()
input_version = join_and_hash(repr("DEFAULTVAL"))
solid_def_version = versioned_solid_default_value.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value"] == step_version
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value)
versions = resolve_step_versions_for_test(speculative_execution_plan)
default_val_version = join_and_hash("DEFAULTVAL")
input_version = join_and_hash(InputHydration.loader_version + default_val_version)
solid_def_version = versioned_solid_ext_input.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value.compute"] == step_version
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(
versioned_pipeline_default_value)
versions = resolve_step_versions_for_test(speculative_execution_plan)
default_val_version = join_and_hash("DEFAULTVAL")
input_version = join_and_hash(
"String" + default_val_version
) # Since we use the type name as the loader version for default types.
solid_def_version = versioned_solid_default_value.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value.compute"] == step_version
def test_step_versions_with_resources():
run_config = {
"resources": {
"basic_resource": {
"config": {
"input_str": "apple"
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config, mode="fakemode")
resolved_run_config = ResolvedRunConfig.build(versioned_modes_pipeline,
run_config=run_config,
mode="fakemode")
versions = resolve_step_versions(versioned_modes_pipeline,
speculative_execution_plan,
resolved_run_config)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
resolved_run_config = ResolvedRunConfig.build(versioned_modes_pipeline,
run_config,
mode="fakemode")
resource_versions_by_key = resolve_resource_versions(
resolved_run_config,
versioned_modes_pipeline,
)
solid_resources_version = join_and_hash(*[
resource_versions_by_key[resource_key] for resource_key in
fake_solid_resources_versioned.required_resource_keys
])
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned"] == step_version
def test_step_versions_separate_io_manager():
mgr = io_manager_factory(VersionedInMemoryIOManager())
@solid(version="39", output_defs=[OutputDefinition(io_manager_key="fake")])
def solid_requires_io_manager():
return Output(5)
@pipeline(mode_defs=[
ModeDefinition(
name="fakemode",
resource_defs={
"fake": mgr,
},
),
])
def io_mgr_pipeline():
solid_requires_io_manager()
speculative_execution_plan = create_execution_plan(io_mgr_pipeline,
run_config={},
mode="fakemode")
resolved_run_config = ResolvedRunConfig.build(io_mgr_pipeline,
run_config={},
mode="fakemode")
versions = resolve_step_versions(io_mgr_pipeline,
speculative_execution_plan,
resolved_run_config)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash(*[])
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["solid_requires_io_manager"] == step_version