def test_step_versions_with_resources():
run_config = {"resources": {"test_resource": {"config": {"input_str": "apple"}}}}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config,
)
versions = resolve_step_versions_for_test(
speculative_execution_plan, run_config=run_config, mode="fakemode"
)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
environment_config = EnvironmentConfig.build(
versioned_modes_pipeline, mode="fakemode", run_config=run_config
)
resource_versions_by_key = resolve_resource_versions(
environment_config, versioned_modes_pipeline.get_mode_definition("fakemode")
)
solid_resources_version = join_and_hash(
*[
resource_versions_by_key[resource_key]
for resource_key in fake_solid_resources_versioned.required_resource_keys
]
)
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned.compute"] == step_version
def versioned_pipeline_expected_step1_version():
solid1_def_version = versioned_solid_no_input.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
return join_and_hash(solid1_version)
def test_resource_versions():
run_config = {
"resources": {
"test_resource": {
"config": {"input_str": "apple"},
},
"test_resource_no_version": {"config": {"input_str": "banana"}},
}
}
execution_plan = create_execution_plan(modes_pipeline, run_config=run_config, mode="fakemode")
resource_versions_by_key = resolve_resource_versions(
execution_plan.environment_config, execution_plan.pipeline.get_definition()
)
assert resource_versions_by_key["test_resource"] == join_and_hash(
resolve_config_version({"input_str": "apple"}), test_resource.version
)
assert resource_versions_by_key["test_resource_no_version"] == None
assert resource_versions_by_key["test_resource_no_config"] == join_and_hash(
join_and_hash(), "42"
)
def test_step_versions_with_resources():
run_config = {"resources": {"test_resource": {"config": {"input_str": "apple"}}}}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config, mode="fakemode"
)
versions = speculative_execution_plan.resolve_step_versions()
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
resource_versions_by_key = resolve_resource_versions(
speculative_execution_plan.environment_config,
speculative_execution_plan.pipeline.get_definition(),
)
solid_resources_version = join_and_hash(
*[
resource_versions_by_key[resource_key]
for resource_key in fake_solid_resources_versioned.required_resource_keys
]
)
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned"] == step_version
def basic_resource_versions():
run_config = {
"resources": {
"basic_resource": {
"config": {
"input_str": "apple"
},
},
"resource_no_version": {
"config": {
"input_str": "banana"
}
},
}
}
resolved_run_config = ResolvedRunConfig.build(modes_pipeline,
run_config,
mode="fakemode")
resource_versions_by_key = resolve_resource_versions(
resolved_run_config, modes_pipeline)
assert resource_versions_by_key["basic_resource"] == join_and_hash(
resolve_config_version({"input_str": "apple"}), basic_resource.version)
assert resource_versions_by_key["resource_no_version"] == None
assert resource_versions_by_key["resource_no_config"] == join_and_hash(
join_and_hash(), "42")
def test_resolve_step_versions_external_dependencies():
run_config = {"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input, run_config=run_config,
)
versions = resolve_step_versions_for_test(speculative_execution_plan, run_config=run_config)
ext_input_version = join_and_hash("a")
input_version = join_and_hash(InputHydration.loader_version + ext_input_version)
solid1_def_version = versioned_solid_ext_input.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(
solid1_def_version, solid1_config_version, solid1_resources_version
)
step1_version = join_and_hash(input_version, solid1_version)
assert versions["versioned_solid_ext_input.compute"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = join_and_hash(output_version)
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(
solid2_def_version, solid2_config_version, solid2_resources_version
)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input.compute"] == step2_version
def test_resource_versions():
run_config = {
"resources": {
"test_resource": {
"config": {
"input_str": "apple"
},
},
"test_resource_no_version": {
"config": {
"input_str": "banana"
}
},
}
}
environment_config = EnvironmentConfig.build(
modes_pipeline,
run_config=run_config,
mode="fakemode",
)
resource_versions_by_key = resolve_resource_versions(
environment_config, modes_pipeline.get_mode_definition("fakemode"))
assert resource_versions_by_key["test_resource"] == join_and_hash(
resolve_config_version({"config": {
"input_str": "apple"
}}), test_resource.version)
assert resource_versions_by_key["test_resource_no_version"] == None
assert resource_versions_by_key[
"test_resource_no_config"] == join_and_hash(join_and_hash(), "42")
def test_join_and_hash():
assert join_and_hash("foo") == hashlib.sha1("foo".encode("utf-8")).hexdigest()
assert join_and_hash("foo", None, "bar") == None
assert join_and_hash("foo", "bar") == hashlib.sha1("barfoo".encode("utf-8")).hexdigest()
assert join_and_hash("foo", "bar", "zab") == join_and_hash("zab", "bar", "foo")
def test_join_and_hash():
assert join_and_hash("foo") == hashlib.sha1(b"foo").hexdigest()
assert join_and_hash("foo", None, "bar") == None
assert join_and_hash("foo", "bar") == hashlib.sha1(b"barfoo").hexdigest()
assert join_and_hash("foo", "bar", "zab") == join_and_hash("zab", "bar", "foo")
def versioned_pipeline_expected_step2_version():
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step1_outputs_hash = versioned_pipeline_expected_step1_output_version()
step2_version = join_and_hash(step1_outputs_hash, solid2_version)
return step2_version
def test_resolve_config_version():
assert resolve_config_version(None) == join_and_hash()
assert resolve_config_version({}) == join_and_hash()
assert resolve_config_version({
"a": "b",
"c": "d"
}) == join_and_hash("a" + join_and_hash("b"), "c" + join_and_hash("d"))
assert resolve_config_version({
"a": "b",
"c": "d"
}) == resolve_config_version({
"c": "d",
"a": "b"
})
assert resolve_config_version({
"a": {
"b": "c"
},
"d": "e"
}) == join_and_hash("a" + join_and_hash("b" + join_and_hash("c")),
"d" + join_and_hash("e"))
def test_resolve_step_versions_no_external_dependencies():
speculative_execution_plan = create_execution_plan(versioned_pipeline)
versions = resolve_step_versions(speculative_execution_plan)
solid1_hash = join_and_hash([versioned_solid_no_input.version])
output1_hash = join_and_hash([solid1_hash + "result"])
outputs_hash = join_and_hash([output1_hash])
solid2_hash = join_and_hash([outputs_hash, versioned_solid_takes_input.version])
assert versions["versioned_solid_no_input.compute"] == solid1_hash
assert versions["versioned_solid_takes_input.compute"] == solid2_hash
def run_test_with_builtin_type(type_to_test, loader_version, type_value):
@solid(version="42",
input_defs=[InputDefinition("_builtin_type", type_to_test)])
def versioned_solid_ext_input_builtin_type(_, _builtin_type):
pass
@pipeline
def versioned_pipeline_ext_input_builtin_type():
return versioned_solid_takes_input(
versioned_solid_ext_input_builtin_type())
run_config = {
"solids": {
"versioned_solid_ext_input_builtin_type": {
"inputs": {
"_builtin_type": type_value
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input_builtin_type,
run_config=run_config,
)
versions = resolve_step_versions_for_test(speculative_execution_plan,
run_config=run_config)
ext_input_version = join_and_hash(str(type_value))
input_version = join_and_hash(loader_version + ext_input_version)
solid1_def_version = versioned_solid_ext_input_builtin_type.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
step1_version = join_and_hash(input_version, solid1_version)
assert versions[
"versioned_solid_ext_input_builtin_type.compute"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = join_and_hash(output_version)
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input.compute"] == step2_version
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value)
versions = speculative_execution_plan.resolve_step_versions()
input_version = join_and_hash(repr("DEFAULTVAL"))
solid_def_version = versioned_solid_default_value.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value"] == step_version
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value)
versions = resolve_step_versions_for_test(speculative_execution_plan)
default_val_version = join_and_hash("DEFAULTVAL")
input_version = join_and_hash(InputHydration.loader_version + default_val_version)
solid_def_version = versioned_solid_ext_input.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version, solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value.compute"] == step_version
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(
versioned_pipeline_default_value)
versions = resolve_step_versions_for_test(speculative_execution_plan)
default_val_version = join_and_hash("DEFAULTVAL")
input_version = join_and_hash(
"String" + default_val_version
) # Since we use the type name as the loader version for default types.
solid_def_version = versioned_solid_default_value.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash()
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(input_version, solid_version)
assert versions["versioned_solid_default_value.compute"] == step_version
def test_step_versions_with_resources():
run_config = {
"resources": {
"basic_resource": {
"config": {
"input_str": "apple"
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_modes_pipeline, run_config=run_config, mode="fakemode")
resolved_run_config = ResolvedRunConfig.build(versioned_modes_pipeline,
run_config=run_config,
mode="fakemode")
versions = resolve_step_versions(versioned_modes_pipeline,
speculative_execution_plan,
resolved_run_config)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
resolved_run_config = ResolvedRunConfig.build(versioned_modes_pipeline,
run_config,
mode="fakemode")
resource_versions_by_key = resolve_resource_versions(
resolved_run_config,
versioned_modes_pipeline,
)
solid_resources_version = join_and_hash(*[
resource_versions_by_key[resource_key] for resource_key in
fake_solid_resources_versioned.required_resource_keys
])
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["fake_solid_resources_versioned"] == step_version
def test_resolve_config_version():
assert resolve_config_version({}) == join_and_hash([])
assert resolve_config_version({"a": "b", "c": "d"}) == join_and_hash(
["a" + join_and_hash(["b"]), "c" + join_and_hash(["d"])]
)
assert resolve_config_version({"a": "b", "c": "d"}) == resolve_config_version(
{"c": "d", "a": "b"}
)
assert resolve_config_version({"a": {"b": "c"}, "d": "e"}) == join_and_hash(
["a" + join_and_hash(["b" + join_and_hash(["c"])]), "d" + join_and_hash(["e"])]
)
def test_step_versions_separate_io_manager():
mgr = io_manager_factory(VersionedInMemoryIOManager())
@solid(version="39", output_defs=[OutputDefinition(io_manager_key="fake")])
def solid_requires_io_manager():
return Output(5)
@pipeline(mode_defs=[
ModeDefinition(
name="fakemode",
resource_defs={
"fake": mgr,
},
),
])
def io_mgr_pipeline():
solid_requires_io_manager()
speculative_execution_plan = create_execution_plan(io_mgr_pipeline,
run_config={},
mode="fakemode")
resolved_run_config = ResolvedRunConfig.build(io_mgr_pipeline,
run_config={},
mode="fakemode")
versions = resolve_step_versions(io_mgr_pipeline,
speculative_execution_plan,
resolved_run_config)
solid_def_version = fake_solid_resources_versioned.version
solid_config_version = resolve_config_version(None)
solid_resources_version = join_and_hash(*[])
solid_version = join_and_hash(solid_def_version, solid_config_version,
solid_resources_version)
step_version = join_and_hash(solid_version)
assert versions["solid_requires_io_manager"] == step_version
def resolve_unmemoized_steps(self, speculative_execution_plan, step_versions):
unmemoized_steps = []
for step in speculative_execution_plan.topological_steps():
step_version = step_versions[step.key]
output_versions = [
join_and_hash([name, step_version]) for name in step.step_output_dict.keys()
]
addresses = _fake_output_address(
output_versions
) # TODO: replace with real version storage once that has landed.
if any([not version in addresses for version in output_versions]):
unmemoized_steps.append(step.key)
return unmemoized_steps
def test_resolve_step_versions_external_dependencies():
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input,
run_config={"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}},
)
versions = resolve_step_versions(speculative_execution_plan)
ext_input_version = join_and_hash(["a"])
input_version = join_and_hash([InputHydration.loader_version + ext_input_version])
step_version1 = join_and_hash([input_version, versioned_solid_ext_input.version])
assert versions["versioned_solid_ext_input.compute"] == step_version1
output_version = join_and_hash([step_version1, "result"])
hashed_input2 = join_and_hash([output_version])
step_version2 = join_and_hash([hashed_input2, versioned_solid_takes_input.version])
assert versions["versioned_solid_takes_input.compute"] == step_version2
def _get_ext_version(config_value):
return join_and_hash(str(config_value))
def versioned_pipeline_expected_step2_output_version():
step2_version = versioned_pipeline_expected_step2_version()
return join_and_hash(step2_version + "result")
def _external_version_fn(val):
from dagster.core.execution.resolve_versions import join_and_hash
return join_and_hash(str(val), )
