def test_memoized_plan_root_input_manager():
@root_input_manager(version="foo")
def my_input_manager():
return 5
@solid(input_defs=[InputDefinition("x", root_manager_key="my_input_manager")], version="foo")
def my_solid_takes_input(x):
return x
@pipeline(
mode_defs=[
ModeDefinition(
resource_defs={
"io_manager": IOManagerDefinition.hardcoded_io_manager(
VersionedInMemoryIOManager()
),
"my_input_manager": my_input_manager,
},
),
],
tags={MEMOIZED_RUN_TAG: "true"},
)
def my_pipeline():
my_solid_takes_input()
with instance_for_test() as instance:
plan = create_execution_plan(my_pipeline, instance_ref=instance.get_ref())
assert (
plan.get_version_for_step_output_handle(
StepOutputHandle("my_solid_takes_input", "result")
)
is not None
)
def test_memoized_plan_default_input_val():
@solid(
version="42",
input_defs=[InputDefinition("_my_input", String, default_value="DEFAULTVAL")],
)
def solid_default_input(_my_input):
pass
@pipeline(
mode_defs=[
ModeDefinition(
resource_defs={
"io_manager": IOManagerDefinition.hardcoded_io_manager(
VersionedInMemoryIOManager()
),
},
)
],
tags={MEMOIZED_RUN_TAG: "true"},
)
def pipeline_default_value():
solid_default_input()
# Ensure that we can build a valid plan with a default input value.
with instance_for_test() as instance:
unmemoized_plan = create_execution_plan(
pipeline_default_value, instance_ref=instance.get_ref()
)
assert unmemoized_plan.step_keys_to_execute == ["solid_default_input"]
def test_memoized_plan_root_input_manager_input_config():
@root_input_manager(version="foo", input_config_schema={"my_str": str})
def my_input_manager():
return 5
@solid(
input_defs=[InputDefinition("x", root_manager_key="my_input_manager")],
version="foo")
def my_solid_takes_input(x):
return x
@pipeline(
mode_defs=[
ModeDefinition(resource_defs={
"io_manager":
IOManagerDefinition.hardcoded_io_manager(
VersionedInMemoryIOManager()),
"my_input_manager":
my_input_manager,
}, ),
],
tags={MEMOIZED_RUN_TAG: "true"},
)
def my_pipeline():
my_solid_takes_input()
input_config = {"my_str": "foo"}
run_config = {
"solids": {
"my_solid_takes_input": {
"inputs": {
"x": input_config
}
}
}
}
with instance_for_test() as instance:
plan = create_execution_plan(
my_pipeline,
instance_ref=instance.get_ref(),
run_config=run_config,
)
output_version = plan.get_version_for_step_output_handle(
StepOutputHandle("my_solid_takes_input", "result"))
assert output_version is not None
input_config["my_str"] = "bar"
plan = create_execution_plan(
my_pipeline,
instance_ref=instance.get_ref(),
run_config=run_config,
)
new_output_version = plan.get_version_for_step_output_handle(
StepOutputHandle("my_solid_takes_input", "result"))
# Ensure that after changing input config, the version changes.
assert not new_output_version == output_version
def run_test_with_builtin_type(type_to_test, type_values):
first_type_val, second_type_val = type_values
manager = VersionedInMemoryIOManager()
@solid(version="42",
input_defs=[InputDefinition("_builtin_type", type_to_test)])
def solid_ext_input(_builtin_type):
pass
@pipeline(
mode_defs=[
ModeDefinition(resource_defs={
"io_manager":
IOManagerDefinition.hardcoded_io_manager(manager),
}, )
],
tags={MEMOIZED_RUN_TAG: "true"},
)
def my_pipeline():
versioned_solid_takes_input(solid_ext_input())
input_config = {"_builtin_type": first_type_val}
run_config = {"solids": {"solid_ext_input": {"inputs": input_config}}}
with instance_for_test() as instance:
unmemoized_plan = create_execution_plan(
my_pipeline,
run_config=run_config,
instance_ref=instance.get_ref(),
)
assert len(unmemoized_plan.step_keys_to_execute) == 2
step_output_handle = StepOutputHandle("solid_ext_input", "result")
version = unmemoized_plan.get_version_for_step_output_handle(
step_output_handle)
manager.values[step_output_handle.step_key,
step_output_handle.output_name, version] = 5
memoized_plan = create_execution_plan(
my_pipeline,
run_config=run_config,
instance_ref=instance.get_ref(),
)
assert memoized_plan.step_keys_to_execute == [
"versioned_solid_takes_input"
]
input_config["_builtin_type"] = second_type_val
unmemoized_plan = create_execution_plan(
my_pipeline,
run_config=run_config,
instance_ref=instance.get_ref(),
)
assert len(unmemoized_plan.step_keys_to_execute) == 2
def run_test_with_builtin_type(type_to_test, loader_version, type_value):
@solid(version="42",
input_defs=[InputDefinition("_builtin_type", type_to_test)])
def versioned_solid_ext_input_builtin_type(_, _builtin_type):
pass
@pipeline
def versioned_pipeline_ext_input_builtin_type():
versioned_solid_takes_input(versioned_solid_ext_input_builtin_type())
run_config = {
"solids": {
"versioned_solid_ext_input_builtin_type": {
"inputs": {
"_builtin_type": type_value
}
}
}
}
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input_builtin_type,
run_config=run_config,
)
resolved_run_config = ResolvedRunConfig.build(
versioned_pipeline_ext_input_builtin_type, run_config=run_config)
versions = resolve_step_versions(versioned_pipeline_ext_input_builtin_type,
speculative_execution_plan,
resolved_run_config)
ext_input_version = join_and_hash(str(type_value))
input_version = join_and_hash(loader_version + ext_input_version)
solid1_def_version = versioned_solid_ext_input_builtin_type.version
solid1_config_version = resolve_config_version(None)
solid1_resources_version = join_and_hash()
solid1_version = join_and_hash(solid1_def_version, solid1_config_version,
solid1_resources_version)
step1_version = join_and_hash(input_version, solid1_version)
assert versions["versioned_solid_ext_input_builtin_type"] == step1_version
output_version = join_and_hash(step1_version, "result")
hashed_input2 = output_version
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(solid2_def_version, solid2_config_version,
solid2_resources_version)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input"] == step2_version
return join_and_hash(str(config_value))
@dagster_type_loader(String,
loader_version="97",
external_version_fn=_get_ext_version)
def InputHydration(_, _hello):
return "Hello"
@usable_as_dagster_type(loader=InputHydration)
class CustomType(str):
pass
@solid(version="42", input_defs=[InputDefinition("custom_type", CustomType)])
def versioned_solid_ext_input(_, custom_type):
return custom_type * 4
@pipeline
def versioned_pipeline_ext_input():
return versioned_solid_takes_input(versioned_solid_ext_input())
def test_resolve_step_versions_external_dependencies():
run_config = {
"solids": {
"versioned_solid_ext_input": {
"inputs": {
"custom_type": "a"
hashed_input2 = output_version
solid2_def_version = versioned_solid_takes_input.version
solid2_config_version = resolve_config_version(None)
solid2_resources_version = join_and_hash()
solid2_version = join_and_hash(
solid2_def_version, solid2_config_version, solid2_resources_version
)
step2_version = join_and_hash(hashed_input2, solid2_version)
assert versions["versioned_solid_takes_input"] == step2_version
@solid(
version="42",
input_defs=[InputDefinition("default_input", String, default_value="DEFAULTVAL")],
)
def versioned_solid_default_value(_, default_input):
return default_input * 4
@pipeline
def versioned_pipeline_default_value():
versioned_solid_default_value()
def test_resolve_step_versions_default_value():
speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value)
versions = speculative_execution_plan.resolve_step_versions()
input_version = join_and_hash(repr("DEFAULTVAL"))
r: requests.Response = requests.get(url)
# Checks whether the response is OK
if (r.ok):
# Returns JSON parsed response
return r.json()
# If response is not OK, build a message and both log it
# and raise it as an exception
msg: str = f"Failed to fetch URL {url}: status_code {r.status_code}, message \"{r.text}\""
context.log.error(msg)
raise Exception(msg)
@solid(config_schema={"data_key": str},
input_defs=[InputDefinition(name="data", dagster_type=DictOrList)])
def generate_dataframe(context: dagster.SolidExecutionContext, data):
""" Builds a Pandas dataframe from data dictionary, using "data_key"
for extracting a list of items
Configuration schema:
- data_key (str): the key of the dictionary which contains the data
(e.g on "{"asd": [{"a": 1}, {"b": 2}]}", "asd" is the data_key)
"""
# Parses key from solid configuration
key: str = context.solid_config["data_key"]
# If key is an empty string, use data as is
if key == "":
extracted_data: list = data
time.sleep(0.01)
run_stats = instance.get_run_stats(runs[i].run_id)
if run_stats is None:
i += 1
next_tick += tick_delay
continue
launch_time = run_stats.launch_time
if launch_time is None:
i += 1
next_tick += tick_delay
continue
launch_time = convert_unix_time_to_datetime(launch_time)
if launch_time <= compare_timestamp:
instance.delete_run(runs[i].run_id)
deleted_runs += 1
next_tick += tick_delay
i += 1
context.log.info(f"Deleted {deleted_runs} out of {total_runs} runs")
@solid(input_defs=[
InputDefinition("seconds", int, default_value=0),
InputDefinition("minutes", int, default_value=0),
InputDefinition("hours", int, default_value=0),
InputDefinition("days", int, default_value=0),
])
def get_compare_timestamp(context: SolidExecutionContext, seconds, minutes,
hours, days) -> datetime:
return datetime.now() - timedelta(
seconds=seconds, minutes=minutes, hours=hours, days=days)
def _get_ext_version(config_value):
return join_and_hash([str(config_value)])
@dagster_type_loader(String, loader_version="97", external_version_fn=_get_ext_version)
def InputHydration(_, _hello):
return "Hello"
@usable_as_dagster_type(loader=InputHydration)
class CustomType(str):
pass
@solid(version="42", input_defs=[InputDefinition("custom_type", CustomType)])
def versioned_solid_ext_input(_, custom_type):
return custom_type * 4
@pipeline
def versioned_pipeline_ext_input():
return versioned_solid_takes_input(versioned_solid_ext_input())
def test_resolve_step_versions_external_dependencies():
speculative_execution_plan = create_execution_plan(
versioned_pipeline_ext_input,
run_config={"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}},
)