def test_memoized_plan_root_input_manager(): @root_input_manager(version="foo") def my_input_manager(): return 5 @solid(input_defs=[InputDefinition("x", root_manager_key="my_input_manager")], version="foo") def my_solid_takes_input(x): return x @pipeline( mode_defs=[ ModeDefinition( resource_defs={ "io_manager": IOManagerDefinition.hardcoded_io_manager( VersionedInMemoryIOManager() ), "my_input_manager": my_input_manager, }, ), ], tags={MEMOIZED_RUN_TAG: "true"}, ) def my_pipeline(): my_solid_takes_input() with instance_for_test() as instance: plan = create_execution_plan(my_pipeline, instance_ref=instance.get_ref()) assert ( plan.get_version_for_step_output_handle( StepOutputHandle("my_solid_takes_input", "result") ) is not None )
def test_memoized_plan_default_input_val(): @solid( version="42", input_defs=[InputDefinition("_my_input", String, default_value="DEFAULTVAL")], ) def solid_default_input(_my_input): pass @pipeline( mode_defs=[ ModeDefinition( resource_defs={ "io_manager": IOManagerDefinition.hardcoded_io_manager( VersionedInMemoryIOManager() ), }, ) ], tags={MEMOIZED_RUN_TAG: "true"}, ) def pipeline_default_value(): solid_default_input() # Ensure that we can build a valid plan with a default input value. with instance_for_test() as instance: unmemoized_plan = create_execution_plan( pipeline_default_value, instance_ref=instance.get_ref() ) assert unmemoized_plan.step_keys_to_execute == ["solid_default_input"]
def test_memoized_plan_root_input_manager_input_config(): @root_input_manager(version="foo", input_config_schema={"my_str": str}) def my_input_manager(): return 5 @solid( input_defs=[InputDefinition("x", root_manager_key="my_input_manager")], version="foo") def my_solid_takes_input(x): return x @pipeline( mode_defs=[ ModeDefinition(resource_defs={ "io_manager": IOManagerDefinition.hardcoded_io_manager( VersionedInMemoryIOManager()), "my_input_manager": my_input_manager, }, ), ], tags={MEMOIZED_RUN_TAG: "true"}, ) def my_pipeline(): my_solid_takes_input() input_config = {"my_str": "foo"} run_config = { "solids": { "my_solid_takes_input": { "inputs": { "x": input_config } } } } with instance_for_test() as instance: plan = create_execution_plan( my_pipeline, instance_ref=instance.get_ref(), run_config=run_config, ) output_version = plan.get_version_for_step_output_handle( StepOutputHandle("my_solid_takes_input", "result")) assert output_version is not None input_config["my_str"] = "bar" plan = create_execution_plan( my_pipeline, instance_ref=instance.get_ref(), run_config=run_config, ) new_output_version = plan.get_version_for_step_output_handle( StepOutputHandle("my_solid_takes_input", "result")) # Ensure that after changing input config, the version changes. assert not new_output_version == output_version
def run_test_with_builtin_type(type_to_test, type_values): first_type_val, second_type_val = type_values manager = VersionedInMemoryIOManager() @solid(version="42", input_defs=[InputDefinition("_builtin_type", type_to_test)]) def solid_ext_input(_builtin_type): pass @pipeline( mode_defs=[ ModeDefinition(resource_defs={ "io_manager": IOManagerDefinition.hardcoded_io_manager(manager), }, ) ], tags={MEMOIZED_RUN_TAG: "true"}, ) def my_pipeline(): versioned_solid_takes_input(solid_ext_input()) input_config = {"_builtin_type": first_type_val} run_config = {"solids": {"solid_ext_input": {"inputs": input_config}}} with instance_for_test() as instance: unmemoized_plan = create_execution_plan( my_pipeline, run_config=run_config, instance_ref=instance.get_ref(), ) assert len(unmemoized_plan.step_keys_to_execute) == 2 step_output_handle = StepOutputHandle("solid_ext_input", "result") version = unmemoized_plan.get_version_for_step_output_handle( step_output_handle) manager.values[step_output_handle.step_key, step_output_handle.output_name, version] = 5 memoized_plan = create_execution_plan( my_pipeline, run_config=run_config, instance_ref=instance.get_ref(), ) assert memoized_plan.step_keys_to_execute == [ "versioned_solid_takes_input" ] input_config["_builtin_type"] = second_type_val unmemoized_plan = create_execution_plan( my_pipeline, run_config=run_config, instance_ref=instance.get_ref(), ) assert len(unmemoized_plan.step_keys_to_execute) == 2
def run_test_with_builtin_type(type_to_test, loader_version, type_value): @solid(version="42", input_defs=[InputDefinition("_builtin_type", type_to_test)]) def versioned_solid_ext_input_builtin_type(_, _builtin_type): pass @pipeline def versioned_pipeline_ext_input_builtin_type(): versioned_solid_takes_input(versioned_solid_ext_input_builtin_type()) run_config = { "solids": { "versioned_solid_ext_input_builtin_type": { "inputs": { "_builtin_type": type_value } } } } speculative_execution_plan = create_execution_plan( versioned_pipeline_ext_input_builtin_type, run_config=run_config, ) resolved_run_config = ResolvedRunConfig.build( versioned_pipeline_ext_input_builtin_type, run_config=run_config) versions = resolve_step_versions(versioned_pipeline_ext_input_builtin_type, speculative_execution_plan, resolved_run_config) ext_input_version = join_and_hash(str(type_value)) input_version = join_and_hash(loader_version + ext_input_version) solid1_def_version = versioned_solid_ext_input_builtin_type.version solid1_config_version = resolve_config_version(None) solid1_resources_version = join_and_hash() solid1_version = join_and_hash(solid1_def_version, solid1_config_version, solid1_resources_version) step1_version = join_and_hash(input_version, solid1_version) assert versions["versioned_solid_ext_input_builtin_type"] == step1_version output_version = join_and_hash(step1_version, "result") hashed_input2 = output_version solid2_def_version = versioned_solid_takes_input.version solid2_config_version = resolve_config_version(None) solid2_resources_version = join_and_hash() solid2_version = join_and_hash(solid2_def_version, solid2_config_version, solid2_resources_version) step2_version = join_and_hash(hashed_input2, solid2_version) assert versions["versioned_solid_takes_input"] == step2_version
return join_and_hash(str(config_value)) @dagster_type_loader(String, loader_version="97", external_version_fn=_get_ext_version) def InputHydration(_, _hello): return "Hello" @usable_as_dagster_type(loader=InputHydration) class CustomType(str): pass @solid(version="42", input_defs=[InputDefinition("custom_type", CustomType)]) def versioned_solid_ext_input(_, custom_type): return custom_type * 4 @pipeline def versioned_pipeline_ext_input(): return versioned_solid_takes_input(versioned_solid_ext_input()) def test_resolve_step_versions_external_dependencies(): run_config = { "solids": { "versioned_solid_ext_input": { "inputs": { "custom_type": "a"
hashed_input2 = output_version solid2_def_version = versioned_solid_takes_input.version solid2_config_version = resolve_config_version(None) solid2_resources_version = join_and_hash() solid2_version = join_and_hash( solid2_def_version, solid2_config_version, solid2_resources_version ) step2_version = join_and_hash(hashed_input2, solid2_version) assert versions["versioned_solid_takes_input"] == step2_version @solid( version="42", input_defs=[InputDefinition("default_input", String, default_value="DEFAULTVAL")], ) def versioned_solid_default_value(_, default_input): return default_input * 4 @pipeline def versioned_pipeline_default_value(): versioned_solid_default_value() def test_resolve_step_versions_default_value(): speculative_execution_plan = create_execution_plan(versioned_pipeline_default_value) versions = speculative_execution_plan.resolve_step_versions() input_version = join_and_hash(repr("DEFAULTVAL"))
r: requests.Response = requests.get(url) # Checks whether the response is OK if (r.ok): # Returns JSON parsed response return r.json() # If response is not OK, build a message and both log it # and raise it as an exception msg: str = f"Failed to fetch URL {url}: status_code {r.status_code}, message \"{r.text}\"" context.log.error(msg) raise Exception(msg) @solid(config_schema={"data_key": str}, input_defs=[InputDefinition(name="data", dagster_type=DictOrList)]) def generate_dataframe(context: dagster.SolidExecutionContext, data): """ Builds a Pandas dataframe from data dictionary, using "data_key" for extracting a list of items Configuration schema: - data_key (str): the key of the dictionary which contains the data (e.g on "{"asd": [{"a": 1}, {"b": 2}]}", "asd" is the data_key) """ # Parses key from solid configuration key: str = context.solid_config["data_key"] # If key is an empty string, use data as is if key == "": extracted_data: list = data
time.sleep(0.01) run_stats = instance.get_run_stats(runs[i].run_id) if run_stats is None: i += 1 next_tick += tick_delay continue launch_time = run_stats.launch_time if launch_time is None: i += 1 next_tick += tick_delay continue launch_time = convert_unix_time_to_datetime(launch_time) if launch_time <= compare_timestamp: instance.delete_run(runs[i].run_id) deleted_runs += 1 next_tick += tick_delay i += 1 context.log.info(f"Deleted {deleted_runs} out of {total_runs} runs") @solid(input_defs=[ InputDefinition("seconds", int, default_value=0), InputDefinition("minutes", int, default_value=0), InputDefinition("hours", int, default_value=0), InputDefinition("days", int, default_value=0), ]) def get_compare_timestamp(context: SolidExecutionContext, seconds, minutes, hours, days) -> datetime: return datetime.now() - timedelta( seconds=seconds, minutes=minutes, hours=hours, days=days)
def _get_ext_version(config_value): return join_and_hash([str(config_value)]) @dagster_type_loader(String, loader_version="97", external_version_fn=_get_ext_version) def InputHydration(_, _hello): return "Hello" @usable_as_dagster_type(loader=InputHydration) class CustomType(str): pass @solid(version="42", input_defs=[InputDefinition("custom_type", CustomType)]) def versioned_solid_ext_input(_, custom_type): return custom_type * 4 @pipeline def versioned_pipeline_ext_input(): return versioned_solid_takes_input(versioned_solid_ext_input()) def test_resolve_step_versions_external_dependencies(): speculative_execution_plan = create_execution_plan( versioned_pipeline_ext_input, run_config={"solids": {"versioned_solid_ext_input": {"inputs": {"custom_type": "a"}}}}, )