def dynamic_id_ranges_for_time(context): """ For the configured partition start/end, searches for the range of ids that were created in that time """ id_range, metadata_entries = _id_range_for_time( context.resources.partition_start, context.resources.partition_end, context.resources.hn_client, ) start_id, end_id = id_range batch_size = context.solid_config.get("batch_size") if batch_size is not None and batch_size > 1: start = start_id while start < end_id: end = start + batch_size end = end if end <= end_id else end_id yield DynamicOutput((start, end), mapping_key=f"{start}_{end}") start += batch_size else: yield DynamicOutput( (start_id, end_id), mapping_key=f"{start_id}_{end_id}", metadata_entries=metadata_entries, )
def should_work(_): yield DynamicOutput(1, output_name="numbers", mapping_key="1") yield DynamicOutput(2, output_name="numbers", mapping_key="2") yield DynamicOutput("a", output_name="letters", mapping_key="a") yield DynamicOutput("b", output_name="letters", mapping_key="b") yield DynamicOutput("c", output_name="letters", mapping_key="c") yield Output("*", "wildcard")
def test_invalid_mapping_keys(): with pytest.raises(DagsterInvalidDefinitionError): DynamicOutput(True, mapping_key="") with pytest.raises(DagsterInvalidDefinitionError): DynamicOutput(True, mapping_key="?") with pytest.raises(DagsterInvalidDefinitionError): DynamicOutput(True, mapping_key="foo.baz")
def spawn(): for i in range(10): yield DynamicOutput(DangerNoodle(i), output_name="items", mapping_key=f"num_{i}") gc.collect() yield Output(len(objgraph.by_type("DangerNoodle")), output_name="refs")
def the_op(context): context.add_output_metadata({"one": "one"}, output_name="out1", mapping_key="one") yield DynamicOutput(value=1, output_name="out1", mapping_key="one") context.add_output_metadata({"two": "two"}, output_name="out1", mapping_key="two") context.add_output_metadata({"three": "three"}, output_name="out2", mapping_key="three") yield DynamicOutput(value=2, output_name="out1", mapping_key="two") yield DynamicOutput(value=3, output_name="out2", mapping_key="three") context.add_output_metadata({"four": "four"}, output_name="out2", mapping_key="four") yield DynamicOutput(value=4, output_name="out2", mapping_key="four")
def solid2(_, _input1): for i in range(4): yield DynamicOutput( 7, mapping_key=str(i), output_name="output2", metadata_entries=[entry2], )
def dynamic_optional_output_op(context): for i in range(10): if (context.pipeline_run.parent_run_id and i % 2 == 0 # re-execution run skipped odd numbers or not context.pipeline_run.parent_run_id and i % 2 == 1 # root run skipped even numbers ): yield DynamicOutput(value=i, mapping_key=str(i))
def files_in_directory(context): path = context.op_config["path"] dirname, _, filenames = next(os.walk(path)) for file in filenames: yield DynamicOutput( value=os.path.join(dirname, file), # create a mapping key from the file name mapping_key=file.replace(".", "_").replace("-", "_"), )
def dynamic_add(_, x): yield DynamicOutput(x + 1, mapping_key="1") yield DynamicOutput(x + 2, mapping_key="2")
def dynamic_solid(): yield DynamicOutput(1, mapping_key="1") yield DynamicOutput(2, mapping_key="2")
def should_not_work(): asyncio.sleep(0.01) return DynamicOutput(value=1, mapping_key="1", output_name="a")
def should_not_work(): return DynamicOutput(value=1, mapping_key="1", output_name="a")
def two_outs(): yield DynamicOutput(1, "a") yield DynamicOutput(2, "b")
def emit_dyn(vector): for i in vector: yield DynamicOutput(value=i, mapping_key=f"input_{i}")
def numbers(): for i in range(3): yield DynamicOutput(i, mapping_key=str(i))
def dynamic_echo(_, nums): for x in nums: yield DynamicOutput(value=x, mapping_key=str(x))
def dynamic_solid(_): yield DynamicOutput(1, mapping_key="mapping_1") yield DynamicOutput(2, mapping_key="mapping_2")
def emit(num: int = 2): for i in range(num): yield DynamicOutput(value=i, mapping_key=str(i))
def dynamic_op(): for i in range(10): yield DynamicOutput(value=i, mapping_key=str(i))
def the_op(): yield DynamicOutput(1, mapping_key="one", metadata={"one": "blah"}) yield DynamicOutput(2, mapping_key="two", metadata={"two": "blah"})
def dynamic_echo(x): yield DynamicOutput(x, mapping_key="echo")
def should_fail(_): yield DynamicOutput(True, mapping_key="dunk") yield DynamicOutput(True, mapping_key="dunk")
def dyn_desc(_) -> Iterator[DynamicOutput]: """ Returns: numbers """ yield DynamicOutput(4, "4")
def my_dynamic(): yield DynamicOutput(value=1, mapping_key="1", output_name="a") yield DynamicOutput(value=2, mapping_key="2", output_name="a") yield Output(value="foo", output_name="b")
def dynamic_outs(): for i in range(0, 3): yield DynamicOutput(value=i, mapping_key=f"num_{i}")
async def aio_gen(): yield DynamicOutput(value=1, mapping_key="1", output_name="a") yield DynamicOutput(value=2, mapping_key="2", output_name="a") await asyncio.sleep(0.01) yield Output(value="foo", output_name="b")
def emit(context, num: int = 3): if context.solid_config["fail"]: raise Exception("FAILURE") for i in range(num): yield DynamicOutput(value=i, mapping_key=str(i))
def emit(_context, range_input): for i in range(range_input): yield DynamicOutput(value=i, mapping_key=str(i))
def emit(_): for i in range(3): yield DynamicOutput(value=i, mapping_key=str(i))
def the_op(context): yield DynamicOutput(1, mapping_key="one") context.add_output_metadata({"foo": "bar"}, mapping_key="one")