def dynamic_id_ranges_for_time(context):
"""
For the configured partition start/end, searches for the range of ids that were created in that time
"""
id_range, metadata_entries = _id_range_for_time(
context.resources.partition_start,
context.resources.partition_end,
context.resources.hn_client,
)
start_id, end_id = id_range
batch_size = context.solid_config.get("batch_size")
if batch_size is not None and batch_size > 1:
start = start_id
while start < end_id:
end = start + batch_size
end = end if end <= end_id else end_id
yield DynamicOutput((start, end), mapping_key=f"{start}_{end}")
start += batch_size
else:
yield DynamicOutput(
(start_id, end_id),
mapping_key=f"{start_id}_{end_id}",
metadata_entries=metadata_entries,
)
def should_work(_):
yield DynamicOutput(1, output_name="numbers", mapping_key="1")
yield DynamicOutput(2, output_name="numbers", mapping_key="2")
yield DynamicOutput("a", output_name="letters", mapping_key="a")
yield DynamicOutput("b", output_name="letters", mapping_key="b")
yield DynamicOutput("c", output_name="letters", mapping_key="c")
yield Output("*", "wildcard")
def test_invalid_mapping_keys():
with pytest.raises(DagsterInvalidDefinitionError):
DynamicOutput(True, mapping_key="")
with pytest.raises(DagsterInvalidDefinitionError):
DynamicOutput(True, mapping_key="?")
with pytest.raises(DagsterInvalidDefinitionError):
DynamicOutput(True, mapping_key="foo.baz")
def spawn():
for i in range(10):
yield DynamicOutput(DangerNoodle(i),
output_name="items",
mapping_key=f"num_{i}")
gc.collect()
yield Output(len(objgraph.by_type("DangerNoodle")), output_name="refs")
def the_op(context):
context.add_output_metadata({"one": "one"},
output_name="out1",
mapping_key="one")
yield DynamicOutput(value=1, output_name="out1", mapping_key="one")
context.add_output_metadata({"two": "two"},
output_name="out1",
mapping_key="two")
context.add_output_metadata({"three": "three"},
output_name="out2",
mapping_key="three")
yield DynamicOutput(value=2, output_name="out1", mapping_key="two")
yield DynamicOutput(value=3, output_name="out2", mapping_key="three")
context.add_output_metadata({"four": "four"},
output_name="out2",
mapping_key="four")
yield DynamicOutput(value=4, output_name="out2", mapping_key="four")
def solid2(_, _input1):
for i in range(4):
yield DynamicOutput(
7,
mapping_key=str(i),
output_name="output2",
metadata_entries=[entry2],
)
def dynamic_optional_output_op(context):
for i in range(10):
if (context.pipeline_run.parent_run_id
and i % 2 == 0 # re-execution run skipped odd numbers
or not context.pipeline_run.parent_run_id
and i % 2 == 1 # root run skipped even numbers
):
yield DynamicOutput(value=i, mapping_key=str(i))
def files_in_directory(context):
path = context.op_config["path"]
dirname, _, filenames = next(os.walk(path))
for file in filenames:
yield DynamicOutput(
value=os.path.join(dirname, file),
# create a mapping key from the file name
mapping_key=file.replace(".", "_").replace("-", "_"),
)
def dynamic_add(_, x):
yield DynamicOutput(x + 1, mapping_key="1")
yield DynamicOutput(x + 2, mapping_key="2")
def dynamic_solid():
yield DynamicOutput(1, mapping_key="1")
yield DynamicOutput(2, mapping_key="2")
def should_not_work():
asyncio.sleep(0.01)
return DynamicOutput(value=1, mapping_key="1", output_name="a")
def should_not_work():
return DynamicOutput(value=1, mapping_key="1", output_name="a")
def two_outs():
yield DynamicOutput(1, "a")
yield DynamicOutput(2, "b")
def emit_dyn(vector):
for i in vector:
yield DynamicOutput(value=i, mapping_key=f"input_{i}")
def numbers():
for i in range(3):
yield DynamicOutput(i, mapping_key=str(i))
def dynamic_echo(_, nums):
for x in nums:
yield DynamicOutput(value=x, mapping_key=str(x))
def dynamic_solid(_):
yield DynamicOutput(1, mapping_key="mapping_1")
yield DynamicOutput(2, mapping_key="mapping_2")
def emit(num: int = 2):
for i in range(num):
yield DynamicOutput(value=i, mapping_key=str(i))
def dynamic_op():
for i in range(10):
yield DynamicOutput(value=i, mapping_key=str(i))
def the_op():
yield DynamicOutput(1, mapping_key="one", metadata={"one": "blah"})
yield DynamicOutput(2, mapping_key="two", metadata={"two": "blah"})
def dynamic_echo(x):
yield DynamicOutput(x, mapping_key="echo")
def should_fail(_):
yield DynamicOutput(True, mapping_key="dunk")
yield DynamicOutput(True, mapping_key="dunk")
def dyn_desc(_) -> Iterator[DynamicOutput]:
"""
Returns:
numbers
"""
yield DynamicOutput(4, "4")
def my_dynamic():
yield DynamicOutput(value=1, mapping_key="1", output_name="a")
yield DynamicOutput(value=2, mapping_key="2", output_name="a")
yield Output(value="foo", output_name="b")
def dynamic_outs():
for i in range(0, 3):
yield DynamicOutput(value=i, mapping_key=f"num_{i}")
async def aio_gen():
yield DynamicOutput(value=1, mapping_key="1", output_name="a")
yield DynamicOutput(value=2, mapping_key="2", output_name="a")
await asyncio.sleep(0.01)
yield Output(value="foo", output_name="b")
def emit(context, num: int = 3):
if context.solid_config["fail"]:
raise Exception("FAILURE")
for i in range(num):
yield DynamicOutput(value=i, mapping_key=str(i))
def emit(_context, range_input):
for i in range(range_input):
yield DynamicOutput(value=i, mapping_key=str(i))
def emit(_):
for i in range(3):
yield DynamicOutput(value=i, mapping_key=str(i))
def the_op(context):
yield DynamicOutput(1, mapping_key="one")
context.add_output_metadata({"foo": "bar"}, mapping_key="one")
