class TestConstantResult:
def test_instantiates_with_value(self):
constant_result = ConstantResult(value=5)
assert constant_result.value == 5
constant_result = ConstantResult(value=10)
assert constant_result.value == 10
def test_read_returns_self(self):
constant_result = ConstantResult(value="hello world")
assert constant_result.read("this param isn't used") is constant_result
def test_write_raises(self):
constant_result = ConstantResult(value="untouchable!")
with pytest.raises(ValueError):
constant_result.write("nvm")
with pytest.raises(ValueError):
constant_result.write("untouchable!")
def test_handles_none_as_constant(self):
constant_result = ConstantResult(value=None)
assert constant_result.read("still not used") is constant_result
@pytest.mark.parametrize(
"constant_value", [3, "text", 5.0, Constant(3), Constant("text"), Constant(5.0)]
)
def test_exists(self, constant_value: Union[str, Constant]):
result = ConstantResult(value=constant_value)
result_exists = result.exists("")
assert result_exists is True
def as_task(x: Any, flow: "Optional[Flow]" = None) -> "prefect.Task":
"""
Wraps a function, collection, or constant with the appropriate Task type. If a constant
or collection of constants is passed, a `Constant` task is returned.
Args:
- x (object): any Python object to convert to a prefect Task
- flow (Flow, optional): Flow to which the prefect Task will be bound
Returns:
- a prefect Task representing the passed object
"""
from prefect.tasks.core.constants import Constant
def is_constant(x: Any) -> bool:
"""
Helper function for determining if nested collections are constants without calling
`bind()`, which would create new tasks on the active graph.
"""
if isinstance(x, (prefect.core.Task, unmapped)):
return False
elif isinstance(x, (list, tuple, set)):
return all(is_constant(xi) for xi in x)
elif isinstance(x, dict):
return all(is_constant(xi) for xi in x.values())
return True
# task objects
if isinstance(x, prefect.core.Task): # type: ignore
return x
elif isinstance(x, unmapped):
return x.task
# handle constants, including collections of constants
elif is_constant(x):
return_task = Constant(x) # type: prefect.core.Task
# collections
elif isinstance(x, list):
return_task = prefect.tasks.core.collections.List().bind(*x, flow=flow)
elif isinstance(x, tuple):
return_task = prefect.tasks.core.collections.Tuple().bind(*x,
flow=flow)
elif isinstance(x, set):
return_task = prefect.tasks.core.collections.Set().bind(*x, flow=flow)
elif isinstance(x, dict):
keys, values = [], []
for k, v in x.items():
keys.append(k)
values.append(v)
return_task = prefect.tasks.core.collections.Dict().bind(keys=keys,
values=values,
flow=flow)
else:
return x
return_task.auto_generated = True # type: ignore
return return_task
def test_mapping_over_one_unmappable_input(executor):
with Flow(name="test") as f:
a = AddTask().map(x=Constant(1), y=Constant([1]))
s = f.run(executor=executor)
assert s.is_failed()
assert s.result[a].is_failed()
assert (s.result[a].message ==
"At least one upstream state has an unmappable result.")
def test_merge_can_distinguish_between_a_none_result_and_an_unrun_task():
condition = Condition()
true_branch = Constant(None)
false_branch = Constant(0)
with Flow(name="test") as flow:
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result is None
def test_merge_with_list():
with Flow(name="test") as flow:
condition = Condition()
true_branch = prefect.utilities.tasks.as_task([Constant(1), Constant(2)])
false_branch = Constant(0)
with pytest.warns(prefect.utilities.exceptions.PrefectWarning):
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result == [1, 2]
def test_tasks_have_all_non_unmapped_constant_args_as_transitive_upstream_deps(
self,
):
def func(a, b, c, d):
m = inc.copy(name="m").bind(1)
n = inc.copy(name="n").bind(a)
o = inc.copy(name="o").bind(b)
p = add.copy(name="p").bind(n, o)
q = add.copy(name="q").bind(c, d)
r = add.copy(name="r").bind(q, m)
return m, n, o, p, q, r
with Flow("test") as flow:
a = ranged.copy(name="a").bind(3)
b = inc.copy(name="b").bind(1)
c = Constant(1, name="c")
d = Constant(range(3), name="d")
m, n, o, p, q, r = apply_map(
func, a, edges.unmapped(b), c=edges.unmapped(c), d=d
)
def edge_info(task):
"""Returns a map of {upstream: (is_data_dep, is_mapped)}"""
return {
e.upstream_task: (e.key is not None, e.mapped)
for e in flow.edges_to(task)
}
assert edge_info(m) == {a: (False, True), b: (False, False), d: (False, True)}
assert edge_info(n) == {a: (True, True), b: (False, False), d: (False, True)}
assert edge_info(o) == {a: (False, True), b: (True, False), d: (False, True)}
assert edge_info(p) == {n: (True, True), o: (True, True)}
assert edge_info(q) == {a: (False, True), b: (False, False), d: (True, True)}
assert edge_info(r) == {q: (True, True), m: (True, True)}
state = flow.run()
res = {t: state.result[t].result for t in [m, n, o, p, q, r]}
sol = {
m: [2, 2, 2],
n: [1, 2, 3],
o: [3, 3, 3],
p: [4, 5, 6],
q: [1, 2, 3],
r: [3, 4, 5],
}
assert res == sol
class TestConstantResult:
def test_instantiates_with_value(self):
constant_result = ConstantResult(5)
assert constant_result.value == 5
constant_result = ConstantResult(value=10)
assert constant_result.value == 10
def test_read_returns_value(self):
constant_result = ConstantResult("hello world")
assert constant_result.read("this param isn't used") == "hello world"
def test_write_doesnt_overwrite_value(self):
constant_result = ConstantResult("untouchable!")
constant_result.write()
assert constant_result.value == "untouchable!"
assert constant_result.read("still unused") == "untouchable!"
def test_write_returns_value(self):
constant_result = ConstantResult("constant value")
output = constant_result.write()
assert output == "'constant value'"
def test_handles_none_as_constant(self):
constant_result = ConstantResult(None)
assert constant_result.read("still not used") is None
output = constant_result.write()
assert output == "None"
@pytest.mark.parametrize(
"constant_value",
[3, "text", 5.0,
Constant(3),
Constant("text"),
Constant(5.0)])
def test_exists(self, constant_value: Union[str, Constant]):
result = ConstantResult(constant_value)
result_exists = result.exists()
assert result_exists is True
def test_merge_diamond_flow_with_results():
condition = Condition()
true_branch = Constant(1)
false_branch = Constant(0)
with Flow(name="test") as flow:
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result == 1
with prefect.context(CONDITION=False):
state = flow.run()
assert state.result[merge_task].result == 0
with prefect.context(CONDITION=None):
state = flow.run()
assert state.result[merge_task].result is None
def get_task_kwargs(op, ref, maps):
new_kwargs = {}
for k, v in op.get('kwargs', {}).items():
if isinstance(v, str) and v.startswith(':'):
v = ref[v[1:]]
else:
v = Constant(v)
if k not in maps:
v = unmapped(v)
new_kwargs[k] = v
return new_kwargs
def test_constant_task_returns_its_value(self):
x = Constant("x")
assert x.run() == "x"
y = Constant(100)
assert y.run() == 100
# there's also using the calendar.
# View Python command with `ps`
# Rename `ps` command: https://stackoverflow.com/a/49097964/2138773
# https://github.com/dvarrazzo/py-setproctitle
from setproctitle import setproctitle
setproctitle('prefect: ds arxiv')
# using the Imperitive API: https://docs.prefect.io/core/concepts/flows.html#imperative-api
with Flow('Build Arxiv', state_handlers=[slack_handler]) as flow:
# Dates
date_today = datetime.now().strftime('%Y-%m-%d')
# Begin the flow. Will fail if len(df) = 0
# FIXME: date_query is actually overwritten within the function to just be the max date. See function for details.
df_full = df_get_arxiv(Constant(arx_list), Constant(arx_dict))
filter_to_date = determine_filter_date(df_full)
df = filter_df_arxiv(df=df_full, filter_to_date=filter_to_date)
# Creating the Post folder, save the dataframe there, and build the rmd
dir_post = create_dir_post(date_published=date_today)
dir_post.set_dependencies(upstream_tasks=[df])
written_df = write_df_to_csv(df=df, dir_post=dir_post)
fp_post = copy_rmd_template(dir_post)
knit = knit_rmd_to_html(fp_post=fp_post)
knit.set_dependencies(upstream_tasks=[written_df])
# once knit, re-build with tweet. Requires set_dependencies to avoid conflict
replace = replace_rmd_template_metadata(dir_post=dir_post,
fp_post=fp_post,
date_today=date_today,
