class TestConstantResult: def test_instantiates_with_value(self): constant_result = ConstantResult(value=5) assert constant_result.value == 5 constant_result = ConstantResult(value=10) assert constant_result.value == 10 def test_read_returns_self(self): constant_result = ConstantResult(value="hello world") assert constant_result.read("this param isn't used") is constant_result def test_write_raises(self): constant_result = ConstantResult(value="untouchable!") with pytest.raises(ValueError): constant_result.write("nvm") with pytest.raises(ValueError): constant_result.write("untouchable!") def test_handles_none_as_constant(self): constant_result = ConstantResult(value=None) assert constant_result.read("still not used") is constant_result @pytest.mark.parametrize( "constant_value", [3, "text", 5.0, Constant(3), Constant("text"), Constant(5.0)] ) def test_exists(self, constant_value: Union[str, Constant]): result = ConstantResult(value=constant_value) result_exists = result.exists("") assert result_exists is True
def as_task(x: Any, flow: "Optional[Flow]" = None) -> "prefect.Task": """ Wraps a function, collection, or constant with the appropriate Task type. If a constant or collection of constants is passed, a `Constant` task is returned. Args: - x (object): any Python object to convert to a prefect Task - flow (Flow, optional): Flow to which the prefect Task will be bound Returns: - a prefect Task representing the passed object """ from prefect.tasks.core.constants import Constant def is_constant(x: Any) -> bool: """ Helper function for determining if nested collections are constants without calling `bind()`, which would create new tasks on the active graph. """ if isinstance(x, (prefect.core.Task, unmapped)): return False elif isinstance(x, (list, tuple, set)): return all(is_constant(xi) for xi in x) elif isinstance(x, dict): return all(is_constant(xi) for xi in x.values()) return True # task objects if isinstance(x, prefect.core.Task): # type: ignore return x elif isinstance(x, unmapped): return x.task # handle constants, including collections of constants elif is_constant(x): return_task = Constant(x) # type: prefect.core.Task # collections elif isinstance(x, list): return_task = prefect.tasks.core.collections.List().bind(*x, flow=flow) elif isinstance(x, tuple): return_task = prefect.tasks.core.collections.Tuple().bind(*x, flow=flow) elif isinstance(x, set): return_task = prefect.tasks.core.collections.Set().bind(*x, flow=flow) elif isinstance(x, dict): keys, values = [], [] for k, v in x.items(): keys.append(k) values.append(v) return_task = prefect.tasks.core.collections.Dict().bind(keys=keys, values=values, flow=flow) else: return x return_task.auto_generated = True # type: ignore return return_task
def test_mapping_over_one_unmappable_input(executor): with Flow(name="test") as f: a = AddTask().map(x=Constant(1), y=Constant([1])) s = f.run(executor=executor) assert s.is_failed() assert s.result[a].is_failed() assert (s.result[a].message == "At least one upstream state has an unmappable result.")
def test_merge_can_distinguish_between_a_none_result_and_an_unrun_task(): condition = Condition() true_branch = Constant(None) false_branch = Constant(0) with Flow(name="test") as flow: ifelse(condition, true_branch, false_branch) merge_task = merge(true_branch, false_branch) with prefect.context(CONDITION=True): state = flow.run() assert state.result[merge_task].result is None
def test_merge_with_list(): with Flow(name="test") as flow: condition = Condition() true_branch = prefect.utilities.tasks.as_task([Constant(1), Constant(2)]) false_branch = Constant(0) with pytest.warns(prefect.utilities.exceptions.PrefectWarning): ifelse(condition, true_branch, false_branch) merge_task = merge(true_branch, false_branch) with prefect.context(CONDITION=True): state = flow.run() assert state.result[merge_task].result == [1, 2]
def test_tasks_have_all_non_unmapped_constant_args_as_transitive_upstream_deps( self, ): def func(a, b, c, d): m = inc.copy(name="m").bind(1) n = inc.copy(name="n").bind(a) o = inc.copy(name="o").bind(b) p = add.copy(name="p").bind(n, o) q = add.copy(name="q").bind(c, d) r = add.copy(name="r").bind(q, m) return m, n, o, p, q, r with Flow("test") as flow: a = ranged.copy(name="a").bind(3) b = inc.copy(name="b").bind(1) c = Constant(1, name="c") d = Constant(range(3), name="d") m, n, o, p, q, r = apply_map( func, a, edges.unmapped(b), c=edges.unmapped(c), d=d ) def edge_info(task): """Returns a map of {upstream: (is_data_dep, is_mapped)}""" return { e.upstream_task: (e.key is not None, e.mapped) for e in flow.edges_to(task) } assert edge_info(m) == {a: (False, True), b: (False, False), d: (False, True)} assert edge_info(n) == {a: (True, True), b: (False, False), d: (False, True)} assert edge_info(o) == {a: (False, True), b: (True, False), d: (False, True)} assert edge_info(p) == {n: (True, True), o: (True, True)} assert edge_info(q) == {a: (False, True), b: (False, False), d: (True, True)} assert edge_info(r) == {q: (True, True), m: (True, True)} state = flow.run() res = {t: state.result[t].result for t in [m, n, o, p, q, r]} sol = { m: [2, 2, 2], n: [1, 2, 3], o: [3, 3, 3], p: [4, 5, 6], q: [1, 2, 3], r: [3, 4, 5], } assert res == sol
class TestConstantResult: def test_instantiates_with_value(self): constant_result = ConstantResult(5) assert constant_result.value == 5 constant_result = ConstantResult(value=10) assert constant_result.value == 10 def test_read_returns_value(self): constant_result = ConstantResult("hello world") assert constant_result.read("this param isn't used") == "hello world" def test_write_doesnt_overwrite_value(self): constant_result = ConstantResult("untouchable!") constant_result.write() assert constant_result.value == "untouchable!" assert constant_result.read("still unused") == "untouchable!" def test_write_returns_value(self): constant_result = ConstantResult("constant value") output = constant_result.write() assert output == "'constant value'" def test_handles_none_as_constant(self): constant_result = ConstantResult(None) assert constant_result.read("still not used") is None output = constant_result.write() assert output == "None" @pytest.mark.parametrize( "constant_value", [3, "text", 5.0, Constant(3), Constant("text"), Constant(5.0)]) def test_exists(self, constant_value: Union[str, Constant]): result = ConstantResult(constant_value) result_exists = result.exists() assert result_exists is True
def test_merge_diamond_flow_with_results(): condition = Condition() true_branch = Constant(1) false_branch = Constant(0) with Flow(name="test") as flow: ifelse(condition, true_branch, false_branch) merge_task = merge(true_branch, false_branch) with prefect.context(CONDITION=True): state = flow.run() assert state.result[merge_task].result == 1 with prefect.context(CONDITION=False): state = flow.run() assert state.result[merge_task].result == 0 with prefect.context(CONDITION=None): state = flow.run() assert state.result[merge_task].result is None
def get_task_kwargs(op, ref, maps): new_kwargs = {} for k, v in op.get('kwargs', {}).items(): if isinstance(v, str) and v.startswith(':'): v = ref[v[1:]] else: v = Constant(v) if k not in maps: v = unmapped(v) new_kwargs[k] = v return new_kwargs
def test_constant_task_returns_its_value(self): x = Constant("x") assert x.run() == "x" y = Constant(100) assert y.run() == 100
# there's also using the calendar. # View Python command with `ps` # Rename `ps` command: https://stackoverflow.com/a/49097964/2138773 # https://github.com/dvarrazzo/py-setproctitle from setproctitle import setproctitle setproctitle('prefect: ds arxiv') # using the Imperitive API: https://docs.prefect.io/core/concepts/flows.html#imperative-api with Flow('Build Arxiv', state_handlers=[slack_handler]) as flow: # Dates date_today = datetime.now().strftime('%Y-%m-%d') # Begin the flow. Will fail if len(df) = 0 # FIXME: date_query is actually overwritten within the function to just be the max date. See function for details. df_full = df_get_arxiv(Constant(arx_list), Constant(arx_dict)) filter_to_date = determine_filter_date(df_full) df = filter_df_arxiv(df=df_full, filter_to_date=filter_to_date) # Creating the Post folder, save the dataframe there, and build the rmd dir_post = create_dir_post(date_published=date_today) dir_post.set_dependencies(upstream_tasks=[df]) written_df = write_df_to_csv(df=df, dir_post=dir_post) fp_post = copy_rmd_template(dir_post) knit = knit_rmd_to_html(fp_post=fp_post) knit.set_dependencies(upstream_tasks=[written_df]) # once knit, re-build with tweet. Requires set_dependencies to avoid conflict replace = replace_rmd_template_metadata(dir_post=dir_post, fp_post=fp_post, date_today=date_today,