@task
def add_two(y):
return y + 1
with Flow("Flow 2") as flow_2:
new_num2 = add_one(add_num)
combo_fl = Flow("Add Numbers")
combo_fl.update(flow_1)
combo_fl.update(flow_2, validate=True)
combo_fl.visualize()
# @task
# def return_list():
# return 1
# @task
# def add_one(x):
# return x + 1
# @task
# def divide_num(x):
# return x/2
# @task
# def mul_num
We run our flow on a 1 minute interval schedule and observe that the output of
`return_random_number` only changes every other run, due to output caching.
"""
import datetime
import random
from prefect import Flow, task
from prefect.schedules import IntervalSchedule
@task(cache_for=datetime.timedelta(minutes=1, seconds=30))
def return_random_number():
return random.random()
@task
def print_number(num):
print("=" * 50)
print("Value: {}".format(num))
print("=" * 50)
schedule = IntervalSchedule(start_date=datetime.datetime.utcnow(),
interval=datetime.timedelta(minutes=1))
flow = Flow("cached-task", schedule=schedule)
flow.set_dependencies(task=print_number, upstream_tasks=[return_random_number])
flow.visualize(format='png', filename=__file__.replace('.py', ''))
return n * 10
class Divide(Task):
def run(self, n):
return n / 10
class OutputValue(Task):
def run(self, value):
print(value)
flow = Flow("branches")
n = GetNumber()
m = Multiply()
d = Divide()
m.set_upstream(n, key="n", flow=flow)
d.set_upstream(n, key="n", flow=flow)
output_1 = OutputValue()
output_2 = OutputValue()
output_1.set_upstream(m, key="value", flow=flow)
output_2.set_upstream(d, key="value", flow=flow)
flow.visualize()
composeAllGraphsCollapsed = TBD()
# writes graph to pics
drawGraph = TBD()
# %% SET DEPENDENCIES in flow_deck_graph
# flow_deck_graph.set_dependencies(
# )
deck_slug = None
run_full_flow = True
run_scrapy_crawl = False
run_mtgmetaio_landind_to_deks = False
if __name__ == "__main__":
if deck_slug:
flow_deck_graph.visualize()
flow_state = flow_deck_graph.run()
flow_full_data_pipeline.visualize(flow_state=flow_state)
if run_full_flow:
flow_full_data_pipeline.visualize()
flow_state = flow_full_data_pipeline.run()
flow_full_data_pipeline.visualize(flow_state=flow_state)
if run_scrapy_crawl:
flow_scrapy_crawl.visualize()
flow_state = flow_scrapy_crawl.run()
flow_scrapy_crawl.visualize(flow_state=flow_state)
if run_mtgmetaio_landind_to_deks:
flow_landing_to_decks = get_flow_landing_to_decks(for_urls=[])
