def test_merge_diamond_flow_with_results():
condition = Condition()
@task
def true_branch():
return 1
@task
def false_branch():
return 0
with Flow(name="test") as flow:
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result == 1
with prefect.context(CONDITION=False):
state = flow.run()
assert state.result[merge_task].result == 0
with prefect.context(CONDITION=None):
state = flow.run()
assert state.result[merge_task].result is None
def test_merging_diamond_flow():
"""
Test a flow that branches into two separate chains that later merge back together.
One branch should all get skipped but the merge task should not skip.
"""
with Flow(name="test") as flow:
condition = Condition()
true_branch = [
SuccessTask(name="true branch {}".format(i)) for i in range(3)
]
false_branch = [
SuccessTask(name="false branch {}".format(i)) for i in range(3)
]
ifelse(condition, true_branch[0], false_branch[0])
flow.chain(*true_branch)
flow.chain(*false_branch)
merge_task = merge(true_branch[-1], false_branch[-1])
with prefect.context(CONDITION=True):
state = flow.run()
for t in true_branch:
assert isinstance(state.result[t], Success)
for t in false_branch:
assert isinstance(state.result[t], Skipped)
assert isinstance(state.result[merge_task], Success)
def test_case_execution(self, branch):
with Flow("test") as flow:
cond = identity(branch)
with case(cond, "a"):
a = identity(1)
b = inc(a)
with case(cond, "b"):
c = identity(3)
d = inc(c)
e = merge(b, d)
state = flow.run()
if branch == "a":
assert state.result[a].result == 1
assert state.result[b].result == 2
assert state.result[c].is_skipped()
assert state.result[d].is_skipped()
assert state.result[e].result == 2
elif branch == "b":
assert state.result[a].is_skipped()
assert state.result[b].is_skipped()
assert state.result[c].result == 3
assert state.result[d].result == 4
assert state.result[e].result == 4
elif branch == "c":
for t in [a, b, c, d, e]:
assert state.result[t].is_skipped()
def test_mapped_ifelse_and_merge():
@task
def is_even(x):
return x % 2 == 0
@task
def even():
return "even"
@task
def odd():
return "odd"
with Flow("iterated map") as flow:
mapped_result = is_even.map([1, 2, 3])
ifelse(condition=mapped_result,
true_task=even,
false_task=odd,
mapped=True)
merge_result = merge(even, odd, flow=flow, mapped=True)
state = flow.run()
assert state.result[merge_result].result == ["odd", "even", "odd"]
def test_nested_case_execution(self, branch1, branch2):
with Flow("test") as flow:
cond1 = identity(branch1)
a = identity(0)
with case(cond1, True):
cond2 = identity(branch2)
b = identity(10)
with case(cond2, True):
c = inc(a)
d = inc(c)
with case(cond2, False):
e = inc(b)
f = inc(e)
g = merge(d, f)
with case(cond1, False):
h = identity(3)
i = inc(h)
j = merge(g, i)
state = flow.run()
sol = {a: 0, cond1: branch1}
if branch1:
sol[cond2] = branch2
sol[b] = 10
if branch2:
sol[c] = 1
sol[d] = sol[g] = sol[j] = 2
else:
sol[e] = 11
sol[f] = sol[g] = sol[j] = 12
else:
sol[h] = 3
sol[i] = sol[j] = 4
for t in [cond1, cond2, a, b, c, d, e, f, g, h, i, j]:
if t in sol:
assert state.result[t].result == sol[t]
else:
assert state.result[t].is_skipped()
def main():
with Flow("manual-live-purpleair-control-flow") as flow:
offline = Parameter("offline", default=True)
all_sensors_online = extract_online_live_purpleair()
all_sensors_offline = extract_offline_live_purpleair()
ifelse(offline, all_sensors_offline, all_sensors_online)
all_sensors = merge(all_sensors_offline, all_sensors_online)
# Registers flow to server, which we can then deploy and run in background agents.
flow.register(project_name="caqi-flows")
def test_merge_can_distinguish_between_a_none_result_and_an_unrun_task():
condition = Condition()
true_branch = Constant(None)
false_branch = Constant(0)
with Flow(name="test") as flow:
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result is None
def test_merging_skips_if_all_upstreams_skip():
@task
def skip_task():
raise prefect.engine.signals.SKIP("not today")
with Flow("skipper") as flow:
merge_task = merge(skip_task(), skip_task())
flow_state = flow.run()
assert flow_state.is_successful()
assert flow_state.result[merge_task].is_skipped()
assert flow_state.result[merge_task].result is None
def test_merge_with_list():
with Flow(name="test") as flow:
condition = Condition()
true_branch = prefect.utilities.tasks.as_task([Constant(1), Constant(2)])
false_branch = Constant(0)
with pytest.warns(prefect.utilities.exceptions.PrefectWarning):
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result == [1, 2]
def inc_or_negate(input1, input2, squared):
cond = is_even(input1)
# If x is even, increment it
with case(cond, True):
res1 = inc(input1, unmapped(squared))
# If x is odd, negate it
with case(cond, False):
res2 = negate(input1, input2)
return merge(res1, res2)
def test_merge_order():
@task
def x():
return "x"
@task
def y():
return "y"
with Flow(name="test") as flow:
merge_task = merge(x(), y())
state = flow.run()
assert state.result[merge_task].result == "x"
def test_merge_imperative_flow_checkpoint_false():
flow = Flow("test")
cond = identity.copy().bind(True, flow=flow)
with case(cond, True):
a = inc.copy().bind(1, flow=flow)
with case(cond, False):
b = inc.copy().bind(2, flow=flow)
c = merge(a, b, flow=flow, checkpoint=False)
state = flow.run()
assert c.checkpoint == False
def test_merge_with_list():
@task
def false_branch():
return 0
@task
def true_branch():
return [1, 2]
with Flow(name="test") as flow:
condition = Condition()
ifelse(condition, true_branch, false_branch)
merge_task = merge(true_branch, false_branch)
with prefect.context(CONDITION=True):
state = flow.run()
assert state.result[merge_task].result == [1, 2]
def test_merge_imperative_flow():
flow = Flow("test")
cond = identity.copy().bind(True, flow=flow)
with case(cond, True):
a = inc.copy().bind(1, flow=flow)
with case(cond, False):
b = inc.copy().bind(2, flow=flow)
c = merge(a, b, flow=flow)
state = flow.run()
assert state.result[cond].result is True
assert state.result[a].result == 2
assert state.result[b].is_skipped()
assert state.result[c].result == 2
def test_mapped_switch_and_merge():
with Flow("iterated map") as flow:
mapped_result = identity.copy().map(["a", "b", "c"])
a = identity("a")
b = identity("b")
c = identity("c")
switch(condition=mapped_result, cases=dict(a=a, b=b, c=c), mapped=True)
merge_result = merge(a, b, c, mapped=True)
state = flow.run()
assert state.result[a].result == ["a", None, None]
assert state.result[b].result == [None, "b", None]
assert state.result[c].result == [None, None, "c"]
assert state.result[merge_result].result == ["a", "b", "c"]
def test_merging_with_objects_that_cant_be_equality_compared():
class SpecialObject:
def __eq__(self, other):
return self
def __bool__(self):
raise SyntaxError("You can't handle the truth!")
@task
def return_array():
return SpecialObject()
with Flow("test-merge") as flow:
success = SuccessTask()
ifelse(Condition(), success, return_array)
merge_task = merge(success, return_array)
with prefect.context(CONDITION=False):
flow_state = flow.run()
assert flow_state.is_successful()
assert isinstance(flow_state.result[merge_task].result, SpecialObject)
def gen_pipeline() -> Flow:
"""Generate the prefect flow.
Returns
-------
Flow
The generated flow.
"""
# Initialize the tasks
# Loader tasks
scoreboard_loader = GenericLoader(loader="Scoreboard",
name="Load scoreboard data")
pbp_loader = PlayByPlayLoader(name="Load play-by-play data")
wprob_loader = WinProbabilityLoader(name="Load NBA win probability")
ogetter = FactoryGetter(name="Get overall dataset from the Factory")
lgetter = FactoryGetter(name="Get lineup dataset from the Factory")
log_loader = GameLogLoader(name="Load gamelog data")
lineup_loader = LineupLoader(name="Load lineup data")
rota_loader = RotationLoader(name="Load rotation data")
shotchart_loader = ShotChartLoader(name="Load shotchart data")
box_loader = BoxScoreLoader(name="Load boxscore data")
shotzone_loader = ShotZoneLoader(
name="Load player-level shot zone dashboards")
gshooting_loader = GeneralShootingLoader(
name="Load player-level overall shooting dashboards")
# Transformation tasks
survtime_task = SurvivalTime(name="Add survival time")
wprob_task = AddNBAWinProbability(name="Add NBA win probability")
margin_task = FillMargin(name="Backfill margin")
target_task = CreateTarget(name="Add target label")
team_id_task = AddTeamID(name="Add team ID and game date")
rating_task = AddNetRating(name="Add net rating")
meeting_task = AddLastMeetingResult(name="Add last meeting result")
w_pct_task = AddWinPercentage(name="Add win percentage")
last3_task = GamesInLastXDays(period=3, name="Games in last 3 days")
last5_task = GamesInLastXDays(period=5, name="Games in last 5 days")
last7_task = GamesInLastXDays(period=7, name="Games in last 7 days")
lineup_task = AddLineupPlusMinus(name="Add lineup plus minus")
dedupe_task = DeDupeTime(name="De-dupe time")
# Add shotchart data for player rating
shotdetail = AddShotDetail(name="Add shotchart zone")
shotvalue = AddExpectedShotValue(name="Add shot value")
# Persisting clean data
persist = SaveData()
with Flow(name="Transform raw NBA data") as flow:
# Set some parameters
data_dir = Parameter("data_dir", "nba-data")
output_dir = Parameter("output_dir", "nba-data")
filesystem = Parameter("filesystem", "file")
season = Parameter("Season", DefaultParameters.Season)
gamedate = Parameter("GameDate", DefaultParameters.GameDate)
save_data = Parameter("save_data", True)
mode = Parameter("mode", "model")
# Load data
scoreboard = scoreboard_loader(
output_dir=data_dir,
filesystem=filesystem,
dataset_type=None,
GameDate=gamedate,
)
pbp = pbp_loader(
header=scoreboard["GameHeader"],
output_dir=data_dir,
filesystem=filesystem,
)
wprob = wprob_loader(
header=scoreboard["GameHeader"],
output_dir=data_dir,
filesystem=filesystem,
)
lineupdata = lineup_loader(season=season,
GameDate=gamedate,
linescore=scoreboard["LineScore"])
stats = ogetter(factory=lineupdata, dataset_type="Overall")
boxscore = box_loader(
header=scoreboard["GameHeader"],
output_dir=data_dir,
filesystem=filesystem,
)
# Base transformations
survtime = survtime_task(pbp=pbp)
nbawin = wprob_task(pbp=survtime, winprob=wprob)
margin = margin_task(pbp=nbawin)
target = target_task(pbp=margin)
team_id = team_id_task(pbp=target, header=scoreboard["GameHeader"])
rating = rating_task(pbp=team_id, stats=stats)
with case(mode, "rating"): # type: ignore
# Load shotchart and shot zone data
shotchart = shotchart_loader(
header=scoreboard["GameHeader"],
season=season,
output_dir=data_dir,
filesystem=filesystem,
)
shotzonedashboard = shotzone_loader(
boxscore=boxscore,
season=season,
GameDate=gamedate,
output_dir=data_dir,
filesystem=filesystem,
)
shooting = gshooting_loader(
boxscore=boxscore,
season=season,
output_dir=data_dir,
filesystem=filesystem,
)
# Add variables for the player rating
shotzone = shotdetail(pbp=rating, shotchart=shotchart)
expected_val = shotvalue(
pbp=shotzone,
shotzonedashboard=shotzonedashboard,
overallshooting=shooting,
)
with case(mode, "model"): # type: ignore
# Load data
gamelog = log_loader(
season=season,
output_dir=data_dir,
filesystem=filesystem,
)
lineup_stats = lgetter(factory=lineupdata, dataset_type="Lineups")
rotation = rota_loader(
header=scoreboard["GameHeader"],
output_dir=data_dir,
filesystem=filesystem,
)
# Transform data for the survival model
meeting = meeting_task(pbp=rating,
last_meeting=scoreboard["LastMeeting"])
w_pct = w_pct_task(pbp=meeting, gamelog=gamelog)
last3 = last3_task(pbp=w_pct, gamelog=gamelog)
last5 = last5_task(pbp=last3, gamelog=gamelog)
last7 = last7_task(pbp=last5, gamelog=gamelog)
lineup = lineup_task(
pbp=last7,
lineup_stats=lineup_stats,
home_rotation=rotation["HomeTeam"],
away_rotation=rotation["AwayTeam"],
)
deduped = dedupe_task(pbp=lineup)
# Save
final = merge(expected_val, deduped)
with case(save_data, True): # type: ignore
persist(data=final,
output_dir=output_dir,
filesystem=filesystem,
mode=mode)
return flow
def func(cond, a, b):
with case(cond, True):
res1 = a + 1
with case(cond, False):
res2 = b + 1
return merge(res1, res2)
def func(x):
with case(is_even(x), True):
x2 = add(x, 1)
return merge(x2, x)
@task
def action_if_true():
return "I am true!"
@task
def action_if_false():
return "I am false!"
@task(log_stdout=True)
def another_action(val):
print(val)
with Flow("Example: Conditional Tasks") as flow:
cond = check_condition()
with case(cond, True):
val_if_true = action_if_true()
with case(cond, False):
val_if_false = action_if_false()
val = merge(val_if_true, val_if_false)
another_action(val)
if __name__ == "__main__":
flow.run()
return "PAR!"
@task(name="Somar parâmetros")
def somar(a: int, b: int):
return a + b
@task(name="Imprimir resultado", log_stdout=True)
def imprimir(valor: str):
print(f"Sua soma é um número {valor}")
with Flow("condicionais") as flow:
a = Parameter(name="Primeiro valor")
b = Parameter(name="Segundo valor")
resultado = somar(a, b)
condicao = checar_par_ou_impar(resultado)
with case(condicao, True):
valor_verdadeiro = condicao_verdadeira()
with case(condicao, False):
valor_falso = condicao_falsa()
valor = merge(valor_verdadeiro, valor_falso)
imprimir(valor)
if __name__ == "__main__":
flow.register(project_name="conceitos")
logger = prefect.context.get("logger")
x, op, y = expression.split(' ')
logger.info("Received {} {} {}".format(x, op, y))
return dict(x=float(x), op=op, y=float(y))
# 'Arithmetic' is the name of the flow
with Flow('Arithmetic') as flow:
inputs = parse_input(Parameter('expression'))
# once we have our inputs, we create a dict of operations:
x, y = inputs['x'], inputs['y']
operations = {'+': x + y, '-': x - y, '*': x * y, '/': x / y}
# use prefect's `switch` task (conditional) to branch the flow, selecting the right operation
switch(condition=inputs['op'], cases=operations)
# use prefect's `merge` task to bring bring the branches from `switch` back together, producing a result
result = merge(*operations.values())
# do something with the result...
Result(result)
task_logger = get_logger("Task")
# Register the `Arithmetic` flow in the `Test1` project
flow.register(project_name='Test1')
# flow.run()
# print(flow.serialize())
producer = identity.copy().bind(["a", "b"])
cond = identity.copy().bind(producer, mapped=True)
a = identity.copy().bind("a")
a.set_upstream(producer, mapped=True)
b = identity.copy().bind("b")
b.set_upstream(producer, mapped=True)
c = identity.copy().bind("c")
c.set_upstream(producer, mapped=True)
switch(cond, cases=dict(a=a, b=b, c=c), mapped=True)
d = merge(a, b, mapped=True)
# state = flow.run()
# assert state.result[cond].result == ["a", "b"]
# assert state.result[a].result == ["a", None]
# assert state.result[b].result == [None, "b"]
# assert state.result[c].result == [None, None]
# assert state.result[d].result == ["a", "b"]
@task
def get_data():
return [1, 2, 3, 4]
def func(x, flow):
cond = is_even.copy().bind(x, flow=flow)
with case(cond, True):
x2 = add.copy().bind(x, 1, flow=flow)
return merge(x2, x, flow=flow)
