Ejemplo n.º 1
0
# %%
# The `date_formatter_wf` workflow can be scheduled using either the `CronSchedule` or the `FixedRate` object.
#
# Cron Schedules
# ##############
#
# `Cron <https://en.wikipedia.org/wiki/Cron>`_ expression strings use this :ref:`syntax <concepts-schedules>`.
# An incorrect cron schedule expression would lead to failure in triggering the schedule.
from flytekit import CronSchedule, LaunchPlan

# creates a launch plan that runs every minute.
cron_lp = LaunchPlan.get_or_create(
    name="my_cron_scheduled_lp",
    workflow=date_formatter_wf,
    schedule=CronSchedule(
        # Note that kickoff_time_input_arg matches the workflow input we defined above: kickoff_time
        # But in case you are using the AWS scheme of schedules and not using the native scheduler then switch over the schedule parameter with cron_expression
        schedule="*/1 * * * *",  # Following schedule runs every min
        kickoff_time_input_arg="kickoff_time",
    ),
)

# %%
# The ``kickoff_time_input_arg`` corresponds to the workflow input ``kickoff_time``.
# This means that the workflow gets triggered only after the specified kickoff time, and it thereby runs every minute.

# %%
# Fixed Rate Intervals
# ####################
#
# If you prefer to use an interval rather than a cron scheduler to schedule your workflows, you can use the fixed-rate scheduler.
# A fixed-rate scheduler runs at the specified interval.
Ejemplo n.º 2
0
@workflow
def int_doubler_wf(a: int) -> str:
    doubled = double_int_and_print(a=a)
    return doubled

# %%
# Here are three scenarios that can help deepen your understanding of how notifications work:
# 
# 1. Launch Plan triggers email notifications when the workflow execution reaches the ``SUCCEEDED`` phase.
int_doubler_wf_lp = LaunchPlan.get_or_create(
    name="int_doubler_wf",
    workflow=int_doubler_wf,
    default_inputs={"a": 4},
    notifications=[
        Email(
            phases=[WorkflowExecutionPhase.SUCCEEDED],
            recipients_email=["*****@*****.**"],
        )
    ],
)

# %%
# 2. Notifications shine when used for scheduled workflows to alert for failures.
from datetime import timedelta

from flytekit import FixedRate, PagerDuty

int_doubler_wf_scheduled_lp = LaunchPlan.get_or_create(
    name="int_doubler_wf_scheduled",
    workflow=int_doubler_wf,
Ejemplo n.º 3
0
    wordCount = dict(Counter(words))
    return wordCount


# %%
# We define a workflow that executes the previously defined task.
@workflow
def ext_workflow(my_input: str) -> Dict:
    result = count_freq_words(input_string1=my_input)
    return result


# %%
# Next, we create a launch plan.
external_lp = LaunchPlan.get_or_create(
    ext_workflow,
    "parent_workflow_execution",
)

# %%
# We define another task that returns the repeated keys (in our case, words) from a dictionary.


@task
def count_repetitive_words(word_counter: Dict) -> typing.List[str]:
    repeated_words = [key for key, value in word_counter.items() if value > 1]
    return repeated_words


# %%
# We define a workflow that triggers the launch plan of the previously-defined workflow.
@workflow
Ejemplo n.º 4
0
    return result


default_lp = LaunchPlan.get_default_launch_plan(current_context(), my_wf)
square_3 = default_lp(val=3)

# %%
# The following shows how to specify a user-defined launch plan that defaults the value of 'val' to 4.
my_lp = LaunchPlan.create("default_4_lp", my_wf, default_inputs={"val": 4})
square_4 = my_lp()
square_5 = my_lp(val=5)

# %%
# It is possible to **fix** launch plan inputs, so that they can't be overridden at execution call time.
my_fixed_lp = LaunchPlan.get_or_create(name="always_2_lp",
                                       workflow=my_wf,
                                       fixed_inputs={"val": 4})
square_2 = my_fixed_lp()
# error:
# square_1 = my_fixed_lp(val=1)

# %%
# Putting It All Together
# #######################
#
# Default and fixed inputs can be used together to simplify individual executions
# and programmatic ones.
# Here is a simple example to greet each day of the upcoming week:


@task
Ejemplo n.º 5
0
def double(a: int) -> int:
    return a * 2


@task
def add(a: int, b: int) -> int:
    return a + b


@workflow
def my_childwf(a: int = 42) -> int:
    b = double(a=a)
    return b


child_lp = LaunchPlan.get_or_create(my_childwf,
                                    name="my_fixed_child_lp",
                                    labels=Labels({"l1": "v1"}))


@workflow
def parent_wf(a: int) -> int:
    x = double(a=a)
    y = child_lp(a=x)
    z = add(a=x, b=y)
    return z


if __name__ == "__main__":
    print(f"Running parent_wf(a=3) {parent_wf(a=3)}")