def test_backfill(self, storage):
origin = self.fake_partition_set_origin("fake_partition_set")
backfills = storage.get_backfills()
assert len(backfills) == 0
one = PartitionBackfill(
"one",
origin,
BulkActionStatus.REQUESTED,
["a", "b", "c"],
False,
None,
None,
pendulum.now().timestamp(),
)
storage.add_backfill(one)
assert len(storage.get_backfills()) == 1
assert len(
storage.get_backfills(status=BulkActionStatus.REQUESTED)) == 1
backfill = storage.get_backfill(one.backfill_id)
assert backfill == one
storage.update_backfill(
one.with_status(status=BulkActionStatus.COMPLETED))
assert len(storage.get_backfills()) == 1
assert len(
storage.get_backfills(status=BulkActionStatus.REQUESTED)) == 0
def test_simple(external_repo_context, capfd):
with instance_for_context(external_repo_context) as (
instance,
_grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
launch_process = multiprocessing.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), None],
)
launch_process.start()
launch_process.join(timeout=60)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.COMPLETED
captured = capfd.readouterr()
assert (
captured.out.replace("\r\n", "\n") ==
"""2021-02-16 18:00:00 - BackfillDaemon - INFO - Starting backfill for simple
2021-02-16 18:00:00 - BackfillDaemon - INFO - Backfill completed for simple for 3 partitions
""")
def test_simple(capfd):
with instance_for_context(default_repo) as (
instance,
_grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
launch_process = spawn_ctx.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), None],
)
launch_process.start()
launch_process.join(timeout=60)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.COMPLETED
assert (
get_logger_output_from_capfd(capfd,
"dagster.daemon.BackfillDaemon") ==
"""2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple
2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Backfill completed for simple for 3 partitions"""
)
def test_large_backfill(external_repo_context):
with instance_for_context(external_repo_context) as (
instance,
grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"large_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
list(
execute_backfill_iteration(
instance, grpc_server_registry,
get_default_daemon_logger("BackfillDaemon")))
assert instance.get_runs_count() == 3
def test_unloadable_backfill(external_repo_context):
with instance_for_context(external_repo_context) as (
instance,
workspace,
_external_repo,
):
unloadable_origin = _unloadable_partition_set_origin()
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=unloadable_origin,
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
list(
execute_backfill_iteration(
instance, workspace,
get_default_daemon_logger("BackfillDaemon")))
assert instance.get_runs_count() == 0
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.FAILED
assert isinstance(backfill.error, SerializableErrorInfo)
def create_and_launch_partition_backfill(graphene_info, backfill_params):
from ...schema.backfill import GrapheneLaunchBackfillSuccess
from ...schema.errors import GraphenePartitionSetNotFoundError
partition_set_selector = backfill_params.get("selector")
partition_set_name = partition_set_selector.get("partitionSetName")
repository_selector = RepositorySelector.from_graphql_input(
partition_set_selector.get("repositorySelector"))
location = graphene_info.context.get_repository_location(
repository_selector.location_name)
repository = location.get_repository(repository_selector.repository_name)
matches = [
partition_set
for partition_set in repository.get_external_partition_sets()
if partition_set.name == partition_set_selector.get("partitionSetName")
]
if not matches:
return GraphenePartitionSetNotFoundError(partition_set_name)
check.invariant(
len(matches) == 1,
"Partition set names must be unique: found {num} matches for {partition_set_name}"
.format(num=len(matches), partition_set_name=partition_set_name),
)
external_partition_set = next(iter(matches))
partition_names = backfill_params.get("partitionNames")
backfill_id = make_new_backfill_id()
backfill = PartitionBackfill(
backfill_id=backfill_id,
partition_set_origin=external_partition_set.get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=partition_names,
from_failure=bool(backfill_params.get("fromFailure")),
reexecution_steps=backfill_params.get("reexecutionSteps"),
tags={t["key"]: t["value"]
for t in backfill_params.get("tags", [])},
backfill_timestamp=pendulum.now("UTC").timestamp(),
)
if backfill_params.get("forceSynchronousSubmission"):
# should only be used in a test situation
to_submit = [name for name in partition_names]
submitted_run_ids = []
while to_submit:
chunk = to_submit[:BACKFILL_CHUNK_SIZE]
to_submit = to_submit[BACKFILL_CHUNK_SIZE:]
submitted_run_ids.extend(run_id for run_id in submit_backfill_runs(
graphene_info.context.instance,
location,
backfill,
partition_names=chunk) if run_id != None)
return GrapheneLaunchBackfillSuccess(
backfill_id=backfill_id, launched_run_ids=submitted_run_ids)
graphene_info.context.instance.add_backfill(backfill)
return GrapheneLaunchBackfillSuccess(backfill_id=backfill_id)
def test_canceled_backfill():
with instance_for_context(default_repo) as (
instance,
workspace,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
iterator = execute_backfill_iteration(
instance, workspace, get_default_daemon_logger("BackfillDaemon"))
next(iterator)
assert instance.get_runs_count() == 1
backfill = instance.get_backfills()[0]
assert backfill.status == BulkActionStatus.REQUESTED
instance.update_backfill(
backfill.with_status(BulkActionStatus.CANCELED))
list(iterator)
backfill = instance.get_backfill(backfill.backfill_id)
assert backfill.status == BulkActionStatus.CANCELED
assert instance.get_runs_count() == 1
def test_simple_backfill(external_repo_context):
with instance_for_context(external_repo_context) as (instance,
external_repo):
external_partition_set = external_repo.get_external_partition_set(
"simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
list(
execute_backfill_iteration(
instance, get_default_daemon_logger("BackfillDaemon")))
assert instance.get_runs_count() == 3
runs = instance.get_runs()
three, two, one = runs
assert one.tags[BACKFILL_ID_TAG] == "simple"
assert one.tags[PARTITION_NAME_TAG] == "one"
assert two.tags[BACKFILL_ID_TAG] == "simple"
assert two.tags[PARTITION_NAME_TAG] == "two"
assert three.tags[BACKFILL_ID_TAG] == "simple"
assert three.tags[PARTITION_NAME_TAG] == "three"
def create_and_launch_partition_backfill(graphene_info, backfill_params):
from ...schema.backfill import GraphenePartitionBackfillSuccess
from ...schema.errors import GraphenePartitionSetNotFoundError
partition_set_selector = backfill_params.get("selector")
partition_set_name = partition_set_selector.get("partitionSetName")
repository_selector = RepositorySelector.from_graphql_input(
partition_set_selector.get("repositorySelector"))
location = graphene_info.context.get_repository_location(
repository_selector.location_name)
repository = location.get_repository(repository_selector.repository_name)
matches = [
partition_set
for partition_set in repository.get_external_partition_sets()
if partition_set.name == partition_set_selector.get("partitionSetName")
]
if not matches:
return GraphenePartitionSetNotFoundError(partition_set_name)
check.invariant(
len(matches) == 1,
"Partition set names must be unique: found {num} matches for {partition_set_name}"
.format(num=len(matches), partition_set_name=partition_set_name),
)
external_partition_set = next(iter(matches))
partition_names = backfill_params.get("partitionNames")
backfill_id = make_new_backfill_id()
backfill = PartitionBackfill(
backfill_id=backfill_id,
partition_set_origin=external_partition_set.get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=partition_names,
from_failure=bool(backfill_params.get("fromFailure")),
reexecution_steps=backfill_params.get("reexecutionSteps"),
tags={t["key"]: t["value"]
for t in backfill_params.get("tags", [])},
backfill_timestamp=pendulum.now("UTC").timestamp(),
)
backfill_settings = graphene_info.context.instance.get_settings(
"backfill") or {}
daemonEnabled = backfill_settings.get("daemon_enabled")
if daemonEnabled and not graphene_info.context.instance.has_bulk_actions_table(
):
check.failed(
"A schema migration is required before daemon-based backfills can be supported. "
"Try running `dagster instance migrate` to migrate your instance and try again."
)
elif daemonEnabled:
graphene_info.context.instance.add_backfill(backfill)
return GraphenePartitionBackfillSuccess(backfill_id=backfill_id)
else:
submitted_run_ids = submit_backfill_runs(
graphene_info.context.instance, location, backfill)
return GraphenePartitionBackfillSuccess(
backfill_id=backfill_id, launched_run_ids=submitted_run_ids)
def test_crash_after_submit(external_repo_context, crash_signal, capfd):
with instance_for_context(external_repo_context) as (instance,
external_repo):
external_partition_set = external_repo.get_external_partition_set(
"simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
launch_process = multiprocessing.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), {
"AFTER_SUBMIT": crash_signal
}],
)
launch_process.start()
launch_process.join(timeout=60)
assert launch_process.exitcode != 0
captured = capfd.readouterr()
assert (
captured.out.replace("\r\n", "\n") ==
"""2021-02-16 18:00:00 - BackfillDaemon - INFO - Starting backfill for simple
""")
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.REQUESTED
assert instance.get_runs_count() == 3
# resume backfill
launch_process = multiprocessing.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), None],
)
launch_process.start()
launch_process.join(timeout=60)
captured = capfd.readouterr()
assert (
captured.out.replace("\r\n", "\n") ==
"""2021-02-16 18:00:00 - BackfillDaemon - INFO - Starting backfill for simple
2021-02-16 18:00:00 - BackfillDaemon - INFO - Found 3 existing runs for backfill simple, skipping
2021-02-16 18:00:00 - BackfillDaemon - INFO - Backfill completed for simple for 3 partitions
""")
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.COMPLETED
assert instance.get_runs_count() == 3
def test_before_submit(external_repo_context, crash_signal, capfd):
with instance_for_context(external_repo_context) as (
instance,
_grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set("simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
)
)
launch_process = multiprocessing.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), {"BEFORE_SUBMIT": crash_signal}],
)
launch_process.start()
launch_process.join(timeout=60)
assert launch_process.exitcode != 0
assert (
get_logger_output_from_capfd(capfd, "dagster.daemon.BackfillDaemon")
== """2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple"""
)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.REQUESTED
assert instance.get_runs_count() == 0
# resume backfill
launch_process = multiprocessing.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), None],
)
launch_process.start()
launch_process.join(timeout=60)
assert (
get_logger_output_from_capfd(capfd, "dagster.daemon.BackfillDaemon")
== """2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple
2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Backfill completed for simple for 3 partitions"""
)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.COMPLETED
assert instance.get_runs_count() == 3
def test_backfill_from_failure_for_subselection():
with instance_for_context(default_repo) as (
instance,
workspace,
external_repo,
):
partition = parallel_failure_partition_set.get_partition("one")
run_config = parallel_failure_partition_set.run_config_for_partition(partition)
tags = parallel_failure_partition_set.tags_for_partition(partition)
external_partition_set = external_repo.get_external_partition_set(
"parallel_failure_partition_set"
)
execute_pipeline(
parallel_failure_pipeline,
run_config=run_config,
tags=tags,
instance=instance,
solid_selection=["fail_three", "success_four"],
raise_on_error=False,
)
assert instance.get_runs_count() == 1
wait_for_all_runs_to_finish(instance)
run = instance.get_runs()[0]
assert run.status == PipelineRunStatus.FAILURE
instance.add_backfill(
PartitionBackfill(
backfill_id="fromfailure",
partition_set_origin=external_partition_set.get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one"],
from_failure=True,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
)
)
list(
execute_backfill_iteration(
instance, workspace, get_default_daemon_logger("BackfillDaemon")
)
)
assert instance.get_runs_count() == 2
run = instance.get_runs(limit=1)[0]
assert run.solids_to_execute
assert run.solid_selection
assert len(run.solids_to_execute) == 2
assert len(run.solid_selection) == 2
def test_backfill_from_partitioned_job(external_repo_context):
partition_name_list = [
partition.name
for partition in my_config.partitions_def.get_partitions()
]
with instance_for_context(external_repo_context) as (
instance,
workspace,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"comp_always_succeed_default_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="partition_schedule_from_job",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=partition_name_list[:3],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
list(
execute_backfill_iteration(
instance, workspace,
get_default_daemon_logger("BackfillDaemon")))
assert instance.get_runs_count() == 3
runs = reversed(instance.get_runs())
for idx, run in enumerate(runs):
assert run.tags[BACKFILL_ID_TAG] == "partition_schedule_from_job"
assert run.tags[PARTITION_NAME_TAG] == partition_name_list[idx]
assert run.tags[
PARTITION_SET_TAG] == "comp_always_succeed_default_partition_set"
def _execute_backfill_command_at_location(cli_args, print_fn, instance, workspace, repo_location):
external_repo = get_external_repository_from_repo_location(
repo_location, cli_args.get("repository")
)
external_pipeline = get_external_pipeline_from_external_repo(
external_repo,
cli_args.get("pipeline"),
)
noprompt = cli_args.get("noprompt")
pipeline_partition_set_names = {
external_partition_set.name: external_partition_set
for external_partition_set in external_repo.get_external_partition_sets()
if external_partition_set.pipeline_name == external_pipeline.name
}
if not pipeline_partition_set_names:
raise click.UsageError(
"No partition sets found for pipeline `{}`".format(external_pipeline.name)
)
partition_set_name = cli_args.get("partition_set")
if not partition_set_name:
if len(pipeline_partition_set_names) == 1:
partition_set_name = next(iter(pipeline_partition_set_names.keys()))
elif noprompt:
raise click.UsageError("No partition set specified (see option `--partition-set`)")
else:
partition_set_name = click.prompt(
"Select a partition set to use for backfill: {}".format(
", ".join(x for x in pipeline_partition_set_names.keys())
)
)
partition_set = pipeline_partition_set_names.get(partition_set_name)
if not partition_set:
raise click.UsageError("No partition set found named `{}`".format(partition_set_name))
run_tags = get_tags_from_args(cli_args)
repo_handle = RepositoryHandle(
repository_name=external_repo.name,
repository_location=repo_location,
)
try:
partition_names_or_error = repo_location.get_external_partition_names(
repo_handle,
partition_set_name,
)
except Exception: # pylint: disable=broad-except
error_info = serializable_error_info_from_exc_info(sys.exc_info())
raise DagsterBackfillFailedError(
"Failure fetching partition names for {partition_set_name}: {error_message}".format(
partition_set_name=partition_set_name,
error_message=error_info.message,
),
serialized_error_info=error_info,
)
partition_names = gen_partition_names_from_args(
partition_names_or_error.partition_names, cli_args
)
# Print backfill info
print_fn("\n Pipeline: {}".format(external_pipeline.name))
print_fn("Partition set: {}".format(partition_set_name))
print_fn(" Partitions: {}\n".format(print_partition_format(partition_names, indent_level=15)))
# Confirm and launch
if noprompt or click.confirm(
"Do you want to proceed with the backfill ({} partitions)?".format(len(partition_names))
):
print_fn("Launching runs... ")
backfill_id = make_new_backfill_id()
backfill_job = PartitionBackfill(
backfill_id=backfill_id,
partition_set_origin=partition_set.get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=partition_names,
from_failure=False,
reexecution_steps=None,
tags=run_tags,
backfill_timestamp=pendulum.now("UTC").timestamp(),
)
try:
partition_execution_data = (
repo_location.get_external_partition_set_execution_param_data(
repository_handle=repo_handle,
partition_set_name=partition_set_name,
partition_names=partition_names,
)
)
except Exception: # pylint: disable=broad-except
error_info = serializable_error_info_from_exc_info(sys.exc_info())
instance.add_backfill(
backfill_job.with_status(BulkActionStatus.FAILED).with_error(error_info)
)
return print_fn("Backfill failed: {}".format(error_info))
assert isinstance(partition_execution_data, ExternalPartitionSetExecutionParamData)
for partition_data in partition_execution_data.partition_data:
pipeline_run = create_backfill_run(
instance,
repo_location,
external_pipeline,
partition_set,
backfill_job,
partition_data,
)
if pipeline_run:
instance.submit_run(pipeline_run.run_id, workspace)
instance.add_backfill(backfill_job.with_status(BulkActionStatus.COMPLETED))
print_fn("Launched backfill job `{}`".format(backfill_id))
else:
print_fn("Aborted!")
def test_partial_backfill(external_repo_context):
with instance_for_context(external_repo_context) as (
instance,
grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"partial_partition_set")
# create full runs, where every step is executed
instance.add_backfill(
PartitionBackfill(
backfill_id="full",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
list(
execute_backfill_iteration(
instance, grpc_server_registry,
get_default_daemon_logger("BackfillDaemon")))
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 3
runs = instance.get_runs()
three, two, one = runs
assert one.tags[BACKFILL_ID_TAG] == "full"
assert one.tags[PARTITION_NAME_TAG] == "one"
assert one.status == PipelineRunStatus.SUCCESS
assert step_succeeded(instance, one, "step_one")
assert step_succeeded(instance, one, "step_two")
assert step_succeeded(instance, one, "step_three")
assert two.tags[BACKFILL_ID_TAG] == "full"
assert two.tags[PARTITION_NAME_TAG] == "two"
assert two.status == PipelineRunStatus.SUCCESS
assert step_succeeded(instance, two, "step_one")
assert step_succeeded(instance, two, "step_two")
assert step_succeeded(instance, two, "step_three")
assert three.tags[BACKFILL_ID_TAG] == "full"
assert three.tags[PARTITION_NAME_TAG] == "three"
assert three.status == PipelineRunStatus.SUCCESS
assert step_succeeded(instance, three, "step_one")
assert step_succeeded(instance, three, "step_two")
assert step_succeeded(instance, three, "step_three")
# delete one of the runs, the partial reexecution should still succeed because the steps
# can be executed independently, require no input/output config
instance.delete_run(one.run_id)
assert instance.get_runs_count() == 2
# create partial runs
instance.add_backfill(
PartitionBackfill(
backfill_id="partial",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=["step_one"],
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
list(
execute_backfill_iteration(
instance, grpc_server_registry,
get_default_daemon_logger("BackfillDaemon")))
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 5
partial_filter = PipelineRunsFilter(tags={BACKFILL_ID_TAG: "partial"})
assert instance.get_runs_count(filters=partial_filter) == 3
runs = instance.get_runs(filters=partial_filter)
three, two, one = runs
assert one.status == PipelineRunStatus.SUCCESS
assert step_succeeded(instance, one, "step_one")
assert step_did_not_run(instance, one, "step_two")
assert step_did_not_run(instance, one, "step_three")
assert two.status == PipelineRunStatus.SUCCESS
assert step_succeeded(instance, two, "step_one")
assert step_did_not_run(instance, two, "step_two")
assert step_did_not_run(instance, two, "step_three")
assert three.status == PipelineRunStatus.SUCCESS
assert step_succeeded(instance, three, "step_one")
assert step_did_not_run(instance, three, "step_two")
assert step_did_not_run(instance, three, "step_three")
def test_failure_backfill(external_repo_context):
output_file = _failure_flag_file()
with instance_for_context(external_repo_context) as (
instance,
grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"conditionally_fail_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="shouldfail",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert instance.get_runs_count() == 0
try:
touch_file(output_file)
list(
execute_backfill_iteration(
instance, grpc_server_registry,
get_default_daemon_logger("BackfillDaemon")))
wait_for_all_runs_to_start(instance)
finally:
os.remove(output_file)
assert instance.get_runs_count() == 3
runs = instance.get_runs()
three, two, one = runs
assert one.tags[BACKFILL_ID_TAG] == "shouldfail"
assert one.tags[PARTITION_NAME_TAG] == "one"
assert one.status == PipelineRunStatus.FAILURE
assert step_succeeded(instance, one, "always_succeed")
assert step_failed(instance, one, "conditionally_fail")
assert step_did_not_run(instance, one, "after_failure")
assert two.tags[BACKFILL_ID_TAG] == "shouldfail"
assert two.tags[PARTITION_NAME_TAG] == "two"
assert two.status == PipelineRunStatus.FAILURE
assert step_succeeded(instance, two, "always_succeed")
assert step_failed(instance, two, "conditionally_fail")
assert step_did_not_run(instance, two, "after_failure")
assert three.tags[BACKFILL_ID_TAG] == "shouldfail"
assert three.tags[PARTITION_NAME_TAG] == "three"
assert three.status == PipelineRunStatus.FAILURE
assert step_succeeded(instance, three, "always_succeed")
assert step_failed(instance, three, "conditionally_fail")
assert step_did_not_run(instance, three, "after_failure")
instance.add_backfill(
PartitionBackfill(
backfill_id="fromfailure",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=True,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
assert not os.path.isfile(_failure_flag_file())
list(
execute_backfill_iteration(
instance, grpc_server_registry,
get_default_daemon_logger("BackfillDaemon")))
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 6
from_failure_filter = PipelineRunsFilter(
tags={BACKFILL_ID_TAG: "fromfailure"})
assert instance.get_runs_count(filters=from_failure_filter) == 3
runs = instance.get_runs(filters=from_failure_filter)
three, two, one = runs
assert one.tags[BACKFILL_ID_TAG] == "fromfailure"
assert one.tags[PARTITION_NAME_TAG] == "one"
assert one.status == PipelineRunStatus.SUCCESS
assert step_did_not_run(instance, one, "always_succeed")
assert step_succeeded(instance, one, "conditionally_fail")
assert step_succeeded(instance, one, "after_failure")
assert two.tags[BACKFILL_ID_TAG] == "fromfailure"
assert two.tags[PARTITION_NAME_TAG] == "two"
assert two.status == PipelineRunStatus.SUCCESS
assert step_did_not_run(instance, one, "always_succeed")
assert step_succeeded(instance, one, "conditionally_fail")
assert step_succeeded(instance, one, "after_failure")
assert three.tags[BACKFILL_ID_TAG] == "fromfailure"
assert three.tags[PARTITION_NAME_TAG] == "three"
assert three.status == PipelineRunStatus.SUCCESS
assert step_did_not_run(instance, one, "always_succeed")
assert step_succeeded(instance, one, "conditionally_fail")
assert step_succeeded(instance, one, "after_failure")