def test_large_backfill(external_repo_context): with instance_for_context(external_repo_context) as ( instance, grpc_server_registry, external_repo, ): external_partition_set = external_repo.get_external_partition_set( "large_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="simple", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 list( execute_backfill_iteration( instance, grpc_server_registry, get_default_daemon_logger("BackfillDaemon"))) assert instance.get_runs_count() == 3
def test_canceled_backfill(): with instance_for_context(default_repo) as ( instance, workspace, external_repo, ): external_partition_set = external_repo.get_external_partition_set( "simple_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="simple", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 iterator = execute_backfill_iteration( instance, workspace, get_default_daemon_logger("BackfillDaemon")) next(iterator) assert instance.get_runs_count() == 1 backfill = instance.get_backfills()[0] assert backfill.status == BulkActionStatus.REQUESTED instance.update_backfill( backfill.with_status(BulkActionStatus.CANCELED)) list(iterator) backfill = instance.get_backfill(backfill.backfill_id) assert backfill.status == BulkActionStatus.CANCELED assert instance.get_runs_count() == 1
def test_unloadable_backfill(external_repo_context): with instance_for_context(external_repo_context) as ( instance, workspace, _external_repo, ): unloadable_origin = _unloadable_partition_set_origin() instance.add_backfill( PartitionBackfill( backfill_id="simple", partition_set_origin=unloadable_origin, status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 list( execute_backfill_iteration( instance, workspace, get_default_daemon_logger("BackfillDaemon"))) assert instance.get_runs_count() == 0 backfill = instance.get_backfill("simple") assert backfill.status == BulkActionStatus.FAILED assert isinstance(backfill.error, SerializableErrorInfo)
def test_simple_backfill(external_repo_context): with instance_for_context(external_repo_context) as (instance, external_repo): external_partition_set = external_repo.get_external_partition_set( "simple_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="simple", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 list( execute_backfill_iteration( instance, get_default_daemon_logger("BackfillDaemon"))) assert instance.get_runs_count() == 3 runs = instance.get_runs() three, two, one = runs assert one.tags[BACKFILL_ID_TAG] == "simple" assert one.tags[PARTITION_NAME_TAG] == "one" assert two.tags[BACKFILL_ID_TAG] == "simple" assert two.tags[PARTITION_NAME_TAG] == "two" assert three.tags[BACKFILL_ID_TAG] == "simple" assert three.tags[PARTITION_NAME_TAG] == "three"
def test_backfill_from_failure_for_subselection(): with instance_for_context(default_repo) as ( instance, workspace, external_repo, ): partition = parallel_failure_partition_set.get_partition("one") run_config = parallel_failure_partition_set.run_config_for_partition(partition) tags = parallel_failure_partition_set.tags_for_partition(partition) external_partition_set = external_repo.get_external_partition_set( "parallel_failure_partition_set" ) execute_pipeline( parallel_failure_pipeline, run_config=run_config, tags=tags, instance=instance, solid_selection=["fail_three", "success_four"], raise_on_error=False, ) assert instance.get_runs_count() == 1 wait_for_all_runs_to_finish(instance) run = instance.get_runs()[0] assert run.status == PipelineRunStatus.FAILURE instance.add_backfill( PartitionBackfill( backfill_id="fromfailure", partition_set_origin=external_partition_set.get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one"], from_failure=True, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), ) ) list( execute_backfill_iteration( instance, workspace, get_default_daemon_logger("BackfillDaemon") ) ) assert instance.get_runs_count() == 2 run = instance.get_runs(limit=1)[0] assert run.solids_to_execute assert run.solid_selection assert len(run.solids_to_execute) == 2 assert len(run.solid_selection) == 2
def _test_backfill_in_subprocess(instance_ref, debug_crash_flags): execution_datetime = pendulum.datetime( year=2021, month=2, day=17, ).in_tz("US/Central") with DagsterInstance.from_ref(instance_ref) as instance: try: with pendulum.test(execution_datetime): list( execute_backfill_iteration( instance, get_default_daemon_logger("BackfillDaemon"), debug_crash_flags=debug_crash_flags, )) finally: cleanup_test_instance(instance)
def _test_backfill_in_subprocess(instance_ref, debug_crash_flags): execution_datetime = to_timezone( create_pendulum_time( year=2021, month=2, day=17, ), "US/Central", ) with DagsterInstance.from_ref(instance_ref) as instance: try: with pendulum.test(execution_datetime), ProcessGrpcServerRegistry( wait_for_processes_on_exit=True) as grpc_server_registry: list( execute_backfill_iteration( instance, grpc_server_registry, get_default_daemon_logger("BackfillDaemon"), debug_crash_flags=debug_crash_flags, )) finally: cleanup_test_instance(instance)
def _test_backfill_in_subprocess(instance_ref, debug_crash_flags): execution_datetime = to_timezone( create_pendulum_time( year=2021, month=2, day=17, ), "US/Central", ) with DagsterInstance.from_ref(instance_ref) as instance: try: with pendulum.test(execution_datetime), create_test_daemon_workspace() as workspace: list( execute_backfill_iteration( instance, workspace, get_default_daemon_logger("BackfillDaemon"), debug_crash_flags=debug_crash_flags, ) ) finally: cleanup_test_instance(instance)
def test_backfill_from_partitioned_job(external_repo_context): partition_name_list = [ partition.name for partition in my_config.partitions_def.get_partitions() ] with instance_for_context(external_repo_context) as ( instance, workspace, external_repo, ): external_partition_set = external_repo.get_external_partition_set( "comp_always_succeed_default_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="partition_schedule_from_job", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=partition_name_list[:3], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 list( execute_backfill_iteration( instance, workspace, get_default_daemon_logger("BackfillDaemon"))) assert instance.get_runs_count() == 3 runs = reversed(instance.get_runs()) for idx, run in enumerate(runs): assert run.tags[BACKFILL_ID_TAG] == "partition_schedule_from_job" assert run.tags[PARTITION_NAME_TAG] == partition_name_list[idx] assert run.tags[ PARTITION_SET_TAG] == "comp_always_succeed_default_partition_set"
def test_partial_backfill(external_repo_context): with instance_for_context(external_repo_context) as ( instance, grpc_server_registry, external_repo, ): external_partition_set = external_repo.get_external_partition_set( "partial_partition_set") # create full runs, where every step is executed instance.add_backfill( PartitionBackfill( backfill_id="full", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 list( execute_backfill_iteration( instance, grpc_server_registry, get_default_daemon_logger("BackfillDaemon"))) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 3 runs = instance.get_runs() three, two, one = runs assert one.tags[BACKFILL_ID_TAG] == "full" assert one.tags[PARTITION_NAME_TAG] == "one" assert one.status == PipelineRunStatus.SUCCESS assert step_succeeded(instance, one, "step_one") assert step_succeeded(instance, one, "step_two") assert step_succeeded(instance, one, "step_three") assert two.tags[BACKFILL_ID_TAG] == "full" assert two.tags[PARTITION_NAME_TAG] == "two" assert two.status == PipelineRunStatus.SUCCESS assert step_succeeded(instance, two, "step_one") assert step_succeeded(instance, two, "step_two") assert step_succeeded(instance, two, "step_three") assert three.tags[BACKFILL_ID_TAG] == "full" assert three.tags[PARTITION_NAME_TAG] == "three" assert three.status == PipelineRunStatus.SUCCESS assert step_succeeded(instance, three, "step_one") assert step_succeeded(instance, three, "step_two") assert step_succeeded(instance, three, "step_three") # delete one of the runs, the partial reexecution should still succeed because the steps # can be executed independently, require no input/output config instance.delete_run(one.run_id) assert instance.get_runs_count() == 2 # create partial runs instance.add_backfill( PartitionBackfill( backfill_id="partial", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=["step_one"], tags=None, backfill_timestamp=pendulum.now().timestamp(), )) list( execute_backfill_iteration( instance, grpc_server_registry, get_default_daemon_logger("BackfillDaemon"))) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 5 partial_filter = PipelineRunsFilter(tags={BACKFILL_ID_TAG: "partial"}) assert instance.get_runs_count(filters=partial_filter) == 3 runs = instance.get_runs(filters=partial_filter) three, two, one = runs assert one.status == PipelineRunStatus.SUCCESS assert step_succeeded(instance, one, "step_one") assert step_did_not_run(instance, one, "step_two") assert step_did_not_run(instance, one, "step_three") assert two.status == PipelineRunStatus.SUCCESS assert step_succeeded(instance, two, "step_one") assert step_did_not_run(instance, two, "step_two") assert step_did_not_run(instance, two, "step_three") assert three.status == PipelineRunStatus.SUCCESS assert step_succeeded(instance, three, "step_one") assert step_did_not_run(instance, three, "step_two") assert step_did_not_run(instance, three, "step_three")
def test_failure_backfill(external_repo_context): output_file = _failure_flag_file() with instance_for_context(external_repo_context) as ( instance, grpc_server_registry, external_repo, ): external_partition_set = external_repo.get_external_partition_set( "conditionally_fail_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="shouldfail", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert instance.get_runs_count() == 0 try: touch_file(output_file) list( execute_backfill_iteration( instance, grpc_server_registry, get_default_daemon_logger("BackfillDaemon"))) wait_for_all_runs_to_start(instance) finally: os.remove(output_file) assert instance.get_runs_count() == 3 runs = instance.get_runs() three, two, one = runs assert one.tags[BACKFILL_ID_TAG] == "shouldfail" assert one.tags[PARTITION_NAME_TAG] == "one" assert one.status == PipelineRunStatus.FAILURE assert step_succeeded(instance, one, "always_succeed") assert step_failed(instance, one, "conditionally_fail") assert step_did_not_run(instance, one, "after_failure") assert two.tags[BACKFILL_ID_TAG] == "shouldfail" assert two.tags[PARTITION_NAME_TAG] == "two" assert two.status == PipelineRunStatus.FAILURE assert step_succeeded(instance, two, "always_succeed") assert step_failed(instance, two, "conditionally_fail") assert step_did_not_run(instance, two, "after_failure") assert three.tags[BACKFILL_ID_TAG] == "shouldfail" assert three.tags[PARTITION_NAME_TAG] == "three" assert three.status == PipelineRunStatus.FAILURE assert step_succeeded(instance, three, "always_succeed") assert step_failed(instance, three, "conditionally_fail") assert step_did_not_run(instance, three, "after_failure") instance.add_backfill( PartitionBackfill( backfill_id="fromfailure", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=True, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) assert not os.path.isfile(_failure_flag_file()) list( execute_backfill_iteration( instance, grpc_server_registry, get_default_daemon_logger("BackfillDaemon"))) wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 6 from_failure_filter = PipelineRunsFilter( tags={BACKFILL_ID_TAG: "fromfailure"}) assert instance.get_runs_count(filters=from_failure_filter) == 3 runs = instance.get_runs(filters=from_failure_filter) three, two, one = runs assert one.tags[BACKFILL_ID_TAG] == "fromfailure" assert one.tags[PARTITION_NAME_TAG] == "one" assert one.status == PipelineRunStatus.SUCCESS assert step_did_not_run(instance, one, "always_succeed") assert step_succeeded(instance, one, "conditionally_fail") assert step_succeeded(instance, one, "after_failure") assert two.tags[BACKFILL_ID_TAG] == "fromfailure" assert two.tags[PARTITION_NAME_TAG] == "two" assert two.status == PipelineRunStatus.SUCCESS assert step_did_not_run(instance, one, "always_succeed") assert step_succeeded(instance, one, "conditionally_fail") assert step_succeeded(instance, one, "after_failure") assert three.tags[BACKFILL_ID_TAG] == "fromfailure" assert three.tags[PARTITION_NAME_TAG] == "three" assert three.status == PipelineRunStatus.SUCCESS assert step_did_not_run(instance, one, "always_succeed") assert step_succeeded(instance, one, "conditionally_fail") assert step_succeeded(instance, one, "after_failure")
def run_iteration(self, instance, workspace): yield from execute_backfill_iteration(instance, workspace, self._logger)
def run_iteration(self, instance, daemon_shutdown_event, grpc_server_registry): yield from execute_backfill_iteration(instance, grpc_server_registry, self._logger)
def run_iteration(self, instance): return execute_backfill_iteration(instance, self._logger)