def test_crash_after_submit(crash_signal, capfd): with instance_for_context(default_repo) as ( instance, _grpc_server_registry, external_repo, ): external_partition_set = external_repo.get_external_partition_set( "simple_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="simple", partition_set_origin=external_partition_set. get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), )) launch_process = spawn_ctx.Process( target=_test_backfill_in_subprocess, args=[instance.get_ref(), { "AFTER_SUBMIT": crash_signal }], ) launch_process.start() launch_process.join(timeout=60) assert launch_process.exitcode != 0 assert ( get_logger_output_from_capfd(capfd, "dagster.daemon.BackfillDaemon") == """2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple""" ) backfill = instance.get_backfill("simple") assert backfill.status == BulkActionStatus.REQUESTED assert instance.get_runs_count() == 3 # resume backfill launch_process = spawn_ctx.Process( target=_test_backfill_in_subprocess, args=[instance.get_ref(), None], ) launch_process.start() launch_process.join(timeout=60) assert ( get_logger_output_from_capfd(capfd, "dagster.daemon.BackfillDaemon") == """2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple 2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Found 3 existing runs for backfill simple, skipping 2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Backfill completed for simple for 3 partitions""" ) backfill = instance.get_backfill("simple") assert backfill.status == BulkActionStatus.COMPLETED assert instance.get_runs_count() == 3
def test_simple(external_repo_context, capfd): with instance_for_context(external_repo_context) as ( instance, _grpc_server_registry, external_repo, ): external_partition_set = external_repo.get_external_partition_set("simple_partition_set") instance.add_backfill( PartitionBackfill( backfill_id="simple", partition_set_origin=external_partition_set.get_external_origin(), status=BulkActionStatus.REQUESTED, partition_names=["one", "two", "three"], from_failure=False, reexecution_steps=None, tags=None, backfill_timestamp=pendulum.now().timestamp(), ) ) launch_process = multiprocessing.Process( target=_test_backfill_in_subprocess, args=[instance.get_ref(), None], ) launch_process.start() launch_process.join(timeout=60) backfill = instance.get_backfill("simple") assert backfill.status == BulkActionStatus.COMPLETED assert ( get_logger_output_from_capfd(capfd, "dagster.daemon.BackfillDaemon") == """2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple 2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Backfill completed for simple for 3 partitions""" )
def test_failure_before_run_created(crash_location, crash_signal, capfd): frozen_datetime = to_timezone( create_pendulum_time(year=2019, month=2, day=28, hour=0, minute=0, second=1, tz="UTC"), "US/Central", ) with instance_with_sensors() as ( instance, _grpc_server_registry, external_repo, ): with pendulum.test(frozen_datetime): external_sensor = external_repo.get_external_sensor("simple_sensor") instance.add_instigator_state( InstigatorState( external_sensor.get_external_origin(), InstigatorType.SENSOR, InstigatorStatus.RUNNING, ) ) # create a tick launch_process = spawn_ctx.Process( target=_test_launch_sensor_runs_in_subprocess, args=[instance.get_ref(), frozen_datetime, None], ) launch_process.start() launch_process.join(timeout=60) ticks = instance.get_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 1 assert ticks[0].status == TickStatus.SKIPPED capfd.readouterr() # create a starting tick, but crash debug_crash_flags = {external_sensor.name: {crash_location: crash_signal}} launch_process = spawn_ctx.Process( target=_test_launch_sensor_runs_in_subprocess, args=[instance.get_ref(), frozen_datetime.add(seconds=31), debug_crash_flags], ) launch_process.start() launch_process.join(timeout=60) assert launch_process.exitcode != 0 capfd.readouterr() ticks = instance.get_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 2 assert ticks[0].status == TickStatus.STARTED assert not int(ticks[0].timestamp) % 2 # skip condition for simple_sensor assert instance.get_runs_count() == 0 # create another tick, but ensure that the last evaluation time used is from the first, # successful tick rather than the failed tick launch_process = spawn_ctx.Process( target=_test_launch_sensor_runs_in_subprocess, args=[instance.get_ref(), frozen_datetime.add(seconds=62), None], ) launch_process.start() launch_process.join(timeout=60) assert launch_process.exitcode == 0 wait_for_all_runs_to_start(instance) assert instance.get_runs_count() == 1 run = instance.get_runs()[0] assert ( get_logger_output_from_capfd(capfd, "dagster.daemon.SensorDaemon") == f"""2019-02-27 18:01:03 -0600 - dagster.daemon.SensorDaemon - INFO - Checking for new runs for sensor: simple_sensor 2019-02-27 18:01:03 -0600 - dagster.daemon.SensorDaemon - INFO - Launching run for simple_sensor 2019-02-27 18:01:03 -0600 - dagster.daemon.SensorDaemon - INFO - Completed launch of run {run.run_id} for simple_sensor""" ) ticks = instance.get_ticks(external_sensor.get_external_origin_id()) assert len(ticks) == 3 assert ticks[0].status == TickStatus.SUCCESS