def test_crash_after_submit(crash_signal, capfd):
with instance_for_context(default_repo) as (
instance,
_grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set(
"simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.
get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
))
launch_process = spawn_ctx.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), {
"AFTER_SUBMIT": crash_signal
}],
)
launch_process.start()
launch_process.join(timeout=60)
assert launch_process.exitcode != 0
assert (
get_logger_output_from_capfd(capfd,
"dagster.daemon.BackfillDaemon") ==
"""2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple"""
)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.REQUESTED
assert instance.get_runs_count() == 3
# resume backfill
launch_process = spawn_ctx.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), None],
)
launch_process.start()
launch_process.join(timeout=60)
assert (
get_logger_output_from_capfd(capfd,
"dagster.daemon.BackfillDaemon") ==
"""2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple
2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Found 3 existing runs for backfill simple, skipping
2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Backfill completed for simple for 3 partitions"""
)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.COMPLETED
assert instance.get_runs_count() == 3
def test_simple(external_repo_context, capfd):
with instance_for_context(external_repo_context) as (
instance,
_grpc_server_registry,
external_repo,
):
external_partition_set = external_repo.get_external_partition_set("simple_partition_set")
instance.add_backfill(
PartitionBackfill(
backfill_id="simple",
partition_set_origin=external_partition_set.get_external_origin(),
status=BulkActionStatus.REQUESTED,
partition_names=["one", "two", "three"],
from_failure=False,
reexecution_steps=None,
tags=None,
backfill_timestamp=pendulum.now().timestamp(),
)
)
launch_process = multiprocessing.Process(
target=_test_backfill_in_subprocess,
args=[instance.get_ref(), None],
)
launch_process.start()
launch_process.join(timeout=60)
backfill = instance.get_backfill("simple")
assert backfill.status == BulkActionStatus.COMPLETED
assert (
get_logger_output_from_capfd(capfd, "dagster.daemon.BackfillDaemon")
== """2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Starting backfill for simple
2021-02-16 18:00:00 -0600 - dagster.daemon.BackfillDaemon - INFO - Backfill completed for simple for 3 partitions"""
)
def test_failure_before_run_created(crash_location, crash_signal, capfd):
frozen_datetime = to_timezone(
create_pendulum_time(year=2019, month=2, day=28, hour=0, minute=0, second=1, tz="UTC"),
"US/Central",
)
with instance_with_sensors() as (
instance,
_grpc_server_registry,
external_repo,
):
with pendulum.test(frozen_datetime):
external_sensor = external_repo.get_external_sensor("simple_sensor")
instance.add_instigator_state(
InstigatorState(
external_sensor.get_external_origin(),
InstigatorType.SENSOR,
InstigatorStatus.RUNNING,
)
)
# create a tick
launch_process = spawn_ctx.Process(
target=_test_launch_sensor_runs_in_subprocess,
args=[instance.get_ref(), frozen_datetime, None],
)
launch_process.start()
launch_process.join(timeout=60)
ticks = instance.get_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 1
assert ticks[0].status == TickStatus.SKIPPED
capfd.readouterr()
# create a starting tick, but crash
debug_crash_flags = {external_sensor.name: {crash_location: crash_signal}}
launch_process = spawn_ctx.Process(
target=_test_launch_sensor_runs_in_subprocess,
args=[instance.get_ref(), frozen_datetime.add(seconds=31), debug_crash_flags],
)
launch_process.start()
launch_process.join(timeout=60)
assert launch_process.exitcode != 0
capfd.readouterr()
ticks = instance.get_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 2
assert ticks[0].status == TickStatus.STARTED
assert not int(ticks[0].timestamp) % 2 # skip condition for simple_sensor
assert instance.get_runs_count() == 0
# create another tick, but ensure that the last evaluation time used is from the first,
# successful tick rather than the failed tick
launch_process = spawn_ctx.Process(
target=_test_launch_sensor_runs_in_subprocess,
args=[instance.get_ref(), frozen_datetime.add(seconds=62), None],
)
launch_process.start()
launch_process.join(timeout=60)
assert launch_process.exitcode == 0
wait_for_all_runs_to_start(instance)
assert instance.get_runs_count() == 1
run = instance.get_runs()[0]
assert (
get_logger_output_from_capfd(capfd, "dagster.daemon.SensorDaemon")
== f"""2019-02-27 18:01:03 -0600 - dagster.daemon.SensorDaemon - INFO - Checking for new runs for sensor: simple_sensor
2019-02-27 18:01:03 -0600 - dagster.daemon.SensorDaemon - INFO - Launching run for simple_sensor
2019-02-27 18:01:03 -0600 - dagster.daemon.SensorDaemon - INFO - Completed launch of run {run.run_id} for simple_sensor"""
)
ticks = instance.get_ticks(external_sensor.get_external_origin_id())
assert len(ticks) == 3
assert ticks[0].status == TickStatus.SUCCESS
