def test_event_log_get_stats_for_run(self, storage):
import math
enqueued_time = time.time()
launched_time = enqueued_time + 20
start_time = launched_time + 50
storage.store_event(
EventRecord(
None,
"message",
"debug",
"",
DEFAULT_RUN_ID,
enqueued_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_ENQUEUED.value,
"nonce",
),
))
storage.store_event(
EventRecord(
None,
"message",
"debug",
"",
DEFAULT_RUN_ID,
launched_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_STARTING.value,
"nonce",
),
))
storage.store_event(
EventRecord(
None,
"message",
"debug",
"",
DEFAULT_RUN_ID,
start_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_START.value,
"nonce",
),
))
assert math.isclose(
storage.get_stats_for_run(DEFAULT_RUN_ID).enqueued_time,
enqueued_time)
assert math.isclose(
storage.get_stats_for_run(DEFAULT_RUN_ID).launch_time,
launched_time)
assert math.isclose(
storage.get_stats_for_run(DEFAULT_RUN_ID).start_time, start_time)
def test_event_log_get_stats_for_run(event_storage_factory_cm_fn):
import math
with event_storage_factory_cm_fn() as storage:
enqueued_time = time.time()
launched_time = enqueued_time + 20
start_time = launched_time + 50
storage.store_event(
EventRecord(
None,
"message",
"debug",
"",
"foo",
enqueued_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_ENQUEUED.value,
"nonce",
),
))
storage.store_event(
EventRecord(
None,
"message",
"debug",
"",
"foo",
launched_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_STARTING.value,
"nonce",
),
))
storage.store_event(
EventRecord(
None,
"message",
"debug",
"",
"foo",
start_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_START.value,
"nonce",
),
))
assert math.isclose(
storage.get_stats_for_run("foo").enqueued_time, enqueued_time)
assert math.isclose(
storage.get_stats_for_run("foo").launch_time, launched_time)
assert math.isclose(
storage.get_stats_for_run("foo").start_time, start_time)
def test_correct_timezone(self, storage):
curr_time = time.time()
event = EventRecord(
None,
"Message2",
"debug",
"",
"foo",
curr_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_START.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
storage.store_event(event)
logs = storage.get_logs_for_run("foo")
assert len(logs) == 1
log = logs[0]
stats = storage.get_stats_for_run("foo")
assert int(log.timestamp) == int(stats.start_time)
assert int(log.timestamp) == int(curr_time)
def test_event_log_storage_store_with_multiple_runs(self, storage):
runs = ["foo", "bar", "baz"]
for run_id in runs:
assert len(storage.get_logs_for_run(run_id)) == 0
storage.store_event(
EventRecord(
None,
"Message2",
"debug",
"",
run_id,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.STEP_SUCCESS.value,
"nonce",
event_specific_data=StepSuccessData(duration_ms=100.0),
),
))
for run_id in runs:
assert len(storage.get_logs_for_run(run_id)) == 1
assert storage.get_stats_for_run(run_id).steps_succeeded == 1
if self.can_wipe():
storage.wipe()
for run_id in runs:
assert len(storage.get_logs_for_run(run_id)) == 0
def _event_record(run_id,
solid_name,
timestamp,
event_type,
event_specific_data=None):
pipeline_name = "pipeline_name"
solid_handle = SolidHandle(solid_name, None)
step_handle = StepHandle(solid_handle)
return EventRecord(
None,
"",
"debug",
"",
run_id,
timestamp,
step_key=step_handle.to_key(),
pipeline_name=pipeline_name,
dagster_event=DagsterEvent(
event_type.value,
pipeline_name,
solid_handle=solid_handle,
step_handle=step_handle,
event_specific_data=event_specific_data,
),
)
def _dequeue_run(self, instance, run, location_manager):
external_pipeline = location_manager.get_external_pipeline_from_run(
run)
# double check that the run is still queued before dequeing
reloaded_run = instance.get_run_by_id(run.run_id)
if reloaded_run.status != PipelineRunStatus.QUEUED:
self._logger.info(
"Run {run_id} is now {status} instead of QUEUED, skipping".
format(run_id=reloaded_run.run_id, status=reloaded_run.status))
return
dequeued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = EventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=dequeued_event,
)
instance.handle_new_event(event_record)
instance.launch_run(run.run_id, external_pipeline)
def test_event_sink_serialization():
event_records = []
class TestEventSink(EventSink):
def on_dagster_event(self, dagster_event):
event_records.append(dagster_event)
def on_log_message(self, log_message):
event_records.append(log_message)
@lambda_solid
def no():
raise Exception('no')
@pipeline
def fails():
no()
sink = TestEventSink()
# basic success
execute_pipeline(define_simple(), run_config=RunConfig(event_sink=sink))
# basic failure
execute_pipeline(
fails,
run_config=RunConfig(event_sink=sink),
environment_dict={
'execution': {
'in_process': {
'config': {
'raise_on_error': False
}
}
}
},
)
# multiproc
execute_pipeline(
ExecutionTargetHandle.for_pipeline_fn(
define_simple).build_pipeline_definition(),
run_config=RunConfig(event_sink=sink),
environment_dict={
'storage': {
'filesystem': {}
},
'execution': {
'multiprocess': {}
}
},
)
# kitchen sink
execute_pipeline(many_events, run_config=RunConfig(event_sink=sink))
for dagster_event in event_records:
payload = dagster_event.to_json()
clone = EventRecord.from_json(payload)
assert clone == dagster_event
def create_test_event_log_record(message: str, run_id: str = DEFAULT_RUN_ID):
return EventRecord(
None,
message,
"debug",
"",
run_id,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def evt(name):
return EventRecord(
None,
name,
"debug",
"",
"foo",
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def create_event(count: int, run_id: str = RUN_ID):
return EventRecord(
None,
str(count),
"debug",
"",
run_id,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def _materialization_event_record(run_id, asset_key):
return EventRecord(
None,
"",
"debug",
"",
run_id,
time.time() - 25,
step_key="my_step_key",
pipeline_name="my_pipeline",
dagster_event=DagsterEvent(
DagsterEventType.ASSET_MATERIALIZATION.value,
"my_pipeline",
step_key="my_step_key",
event_specific_data=StepMaterializationData(AssetMaterialization(asset_key=asset_key)),
),
)
def test_event_log_storage_store_events_and_wipe(self, storage):
assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 0
storage.store_event(
EventRecord(
None,
"Message2",
"debug",
"",
DEFAULT_RUN_ID,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 1
assert storage.get_stats_for_run(DEFAULT_RUN_ID)
storage.wipe()
assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 0
def test_event_log_storage_store_events_and_wipe(event_storage_factory_cm_fn):
with event_storage_factory_cm_fn() as storage:
assert len(storage.get_logs_for_run("foo")) == 0
storage.store_event(
EventRecord(
None,
"Message2",
"debug",
"",
"foo",
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run("foo")) == 1
assert storage.get_stats_for_run("foo")
storage.wipe()
assert len(storage.get_logs_for_run("foo")) == 0
def submit_run(self, pipeline_run, external_pipeline):
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.invariant(pipeline_run.status == PipelineRunStatus.NOT_STARTED)
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_ENQUEUED.value,
pipeline_name=pipeline_run.pipeline_name,
)
event_record = EventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
return self._instance.get_run_by_id(pipeline_run.run_id)
def construct_step_failure_event_and_handle(pipeline_run, step_key, err, instance):
step_failure_event = DagsterEvent(
event_type_value=DagsterEventType.STEP_FAILURE.value,
pipeline_name=pipeline_run.pipeline_name,
step_key=step_key,
event_specific_data=StepFailureData(
error=serializable_error_info_from_exc_info(sys.exc_info()),
user_failure_data=UserFailureData(label="K8sError"),
),
)
event_record = EventRecord(
message=str(err),
user_message=str(err),
level=logging.ERROR,
pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id,
error_info=None,
step_key=step_key,
timestamp=time.time(),
dagster_event=step_failure_event,
)
instance.handle_new_event(event_record)
return step_failure_event
