def _dequeue_run(self, run):
with external_pipeline_from_run(run) as external_pipeline:
# double check that the run is still queued before dequeing
reloaded_run = self._instance.get_run_by_id(run.run_id)
if reloaded_run.status != PipelineRunStatus.QUEUED:
self._logger.info(
"Run {run_id} is now {status} instead of QUEUED, skipping".
format(run_id=reloaded_run.run_id,
status=reloaded_run.status))
return
dequeued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=dequeued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def test_construct_log_string_for_event():
step_output_event = DagsterEvent(
event_type_value="STEP_OUTPUT",
pipeline_name="my_pipeline",
step_key="solid2",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepOutputData(
step_output_handle=StepOutputHandle("solid2", "result")),
message=
'Yielded output "result" of type "Any" for step "solid2". (Type check passed).',
pid=54348,
)
message_props = {
"dagster_event": step_output_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_output_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
assert (
construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props) ==
'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2". (Type check passed).'
)
def make_event(event_id):
return DagsterEvent(
event_type_value="ENGINE_EVENT",
pipeline_name="some_pipeline",
event_specific_data=EngineEventData(),
message=str(event_id),
)
def test_construct_log_string_for_event():
step_output_event = DagsterEvent(
event_type_value="STEP_OUTPUT",
pipeline_name="my_pipeline",
step_key="solid2",
solid_handle=NodeHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepOutputData(
step_output_handle=StepOutputHandle("solid2", "result")),
message=
'Yielded output "result" of type "Any" for step "solid2". (Type check passed).',
pid=54348,
)
logging_metadata = DagsterLoggingMetadata(
run_id="f79a8a93-27f1-41b5-b465-b35d0809b26d",
pipeline_name="my_pipeline")
dagster_message_props = DagsterMessageProps(
orig_message=step_output_event.message,
dagster_event=step_output_event,
)
assert (
construct_log_string(logging_metadata=logging_metadata,
message_props=dagster_message_props) ==
'my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_OUTPUT - Yielded output "result" of type "Any" for step "solid2". (Type check passed).'
)
def make_log_string(error):
step_failure_event = DagsterEvent(
event_type_value="STEP_FAILURE",
pipeline_name="my_pipeline",
step_key="solid2",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepFailureData(error=error,
user_failure_data=None),
message='Execution of step "solid2" failed.',
pid=54348,
)
message_props = {
"dagster_event": step_failure_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_failure_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
return construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props)
def attempt_to_launch_runs(self):
max_runs_to_launch = self._max_concurrent_runs - self._count_in_progress_runs(
)
# Possibly under 0 if runs were launched without queuing
if max_runs_to_launch <= 0:
return
runs = self._get_queued_runs(limit=max_runs_to_launch)
for run in runs:
with external_pipeline_from_run(run) as external_pipeline:
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def _dequeue_run(self, run):
with external_pipeline_from_run(run) as external_pipeline:
dequeued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=dequeued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def test_construct_log_string_with_error():
try:
raise ValueError("some error")
except ValueError:
error = serializable_error_info_from_exc_info(sys.exc_info())
step_failure_event = DagsterEvent(
event_type_value="STEP_FAILURE",
pipeline_name="my_pipeline",
step_key="solid2.compute",
solid_handle=SolidHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepFailureData(error=error,
user_failure_data=None),
message='Execution of step "solid2.compute" failed.',
pid=54348,
)
message_props = {
"dagster_event": step_failure_event,
"pipeline_name": "my_pipeline"
}
synth_props = {
"orig_message": step_failure_event.message,
"run_id": "f79a8a93-27f1-41b5-b465-b35d0809b26d",
}
log_string = construct_log_string(message_props=message_props,
logging_tags={},
synth_props=synth_props)
expected_start = textwrap.dedent("""
my_pipeline - f79a8a93-27f1-41b5-b465-b35d0809b26d - 54348 - STEP_FAILURE - Execution of step "solid2.compute" failed.
ValueError: some error
File "
""").strip()
assert log_string.startswith(expected_start)
def construct_step_failure_event_and_handle(pipeline_run, step_key, err, instance):
step_failure_event = DagsterEvent(
event_type_value=DagsterEventType.STEP_FAILURE.value,
pipeline_name=pipeline_run.pipeline_name,
step_key=step_key,
event_specific_data=StepFailureData(
error=serializable_error_info_from_exc_info(sys.exc_info()),
user_failure_data=UserFailureData(label="K8sError"),
),
)
event_record = EventLogEntry(
user_message=str(err),
level=logging.ERROR,
pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id,
error_info=None,
step_key=step_key,
timestamp=time.time(),
dagster_event=step_failure_event,
)
instance.handle_new_event(event_record)
return step_failure_event
def submit_run(self, pipeline_run, external_pipeline):
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.invariant(pipeline_run.status == PipelineRunStatus.NOT_STARTED)
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_ENQUEUED.value,
pipeline_name=pipeline_run.pipeline_name,
)
event_record = EventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
return self._instance.get_run_by_id(pipeline_run.run_id)
def run_iteration(self):
in_progress = self._count_in_progress_runs()
max_runs_to_launch = self._max_concurrent_runs - in_progress
# Possibly under 0 if runs were launched without queuing
if max_runs_to_launch <= 0:
self._logger.info(
"{} runs are currently in progress. Maximum is {}, won't launch more.".format(
in_progress, self._max_concurrent_runs
)
)
return
queued_runs = self._get_queued_runs(limit=max_runs_to_launch)
if not queued_runs:
self._logger.info("Poll returned no queued runs.")
else:
self._logger.info("Retrieved {} queued runs to launch.".format(len(queued_runs)))
for run in queued_runs:
with external_pipeline_from_run(run) as external_pipeline:
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = DagsterEventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
self._instance.launch_run(run.run_id, external_pipeline)
def make_log_string(error, error_source=None):
step_failure_event = DagsterEvent(
event_type_value="STEP_FAILURE",
pipeline_name="my_pipeline",
step_key="solid2",
solid_handle=NodeHandle("solid2", None),
step_kind_value="COMPUTE",
logging_tags={},
event_specific_data=StepFailureData(error=error,
user_failure_data=None,
error_source=error_source),
message='Execution of step "solid2" failed.',
pid=54348,
)
logging_metadata = DagsterLoggingMetadata(
run_id="f79a8a93-27f1-41b5-b465-b35d0809b26d",
pipeline_name="my_pipeline")
dagster_message_props = DagsterMessageProps(
orig_message=step_failure_event.message,
dagster_event=step_failure_event,
)
return construct_log_string(logging_metadata, dagster_message_props)
def submit_run(self, context: SubmitRunContext) -> PipelineRun:
pipeline_run = context.pipeline_run
check.invariant(pipeline_run.status == PipelineRunStatus.NOT_STARTED)
enqueued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_ENQUEUED.value,
pipeline_name=pipeline_run.pipeline_name,
)
event_record = EventLogEntry(
user_message="",
level=logging.INFO,
pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=enqueued_event,
)
self._instance.handle_new_event(event_record)
run = self._instance.get_run_by_id(pipeline_run.run_id)
if run is None:
check.failed(f"Failed to reload run {pipeline_run.run_id}")
return run
def _dequeue_run(self, instance, run, location_manager):
repository_location_origin = (
run.external_pipeline_origin.external_repository_origin.
repository_location_origin)
handle = location_manager.get_handle(repository_location_origin)
external_pipeline = external_pipeline_from_location_handle(
handle, run.external_pipeline_origin, run.solid_selection)
# double check that the run is still queued before dequeing
reloaded_run = instance.get_run_by_id(run.run_id)
if reloaded_run.status != PipelineRunStatus.QUEUED:
self._logger.info(
"Run {run_id} is now {status} instead of QUEUED, skipping".
format(run_id=reloaded_run.run_id, status=reloaded_run.status))
return
dequeued_event = DagsterEvent(
event_type_value=DagsterEventType.PIPELINE_DEQUEUED.value,
pipeline_name=run.pipeline_name,
)
event_record = EventRecord(
message="",
user_message="",
level=logging.INFO,
pipeline_name=run.pipeline_name,
run_id=run.run_id,
error_info=None,
timestamp=time.time(),
dagster_event=dequeued_event,
)
instance.handle_new_event(event_record)
instance.launch_run(run.run_id, external_pipeline)