def launch_step(self, step_context, prior_attempts_count):
step_run_ref = step_context_to_step_run_ref(step_context,
prior_attempts_count)
run_id = step_context.pipeline_run.run_id
step_run_dir = os.path.join(self.scratch_dir, run_id,
step_run_ref.step_key)
os.makedirs(step_run_dir)
step_run_ref_file_path = os.path.join(step_run_dir,
PICKLED_STEP_RUN_REF_FILE_NAME)
with open(step_run_ref_file_path, 'wb') as step_pickle_file:
pickle.dump(step_run_ref, step_pickle_file)
command_tokens = [
'python',
'-m',
'dagster.core.execution.plan.local_external_step_main',
step_run_ref_file_path,
]
subprocess.call(command_tokens, stdout=sys.stdout, stderr=sys.stderr)
events_file_path = os.path.join(step_run_dir, PICKLED_EVENTS_FILE_NAME)
file_manager = LocalFileManager('.')
events_file_handle = LocalFileHandle(events_file_path)
events_data = file_manager.read_data(events_file_handle)
events = pickle.loads(events_data)
for event in events:
yield event
def launch_step(
self,
step_context: StepExecutionContext,
prior_attempts_count: int,
) -> Iterator[DagsterEvent]:
step_run_ref = step_context_to_step_run_ref(step_context, prior_attempts_count)
run_id = step_context.pipeline_run.run_id
step_run_dir = os.path.join(self.scratch_dir, run_id, step_run_ref.step_key)
os.makedirs(step_run_dir)
step_run_ref_file_path = os.path.join(step_run_dir, PICKLED_STEP_RUN_REF_FILE_NAME)
with open(step_run_ref_file_path, "wb") as step_pickle_file:
pickle.dump(step_run_ref, step_pickle_file)
command_tokens = [
sys.executable,
"-m",
"dagster.core.execution.plan.local_external_step_main",
step_run_ref_file_path,
]
# If this is being called within a `capture_interrupts` context, allow interrupts
# while waiting for the subprocess to complete, so that we can terminate slow or
# hanging steps
with raise_execution_interrupts():
subprocess.call(command_tokens, stdout=sys.stdout, stderr=sys.stderr)
events_file_path = os.path.join(step_run_dir, PICKLED_EVENTS_FILE_NAME)
file_manager = LocalFileManager(".")
events_file_handle = LocalFileHandle(events_file_path)
events_data = file_manager.read_data(events_file_handle)
events = pickle.loads(events_data)
yield from events
def main(step_run_ref_path):
file_manager = LocalFileManager(".")
file_handle = LocalFileHandle(step_run_ref_path)
step_run_ref = pickle.loads(file_manager.read_data(file_handle))
events = list(run_step_from_ref(step_run_ref))
events_out_path = os.path.join(os.path.dirname(step_run_ref_path),
PICKLED_EVENTS_FILE_NAME)
with open(events_out_path, "wb") as events_file:
pickle.dump(events, events_file)
def main(step_run_ref_path: str) -> None:
file_manager = LocalFileManager(".")
file_handle = LocalFileHandle(step_run_ref_path)
step_run_ref = pickle.loads(file_manager.read_data(file_handle))
with DagsterInstance.ephemeral() as instance:
events = list(run_step_from_ref(step_run_ref, instance))
events_out_path = os.path.join(os.path.dirname(step_run_ref_path),
PICKLED_EVENTS_FILE_NAME)
with open(events_out_path, "wb") as events_file:
pickle.dump(events, events_file)
def main(step_run_ref_path: str) -> None:
file_manager = LocalFileManager(".")
file_handle = LocalFileHandle(step_run_ref_path)
step_run_ref = pickle.loads(file_manager.read_data(file_handle))
all_events: List[EventLogEntry] = []
try:
instance = external_instance_from_step_run_ref(
step_run_ref, event_listener_fn=all_events.append)
# consume entire step iterator
list(run_step_from_ref(step_run_ref, instance))
finally:
events_out_path = os.path.join(os.path.dirname(step_run_ref_path),
PICKLED_EVENTS_FILE_NAME)
with open(events_out_path, "wb") as events_file:
pickle.dump(serialize_value(all_events), events_file)