def _watch_logs(self, pipeline_run, step_key=None):
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.opt_str_param(step_key, 'step_key')
key = self.get_key(pipeline_run, step_key)
outpath = self.get_local_path(pipeline_run.run_id, key, ComputeIOType.STDOUT)
errpath = self.get_local_path(pipeline_run.run_id, key, ComputeIOType.STDERR)
with mirror_stream_to_file(sys.stdout, outpath):
with mirror_stream_to_file(sys.stderr, errpath):
yield
def test_capture():
with get_temp_file_name() as capture_filepath:
with mirror_stream_to_file(sys.stdout, capture_filepath):
print("HELLO")
with open(capture_filepath, "r") as capture_stream:
assert "HELLO" in capture_stream.read()
import sys
import time
from dagster.core.execution.compute_logs import mirror_stream_to_file
from dagster.utils.interrupts import setup_windows_interrupt_support
if __name__ == "__main__":
stdout_filepath, stderr_filepath, opened_sentinel, interrupt_sentinel = (
sys.argv[1],
sys.argv[2],
sys.argv[3],
sys.argv[4],
)
setup_windows_interrupt_support()
with open(opened_sentinel, "w") as fd:
fd.write("opened_compute_log_subprocess")
with mirror_stream_to_file(sys.stdout, stdout_filepath) as stdout_pids:
with mirror_stream_to_file(sys.stderr, stderr_filepath) as stderr_pids:
sys.stdout.write(
"stdout pids: {pids}".format(pids=str(stdout_pids)))
sys.stdout.flush()
sys.stderr.write(
"stderr pids: {pids}".format(pids=str(stderr_pids)))
sys.stderr.flush()
try:
while True:
time.sleep(0.1)
except KeyboardInterrupt:
with open(interrupt_sentinel, "w") as fd:
fd.write("compute_log_subprocess_interrupt")
