def test_server_socket():
with safe_tempfile_path() as skt:
server_process = open_server_process(port=None, socket=skt)
try:
assert DagsterGrpcClient(socket=skt).ping('foobar') == 'foobar'
finally:
interrupt_ipc_subprocess(server_process)
def test_interrupt_ipc_subprocess_grandchild():
with ExitStack() as context_stack:
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
child_interrupt_sentinel,
) = [
context_stack.enter_context(safe_tempfile_path()) for _ in range(4)
]
child_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__,
"parent_subprocess_with_interrupt_support.py"),
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
child_interrupt_sentinel,
])
wait_for_file(child_opened_sentinel)
wait_for_file(child_started_sentinel)
interrupt_ipc_subprocess(child_process)
wait_for_file(child_interrupt_sentinel)
with open(child_interrupt_sentinel, "r") as fd:
assert fd.read().startswith("received_keyboard_interrupt")
wait_for_file(parent_interrupt_sentinel)
with open(parent_interrupt_sentinel, "r") as fd:
assert fd.read().startswith("parent_received_keyboard_interrupt")
def execute_windows_tail(path, stream):
# Cannot use multiprocessing here because we already may be in a daemonized process
# Instead, invoke a thin script to poll a file and dump output to stdout. We pass the current
# pid so that the poll process kills itself if it becomes orphaned
poll_file = os.path.abspath(poll_compute_logs.__file__)
stream = stream if _fileno(stream) else None
with tempfile.TemporaryDirectory() as temp_dir:
ipc_output_file = os.path.join(
temp_dir,
"execute-windows-tail-{uuid}".format(uuid=uuid.uuid4().hex))
try:
tail_process = open_ipc_subprocess([
sys.executable, poll_file, path,
str(os.getpid()), ipc_output_file
],
stdout=stream)
yield (tail_process.pid, None)
finally:
if tail_process:
start_time = time.time()
while not os.path.isfile(ipc_output_file):
if time.time() - start_time > 15:
raise Exception(
"Timed out waiting for tail process to start")
time.sleep(1)
# Now that we know the tail process has started, tell it to terminate once there is
# nothing more to output
interrupt_ipc_subprocess(tail_process)
wait_for_process(tail_process)
def start_daemon(timeout=60):
p = open_ipc_subprocess(["dagster-daemon", "run", "--empty-workspace"])
try:
yield
finally:
interrupt_ipc_subprocess(p)
seven.wait_for_process(p, timeout=timeout)
def start_daemon():
p = open_ipc_subprocess(["dagster-daemon", "run"])
try:
yield
finally:
interrupt_ipc_subprocess(p)
seven.wait_for_process(p)
def test_interrupt_compute_log_tail_child(
windows_legacy_stdio_env, # pylint: disable=redefined-outer-name, unused-argument
):
with ExitStack() as context_stack:
(stdout_pids_file, stderr_pids_file, opened_sentinel, interrupt_sentinel) = [
context_stack.enter_context(safe_tempfile_path()) for _ in range(4)
]
child_process = open_ipc_subprocess(
[
sys.executable,
file_relative_path(__file__, "compute_log_subprocess.py"),
stdout_pids_file,
stderr_pids_file,
opened_sentinel,
interrupt_sentinel,
]
)
wait_for_file(opened_sentinel)
wait_for_file(stdout_pids_file)
wait_for_file(stderr_pids_file)
with open(opened_sentinel, "r") as opened_sentinel_fd:
assert opened_sentinel_fd.read().startswith("opened_compute_log_subprocess")
with open(stdout_pids_file, "r") as stdout_pids_fd:
stdout_pids_str = stdout_pids_fd.read()
assert stdout_pids_str.startswith("stdout pids:")
stdout_pids = list(
map(
lambda x: int(x) if x != "None" else None,
[x.strip("(),") for x in stdout_pids_str.split(" ")[2:]],
)
)
with open(stderr_pids_file, "r") as stderr_pids_fd:
stderr_pids_str = stderr_pids_fd.read()
assert stderr_pids_str.startswith("stderr pids:")
stderr_pids = list(
map(
lambda x: int(x) if x != "None" else None,
[x.strip("(),") for x in stderr_pids_str.split(" ")[2:]],
)
)
interrupt_ipc_subprocess(child_process)
for stdout_pid in stdout_pids:
if stdout_pid is not None:
wait_for_process(stdout_pid)
for stderr_pid in stderr_pids:
if stderr_pid is not None:
wait_for_process(stderr_pid)
wait_for_file(interrupt_sentinel)
with open(interrupt_sentinel, "r") as fd:
assert fd.read().startswith("compute_log_subprocess_interrupt")
def test_server_port():
port = find_free_port()
server_process = open_server_process(port=port, socket=None)
assert server_process is not None
try:
assert DagsterGrpcClient(port=port).ping('foobar') == 'foobar'
finally:
if server_process is not None:
interrupt_ipc_subprocess(server_process)
def start_daemon(timeout=60, workspace_file=None):
p = open_ipc_subprocess(
["dagster-daemon", "run"]
+ (["--python-file", workspace_file] if workspace_file else ["--empty-workspace"])
)
try:
yield
finally:
interrupt_ipc_subprocess(p)
seven.wait_for_process(p, timeout=timeout)
def test_interrupt_ipc_subprocess():
with safe_tempfile_path() as started_sentinel:
with safe_tempfile_path() as interrupt_sentinel:
sleepy_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__,
"subprocess_with_interrupt_support.py"),
started_sentinel,
interrupt_sentinel,
])
wait_for_file(started_sentinel)
interrupt_ipc_subprocess(sleepy_process)
wait_for_file(interrupt_sentinel)
with open(interrupt_sentinel, "r") as fd:
assert fd.read().startswith("received_keyboard_interrupt")
def terminate(self, run_id):
check.str_param(run_id, 'run_id')
process = self._get_process(run_id)
if not process:
return False
if not process.poll() is None:
return False
# Pipeline execution machinery is set up to gracefully
# terminate and report to instance on KeyboardInterrupt
interrupt_ipc_subprocess(process)
process.wait()
return True
def terminate(self, run_id):
check.str_param(run_id, "run_id")
process = self._get_process(run_id)
if not process:
return False
if not _is_alive(process):
return False
# Pipeline execution machinery is set up to gracefully
# terminate and report to instance on KeyboardInterrupt
interrupt_ipc_subprocess(process)
seven.wait_for_process(process, timeout=30)
return True
def open_server_process(port, socket):
check.invariant((port or socket) and not (port and socket),
'Set only port or socket')
server_process = open_ipc_subprocess(
['dagster', 'api', 'grpc'] + (['-p', str(port)] if port else []) +
(['-f', socket] if socket else []),
stdout=subprocess.PIPE,
)
ready = _wait_for_grpc_server(server_process)
if ready:
return server_process
else:
if server_process.poll() is None:
interrupt_ipc_subprocess(server_process)
return None
def terminate(self, run_id):
check.str_param(run_id, "run_id")
if not self._instance:
return False
run = self._instance.get_run_by_id(run_id)
if not run:
return False
self._instance.report_engine_event(
message="Received pipeline termination request.",
pipeline_run=run,
cls=self.__class__)
process = self._get_process(run_id)
if not process:
self._instance.report_engine_event(
message=
"Pipeline was not terminated since process is not found.",
pipeline_run=run,
cls=self.__class__,
)
return False
if not _is_alive(process):
self._instance.report_engine_event(
message=
"Pipeline was not terminated since process is not alive.",
pipeline_run=run,
cls=self.__class__,
)
return False
# Pipeline execution machinery is set up to gracefully
# terminate and report to instance on KeyboardInterrupt
interrupt_ipc_subprocess(process)
seven.wait_for_process(process, timeout=30)
self._instance.report_engine_event(
message="Pipeline was terminated successfully.",
pipeline_run=run,
cls=self.__class__)
return True
if __name__ == "__main__":
setup_interrupt_handlers()
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
stdout_pids_file,
stderr_pids_file,
child_interrupt_sentinel,
) = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], sys.argv[6])
child_process = open_ipc_subprocess(
[
sys.executable,
file_relative_path(__file__, "compute_log_subprocess.py"),
stdout_pids_file,
stderr_pids_file,
child_started_sentinel,
child_interrupt_sentinel,
]
)
with open(child_opened_sentinel, "w") as fd:
fd.write("opened_ipc_subprocess")
try:
while True:
time.sleep(0.1)
except KeyboardInterrupt:
interrupt_ipc_subprocess(child_process)
with open(parent_interrupt_sentinel, "w") as fd:
fd.write("parent_received_keyboard_interrupt")
def terminate_server_process(self):
if self._server_process is not None:
interrupt_ipc_subprocess(self._server_process)
self._server_process = None
def test_interrupt_compute_log_tail_grandchild(
windows_legacy_stdio_env, # pylint: disable=redefined-outer-name, unused-argument
):
with ExitStack() as context_stack:
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
stdout_pids_file,
stderr_pids_file,
child_interrupt_sentinel,
) = [
context_stack.enter_context(safe_tempfile_path()) for _ in range(6)
]
parent_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__, 'parent_compute_log_subprocess.py'),
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
stdout_pids_file,
stderr_pids_file,
child_interrupt_sentinel,
])
wait_for_file(child_opened_sentinel)
wait_for_file(child_started_sentinel)
wait_for_file(stdout_pids_file)
with open(stdout_pids_file, 'r') as stdout_pids_fd:
stdout_pids_str = stdout_pids_fd.read()
assert stdout_pids_str.startswith('stdout pids:')
stdout_pids = list(
map(
lambda x: int(x) if x != 'None' else None,
[x.strip('(),') for x in stdout_pids_str.split(' ')[2:]],
))
wait_for_file(stderr_pids_file)
with open(stderr_pids_file, 'r') as stderr_pids_fd:
stderr_pids_str = stderr_pids_fd.read()
assert stderr_pids_str.startswith('stderr pids:')
stderr_pids = list(
map(
lambda x: int(x) if x != 'None' else None,
[x.strip('(),') for x in stderr_pids_str.split(' ')[2:]],
))
interrupt_ipc_subprocess(parent_process)
wait_for_file(child_interrupt_sentinel)
with open(child_interrupt_sentinel, 'r') as fd:
assert fd.read().startswith('compute_log_subprocess_interrupt')
wait_for_file(parent_interrupt_sentinel)
with open(parent_interrupt_sentinel, 'r') as fd:
assert fd.read().startswith('parent_received_keyboard_interrupt')
for stdout_pid in stdout_pids:
if stdout_pid is not None:
wait_for_process(stdout_pid)
for stderr_pid in stderr_pids:
if stderr_pid is not None:
wait_for_process(stderr_pid)
