def launch_run(self, instance, run, external_pipeline):
'''Subclasses must implement this method.'''
check.inst_param(run, 'run', PipelineRun)
check.inst_param(external_pipeline, 'external_pipeline',
ExternalPipeline)
# We use the IPC subprocess machinery here because we want to be able to interrupt this
# process. We could use multiprocessing and a thread to poll a shared multiprocessing.Event,
# interrupting when the event is set, but in this case that's six of one, half a dozen of
# the other.
process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__, 'sync_cli_api_execute_run.py'),
json.dumps({
'instance_ref':
serialize_dagster_namedtuple(self._instance.get_ref()),
'pipeline_origin':
serialize_dagster_namedtuple(external_pipeline.get_origin()),
'pipeline_run_id':
run.run_id,
}),
])
with self._processes_lock:
self._living_process_by_run_id[run.run_id] = process
return run
def cli_api_execute_run(output_file, instance, repo_cli_args, pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.str_param(repo_cli_args, 'repo_cli_args')
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
parts = ([
'dagster',
'api',
'execute_run',
output_file,
] + xplat_shlex_split(repo_cli_args) + [
'--instance-ref',
'{instance_ref}'.format(
instance_ref=serialize_dagster_namedtuple(instance.get_ref())),
'--pipeline-run',
'{pipeline_run}'.format(
pipeline_run=serialize_dagster_namedtuple(pipeline_run)),
])
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def test_interrupt_ipc_subprocess_grandchild():
with ExitStack() as context_stack:
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
child_interrupt_sentinel,
) = [
context_stack.enter_context(safe_tempfile_path()) for _ in range(4)
]
child_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__,
"parent_subprocess_with_interrupt_support.py"),
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
child_interrupt_sentinel,
])
wait_for_file(child_opened_sentinel)
wait_for_file(child_started_sentinel)
interrupt_ipc_subprocess(child_process)
wait_for_file(child_interrupt_sentinel)
with open(child_interrupt_sentinel, "r") as fd:
assert fd.read().startswith("received_keyboard_interrupt")
wait_for_file(parent_interrupt_sentinel)
with open(parent_interrupt_sentinel, "r") as fd:
assert fd.read().startswith("parent_received_keyboard_interrupt")
def start_daemon(timeout=60):
p = open_ipc_subprocess(["dagster-daemon", "run", "--empty-workspace"])
try:
yield
finally:
interrupt_ipc_subprocess(p)
seven.wait_for_process(p, timeout=timeout)
def _cli_api_execute_run_process(input_file, output_file, instance, pipeline_origin, pipeline_run):
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
'-m',
'dagster',
'api',
'execute_run',
input_file,
output_file,
]
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'.format(
pipeline_name=pipeline_run.pipeline_name, run_id=pipeline_run.run_id
),
pipeline_run,
engine_event_data=EngineEventData(marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
fixed_server_id=None,
startup_timeout=20,
):
check.invariant((port or socket) and not (port and socket), "Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.opt_int_param(max_workers, "max_workers")
from dagster.core.test_utils import get_mocked_system_timezone
mocked_system_timezone = get_mocked_system_timezone()
executable_path = loadable_target_origin.executable_path if loadable_target_origin else None
subprocess_args = (
(
get_python_environment_entry_point(executable_path)
if executable_path
else DEFAULT_DAGSTER_ENTRY_POINT
)
+ ["api", "grpc"]
+ ["--lazy-load-user-code"]
+ (["--port", str(port)] if port else [])
+ (["--socket", socket] if socket else [])
+ (["-n", str(max_workers)] if max_workers else [])
+ (["--heartbeat"] if heartbeat else [])
+ (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else [])
+ (["--fixed-server-id", fixed_server_id] if fixed_server_id else [])
+ (["--override-system-timezone", mocked_system_timezone] if mocked_system_timezone else [])
+ (["--log-level", "WARNING"]) # don't log INFO messages for automatically spun up servers
+ (["--use-python-environment-entry-point"] if executable_path else [])
)
if loadable_target_origin:
subprocess_args += loadable_target_origin.get_cli_args()
server_process = open_ipc_subprocess(subprocess_args)
from dagster.grpc.client import DagsterGrpcClient
client = DagsterGrpcClient(
port=port,
socket=socket,
host="localhost",
)
try:
wait_for_grpc_server(server_process, client, subprocess_args, timeout=startup_timeout)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def start_daemon():
p = open_ipc_subprocess(["dagster-daemon", "run"])
try:
yield
finally:
interrupt_ipc_subprocess(p)
seven.wait_for_process(p)
def execute_windows_tail(path, stream):
# Cannot use multiprocessing here because we already may be in a daemonized process
# Instead, invoke a thin script to poll a file and dump output to stdout. We pass the current
# pid so that the poll process kills itself if it becomes orphaned
poll_file = os.path.abspath(poll_compute_logs.__file__)
stream = stream if _fileno(stream) else None
with tempfile.TemporaryDirectory() as temp_dir:
ipc_output_file = os.path.join(
temp_dir,
"execute-windows-tail-{uuid}".format(uuid=uuid.uuid4().hex))
try:
tail_process = open_ipc_subprocess([
sys.executable, poll_file, path,
str(os.getpid()), ipc_output_file
],
stdout=stream)
yield (tail_process.pid, None)
finally:
if tail_process:
start_time = time.time()
while not os.path.isfile(ipc_output_file):
if time.time() - start_time > 15:
raise Exception(
"Timed out waiting for tail process to start")
time.sleep(1)
# Now that we know the tail process has started, tell it to terminate once there is
# nothing more to output
interrupt_ipc_subprocess(tail_process)
wait_for_process(tail_process)
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=1,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
):
check.invariant((port or socket) and not (port and socket), "Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.int_param(max_workers, "max_workers")
subprocess_args = (
[
loadable_target_origin.executable_path
if loadable_target_origin and loadable_target_origin.executable_path
else sys.executable,
"-m",
"dagster.grpc",
]
+ (["--port", str(port)] if port else [])
+ (["--socket", socket] if socket else [])
+ ["-n", str(max_workers)]
+ (["--heartbeat"] if heartbeat else [])
+ (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else [])
+ (["--lazy-load-user-code"] if lazy_load_user_code else [])
)
if loadable_target_origin:
subprocess_args += (
(
["-f", loadable_target_origin.python_file]
if loadable_target_origin.python_file
else []
)
+ (
["-m", loadable_target_origin.module_name]
if loadable_target_origin.module_name
else []
)
+ (
["-d", loadable_target_origin.working_directory]
if loadable_target_origin.working_directory
else []
)
+ (["-a", loadable_target_origin.attribute] if loadable_target_origin.attribute else [])
)
server_process = open_ipc_subprocess(subprocess_args, stdout=subprocess.PIPE)
ready = wait_for_grpc_server(server_process)
if ready:
return server_process
else:
if server_process.poll() is None:
server_process.terminate()
return None
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=1,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
fixed_server_id=None,
):
check.invariant((port or socket) and not (port and socket), "Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.int_param(max_workers, "max_workers")
from dagster.core.test_utils import get_mocked_system_timezone
with seven.TemporaryDirectory() as temp_dir:
output_file = os.path.join(
temp_dir, "grpc-server-startup-{uuid}".format(uuid=uuid.uuid4().hex)
)
mocked_system_timezone = get_mocked_system_timezone()
subprocess_args = (
[
loadable_target_origin.executable_path
if loadable_target_origin and loadable_target_origin.executable_path
else sys.executable,
"-m",
"dagster.grpc",
]
+ (["--port", str(port)] if port else [])
+ (["--socket", socket] if socket else [])
+ ["-n", str(max_workers)]
+ (["--heartbeat"] if heartbeat else [])
+ (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else [])
+ (["--lazy-load-user-code"] if lazy_load_user_code else [])
+ (["--ipc-output-file", output_file])
+ (["--fixed-server-id", fixed_server_id] if fixed_server_id else [])
+ (
["--override-system-timezone", mocked_system_timezone]
if mocked_system_timezone
else []
)
)
if loadable_target_origin:
subprocess_args += loadable_target_origin.get_cli_args()
server_process = open_ipc_subprocess(subprocess_args)
try:
wait_for_grpc_server(server_process, output_file)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def test_interrupt_compute_log_tail_child(
windows_legacy_stdio_env, # pylint: disable=redefined-outer-name, unused-argument
):
with ExitStack() as context_stack:
(stdout_pids_file, stderr_pids_file, opened_sentinel, interrupt_sentinel) = [
context_stack.enter_context(safe_tempfile_path()) for _ in range(4)
]
child_process = open_ipc_subprocess(
[
sys.executable,
file_relative_path(__file__, "compute_log_subprocess.py"),
stdout_pids_file,
stderr_pids_file,
opened_sentinel,
interrupt_sentinel,
]
)
wait_for_file(opened_sentinel)
wait_for_file(stdout_pids_file)
wait_for_file(stderr_pids_file)
with open(opened_sentinel, "r") as opened_sentinel_fd:
assert opened_sentinel_fd.read().startswith("opened_compute_log_subprocess")
with open(stdout_pids_file, "r") as stdout_pids_fd:
stdout_pids_str = stdout_pids_fd.read()
assert stdout_pids_str.startswith("stdout pids:")
stdout_pids = list(
map(
lambda x: int(x) if x != "None" else None,
[x.strip("(),") for x in stdout_pids_str.split(" ")[2:]],
)
)
with open(stderr_pids_file, "r") as stderr_pids_fd:
stderr_pids_str = stderr_pids_fd.read()
assert stderr_pids_str.startswith("stderr pids:")
stderr_pids = list(
map(
lambda x: int(x) if x != "None" else None,
[x.strip("(),") for x in stderr_pids_str.split(" ")[2:]],
)
)
interrupt_ipc_subprocess(child_process)
for stdout_pid in stdout_pids:
if stdout_pid is not None:
wait_for_process(stdout_pid)
for stderr_pid in stderr_pids:
if stderr_pid is not None:
wait_for_process(stderr_pid)
wait_for_file(interrupt_sentinel)
with open(interrupt_sentinel, "r") as fd:
assert fd.read().startswith("compute_log_subprocess_interrupt")
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
fixed_server_id=None,
):
check.invariant((port or socket) and not (port and socket),
"Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin",
LoadableTargetOrigin)
check.opt_int_param(max_workers, "max_workers")
from dagster.core.test_utils import get_mocked_system_timezone
mocked_system_timezone = get_mocked_system_timezone()
subprocess_args = (
[
loadable_target_origin.executable_path if loadable_target_origin
and loadable_target_origin.executable_path else sys.executable,
"-m",
"dagster.grpc",
] + ["--lazy-load-user-code"] +
(["--port", str(port)] if port else []) +
(["--socket", socket] if socket else []) +
(["-n", str(max_workers)] if max_workers else []) +
(["--heartbeat"] if heartbeat else []) +
(["--heartbeat-timeout", str(heartbeat_timeout)]
if heartbeat_timeout else []) +
(["--fixed-server-id", fixed_server_id] if fixed_server_id else []) +
(["--override-system-timezone", mocked_system_timezone]
if mocked_system_timezone else []))
if loadable_target_origin:
subprocess_args += loadable_target_origin.get_cli_args()
server_process = open_ipc_subprocess(subprocess_args)
from dagster.grpc.client import DagsterGrpcClient
client = DagsterGrpcClient(
port=port,
socket=socket,
host="localhost",
)
try:
wait_for_grpc_server(server_process, client, subprocess_args)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=1,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
):
check.invariant((port or socket) and not (port and socket),
"Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin",
LoadableTargetOrigin)
check.int_param(max_workers, "max_workers")
output_file = os.path.join(
get_system_temp_directory(),
"grpc-server-startup-{uuid}".format(uuid=uuid.uuid4().hex))
subprocess_args = (
[
loadable_target_origin.executable_path if loadable_target_origin
and loadable_target_origin.executable_path else sys.executable,
"-m",
"dagster.grpc",
] + (["--port", str(port)] if port else []) +
(["--socket", socket] if socket else []) +
["-n", str(max_workers)] + (["--heartbeat"] if heartbeat else []) +
(["--heartbeat-timeout", str(heartbeat_timeout)]
if heartbeat_timeout else []) +
(["--lazy-load-user-code"] if lazy_load_user_code else []) +
(["--ipc-output-file", output_file]))
if loadable_target_origin:
subprocess_args += ((([
"-f",
loadable_target_origin.python_file,
] + (["-d", loadable_target_origin.working_directory]
if loadable_target_origin.working_directory else
["--empty-working-directory"]))
if loadable_target_origin.python_file else []) +
(["-m", loadable_target_origin.module_name]
if loadable_target_origin.module_name else []) +
(["-a", loadable_target_origin.attribute]
if loadable_target_origin.attribute else []))
server_process = open_ipc_subprocess(subprocess_args)
try:
wait_for_grpc_server(output_file)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def start_daemon(timeout=60, workspace_file=None):
p = open_ipc_subprocess(
["dagster-daemon", "run"]
+ (["--python-file", workspace_file] if workspace_file else ["--empty-workspace"])
)
try:
yield
finally:
interrupt_ipc_subprocess(p)
seven.wait_for_process(p, timeout=timeout)
def test_interrupt_ipc_subprocess_by_pid():
with safe_tempfile_path() as started_sentinel:
with safe_tempfile_path() as interrupt_sentinel:
sleepy_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__,
"subprocess_with_interrupt_support.py"),
started_sentinel,
interrupt_sentinel,
])
wait_for_file(started_sentinel)
interrupt_ipc_subprocess_pid(sleepy_process.pid)
wait_for_file(interrupt_sentinel)
with open(interrupt_sentinel, "r") as fd:
assert fd.read().startswith("received_keyboard_interrupt")
def open_server_process(port, socket):
check.invariant((port or socket) and not (port and socket),
'Set only port or socket')
server_process = open_ipc_subprocess(
['dagster', 'api', 'grpc'] + (['-p', str(port)] if port else []) +
(['-f', socket] if socket else []),
stdout=subprocess.PIPE,
)
ready = _wait_for_grpc_server(server_process)
if ready:
return server_process
else:
if server_process.poll() is None:
interrupt_ipc_subprocess(server_process)
return None
def open_server_process(port,
socket,
loadable_target_origin=None,
max_workers=1,
heartbeat=False,
heartbeat_timeout=30):
check.invariant((port or socket) and not (port and socket),
'Set only port or socket')
check.opt_inst_param(loadable_target_origin, 'loadable_target_origin',
LoadableTargetOrigin)
check.int_param(max_workers, 'max_workers')
subprocess_args = ([
loadable_target_origin.executable_path if loadable_target_origin
and loadable_target_origin.executable_path else sys.executable,
'-m',
'dagster.grpc',
] + (['--port', str(port)] if port else []) +
(['--socket', socket] if socket else []) +
['-n', str(max_workers)] +
(['--heartbeat'] if heartbeat else []) +
(['--heartbeat-timeout',
str(heartbeat_timeout)] if heartbeat_timeout else []))
if loadable_target_origin:
subprocess_args += (
(['-f', loadable_target_origin.python_file]
if loadable_target_origin.python_file else []) +
(['-m', loadable_target_origin.module_name]
if loadable_target_origin.module_name else []) +
(['-d', loadable_target_origin.working_directory]
if loadable_target_origin.working_directory else []) +
(['-a', loadable_target_origin.attribute]
if loadable_target_origin.attribute else []))
server_process = open_ipc_subprocess(subprocess_args,
stdout=subprocess.PIPE)
ready = wait_for_grpc_server(server_process)
if ready:
return server_process
else:
if server_process.poll() is None:
server_process.terminate()
return None
def cli_api_execute_run(output_file, instance, repository_handle,
pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
pointer = repository_handle.get_pointer()
location_handle = repository_handle.repository_location_handle
if isinstance(location_handle, PythonEnvRepositoryLocationHandle):
executable_path = location_handle.executable_path
elif isinstance(location_handle, InProcessRepositoryLocationHandle):
# [legacy] default to using sys.executable for the in process
# location handle
executable_path = sys.executable
else:
raise DagsterInvariantViolationError(
"Unable to resolve executable_path for repository location handle of type {}"
.format(location_handle.__class__))
executable_path = (location_handle.executable_path if isinstance(
location_handle, PythonEnvRepositoryLocationHandle) else
sys.executable)
parts = (
[executable_path, '-m', 'dagster', 'api', 'execute_run', output_file] +
xplat_shlex_split(pointer.get_cli_args()) + [
'--instance-ref',
'{instance_ref}'.format(
instance_ref=serialize_dagster_namedtuple(instance.get_ref())),
'--pipeline-run-id',
'{pipeline_run_id}'.format(pipeline_run_id=pipeline_run.run_id),
])
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
return open_ipc_subprocess(parts)
def cli_api_execute_run(output_file, instance, pipeline_origin, pipeline_run):
check.str_param(output_file, 'output_file')
check.inst_param(instance, 'instance', DagsterInstance)
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
from dagster.cli.api import ExecuteRunArgsLoadComplete
with safe_tempfile_path() as input_file:
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
'-m',
'dagster',
'api',
'execute_run',
input_file,
output_file,
]
instance.report_engine_event(
'About to start process for pipeline "{pipeline_name}" (run_id: {run_id}).'
.format(pipeline_name=pipeline_run.pipeline_name,
run_id=pipeline_run.run_id),
pipeline_run,
engine_event_data=EngineEventData(
marker_start='cli_api_subprocess_init'),
)
process = open_ipc_subprocess(parts)
# we need to process this event in order to ensure that the called process loads the input
event = next(ipc_read_event_stream(output_file))
check.inst(event, ExecuteRunArgsLoadComplete)
return process
def test_segfault_compute_log_tail(
windows_legacy_stdio_env, # pylint: disable=redefined-outer-name, unused-argument
):
with safe_tempfile_path() as stdout_pids_file:
with safe_tempfile_path() as stderr_pids_file:
child_process = open_ipc_subprocess(
[
sys.executable,
file_relative_path(__file__, "compute_log_subprocess_segfault.py"),
stdout_pids_file,
stderr_pids_file,
]
)
child_process.wait()
wait_for_file(stdout_pids_file)
with open(stdout_pids_file, "r") as stdout_pids_fd:
stdout_pids_str = stdout_pids_fd.read()
assert stdout_pids_str.startswith("stdout pids:")
stdout_pids = list(
map(
lambda x: int(x) if x != "None" else None,
stdout_pids_str.split(" ")[-1].strip("()").split(","),
)
)
wait_for_file(stderr_pids_file)
with open(stderr_pids_file, "r") as stderr_pids_fd:
stderr_pids_str = stderr_pids_fd.read()
assert stderr_pids_str.startswith("stderr pids:")
stderr_pids = list(
map(
lambda x: int(x) if x != "None" else None,
stderr_pids_str.split(" ")[-1].strip("()").split(","),
)
)
for stdout_pid in stdout_pids:
if stdout_pid is not None:
wait_for_process(stdout_pid)
for stderr_pid in stderr_pids:
if stderr_pid is not None:
wait_for_process(stderr_pid)
def open_server_process(port, socket, python_executable_path=None):
check.invariant((port or socket) and not (port and socket),
'Set only port or socket')
python_executable_path = check.opt_str_param(python_executable_path,
'python_executable_path',
default=sys.executable)
server_process = open_ipc_subprocess(
[python_executable_path, '-m', 'dagster.grpc'] +
(['-p', str(port)] if port else []) +
(['-s', socket] if socket else []),
stdout=subprocess.PIPE,
)
ready = _wait_for_grpc_server(server_process)
if ready:
return server_process
else:
if server_process.poll() is None:
server_process.terminate()
return None
def _cli_api_execute_run_process(input_file, output_file, instance,
pipeline_origin, pipeline_run):
write_unary_input(
input_file,
ExecuteRunArgs(
pipeline_origin=pipeline_origin,
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
),
)
parts = [
pipeline_origin.executable_path,
"-m",
"dagster",
"api",
"execute_run",
input_file,
output_file,
]
return open_ipc_subprocess(parts)
import sys
import time
from dagster.serdes.ipc import interrupt_ipc_subprocess, open_ipc_subprocess
from dagster.utils import file_relative_path
from dagster.utils.interrupts import setup_interrupt_handlers
if __name__ == "__main__":
setup_interrupt_handlers()
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
child_interrupt_sentinel,
) = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
child_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__, "subprocess_with_interrupt_support.py"),
child_started_sentinel,
child_interrupt_sentinel,
])
with open(child_opened_sentinel, "w") as fd:
fd.write("opened_ipc_subprocess")
try:
while True:
time.sleep(0.1)
except KeyboardInterrupt:
interrupt_ipc_subprocess(child_process)
with open(parent_interrupt_sentinel, "w") as fd:
fd.write("parent_received_keyboard_interrupt")
if __name__ == "__main__":
setup_interrupt_handlers()
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
stdout_pids_file,
stderr_pids_file,
child_interrupt_sentinel,
) = (sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5], sys.argv[6])
child_process = open_ipc_subprocess(
[
sys.executable,
file_relative_path(__file__, "compute_log_subprocess.py"),
stdout_pids_file,
stderr_pids_file,
child_started_sentinel,
child_interrupt_sentinel,
]
)
with open(child_opened_sentinel, "w") as fd:
fd.write("opened_ipc_subprocess")
try:
while True:
time.sleep(0.1)
except KeyboardInterrupt:
interrupt_ipc_subprocess(child_process)
with open(parent_interrupt_sentinel, "w") as fd:
fd.write("parent_received_keyboard_interrupt")
def test_interrupt_compute_log_tail_grandchild(
windows_legacy_stdio_env, # pylint: disable=redefined-outer-name, unused-argument
):
with ExitStack() as context_stack:
(
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
stdout_pids_file,
stderr_pids_file,
child_interrupt_sentinel,
) = [
context_stack.enter_context(safe_tempfile_path()) for _ in range(6)
]
parent_process = open_ipc_subprocess([
sys.executable,
file_relative_path(__file__, 'parent_compute_log_subprocess.py'),
child_opened_sentinel,
parent_interrupt_sentinel,
child_started_sentinel,
stdout_pids_file,
stderr_pids_file,
child_interrupt_sentinel,
])
wait_for_file(child_opened_sentinel)
wait_for_file(child_started_sentinel)
wait_for_file(stdout_pids_file)
with open(stdout_pids_file, 'r') as stdout_pids_fd:
stdout_pids_str = stdout_pids_fd.read()
assert stdout_pids_str.startswith('stdout pids:')
stdout_pids = list(
map(
lambda x: int(x) if x != 'None' else None,
[x.strip('(),') for x in stdout_pids_str.split(' ')[2:]],
))
wait_for_file(stderr_pids_file)
with open(stderr_pids_file, 'r') as stderr_pids_fd:
stderr_pids_str = stderr_pids_fd.read()
assert stderr_pids_str.startswith('stderr pids:')
stderr_pids = list(
map(
lambda x: int(x) if x != 'None' else None,
[x.strip('(),') for x in stderr_pids_str.split(' ')[2:]],
))
interrupt_ipc_subprocess(parent_process)
wait_for_file(child_interrupt_sentinel)
with open(child_interrupt_sentinel, 'r') as fd:
assert fd.read().startswith('compute_log_subprocess_interrupt')
wait_for_file(parent_interrupt_sentinel)
with open(parent_interrupt_sentinel, 'r') as fd:
assert fd.read().startswith('parent_received_keyboard_interrupt')
for stdout_pid in stdout_pids:
if stdout_pid is not None:
wait_for_process(stdout_pid)
for stderr_pid in stderr_pids:
if stderr_pid is not None:
wait_for_process(stderr_pid)
