def raise_child_failure_error_fn(name, child_error_file=""):
if child_error_file:
_write_error(SentinelError("foobar"), child_error_file)
pf = ProcessFailure(local_rank=0,
pid=997,
exitcode=1,
error_file=child_error_file)
raise ChildFailedError(name, {0: pf})
def monres(state: WorkerState):
if state == WorkerState.SUCCEEDED:
return RunResult(state=state, return_values={0: 0}, failures={})
elif state in {WorkerState.UNHEALTHY, WorkerState.FAILED}:
pf = ProcessFailure(local_rank=0, pid=999, exitcode=1, error_file="<none>")
return RunResult(state=state, return_values={}, failures={0: pf})
else:
return RunResult(state=state)
def test_failure_incorrect_reply_file(self):
content = {"unknown_key": "unknown_value"}
with open(self.test_error_file, "w") as fp:
json.dump(content, fp)
with self.assertRaises(Exception):
ProcessFailure(
local_rank=0, pid=997, exitcode=1, error_file=self.test_error_file
)
def failure_with_error_file(self, exception):
with mock.patch.dict(
os.environ, {"TORCHELASTIC_ERROR_FILE": self.test_error_file}):
ErrorHandler().record_exception(exception)
return ProcessFailure(local_rank=0,
pid=997,
exitcode=1,
error_file=self.test_error_file)
def test_process_mast_error_format(self):
error_data = {"message": "test error message", "timestamp": "10"}
with open(self.test_error_file, "w") as fp:
json.dump(error_data, fp)
pf = ProcessFailure(
local_rank=0, pid=997, exitcode=1, error_file=self.test_error_file
)
self.assertEqual("test error message", pf.message)
self.assertEqual(10, pf.timestamp)
def raise_child_failure_error_fn(name, child_error_file=""):
if child_error_file:
with mock.patch.dict(os.environ,
{"TORCHELASTIC_ERROR_FILE": child_error_file}):
ErrorHandler().record_exception(SentinelError("foobar"))
pf = ProcessFailure(local_rank=0,
pid=997,
exitcode=1,
error_file=child_error_file)
raise ChildFailedError(name, {0: pf})
def _poll(self) -> Optional[RunProcsResult]:
done_local_ranks = set()
for local_rank in self._running_local_ranks:
handler = self.subprocess_handlers[local_rank]
exitcode = handler.proc.poll()
if exitcode is not None:
done_local_ranks.add(local_rank)
if exitcode != 0: # failed or signaled
self._failures[local_rank] = ProcessFailure(
local_rank=local_rank,
pid=handler.proc.pid,
exitcode=exitcode,
error_file=self.error_files[local_rank],
)
# else: --> succeeded; nothing to do
self._running_local_ranks.difference_update(done_local_ranks)
# if ALL procs are finished or ANY have failed
if not self._running_local_ranks or self._failures:
self.close() # terminate all running procs
result = RunProcsResult(
failures=self._failures,
stdouts=self.stdouts,
stderrs=self.stderrs,
)
if result.is_failed():
first_failure = min(result.failures.values(),
key=lambda f: f.timestamp)
log.error(
f"failed (exitcode: {first_failure.exitcode})"
f" local_rank: {first_failure.local_rank} (pid: {first_failure.pid})"
f" of binary: {self.entrypoint}")
else:
# Populate return with dummy values. This provides consistency with MultiprocessingHandler
result.return_values = {
local_rank: None
for local_rank in range(self.nprocs)
}
return result
else: # there are no failures and procs still running
return None
def failure_without_error_file(self, exitcode):
return ProcessFailure(local_rank=0,
pid=997,
exitcode=exitcode,
error_file="ignored.json")
def failure_with_error_file(self, exception):
_write_error(exception, self.test_error_file)
return ProcessFailure(local_rank=0,
pid=997,
exitcode=1,
error_file=self.test_error_file)
def _poll(self) -> Optional[RunProcsResult]:
assert self._pc is not None # assertion for mypy type checker
try:
# torch.mp.ProcessContext Throws an Exception if some/all of
# worker processes failed
# timeout < 0 checks worker status and return immediately
# Join will never return success since we use synchronize.Event to wait
# for all processes to finish.
self._pc.join(-1)
# IMPORTANT: we use multiprocessing.Queue to carry worker return values
# back to the parent, the worker process will wait before terminating
# until all the buffered items are fed by the feeder thread to the underlying
# pipe. Hence to prevent deadlocks on large return values,
# we opportunistically try queue.get on each join call
# See: https://docs.python.org/2/library/multiprocessing.html#all-platforms
for local_rank in range(0, self.nprocs):
return_queue = self._ret_vals[local_rank]
if not return_queue.empty():
# save the return values temporarily into a member var
self._return_values[local_rank] = return_queue.get()
if self._is_done():
# we should ALWAYS have ALL the return values when all the processes are done
self._worker_finished_event.set()
# Wait untill all processes are finished. At this point workers finished executing
# user function
self._pc.join()
_validate_full_rank(self._return_values, self.nprocs,
"return_value queue")
self.close()
return RunProcsResult(
return_values=self._return_values,
stdouts=self.stdouts,
stderrs=self.stderrs,
)
else:
return None
except (mp.ProcessRaisedException, mp.ProcessExitedException) as e:
failed_local_rank = e.error_index
# entrypoint for MultiprocessContext will always be a Callable
fn_name = self.entrypoint.__qualname__ # type: ignore[union-attr]
failed_proc = self._pc.processes[failed_local_rank]
error_filepath = self.error_files[failed_local_rank]
log.error(
f"failed (exitcode: {failed_proc.exitcode})"
f" local_rank: {failed_local_rank} (pid: {e.pid})"
f" of fn: {fn_name} (start_method: {self.start_method})",
exc_info=True,
)
self.close()
return RunProcsResult(
failures={
failed_local_rank:
ProcessFailure(
local_rank=failed_local_rank,
pid=e.pid,
exitcode=failed_proc.exitcode,
error_file=error_filepath,
)
},
stdouts=self.stdouts,
stderrs=self.stderrs,
)