def _record(self, message: str, node_state: NodeState = NodeState.RUNNING): construct_and_record_rdzv_event( name=f"{self.__class__.__name__}.{get_method_name()}", run_id=self._settings.run_id, message=message, node_state=node_state, )
def test_construct_and_record_rdzv_event(self, get_mock, record_mock): get_mock.return_value = logging.StreamHandler() construct_and_record_rdzv_event( run_id="test_run_id", message="test_message", node_state=NodeState.RUNNING, ) record_mock.assert_called_once()
def test_construct_and_record_rdzv_event_does_not_run_if_invalid_dest(self, get_mock, record_mock): get_mock.return_value = logging.NullHandler() construct_and_record_rdzv_event( run_id="test_run_id", message="test_message", node_state=NodeState.RUNNING, ) record_mock.assert_not_called()
def _record(self, message: str, node_state: NodeState = NodeState.RUNNING) -> None: construct_and_record_rdzv_event( name=f"{self.__class__.__name__}.{get_method_name()}", run_id=self._settings.run_id, message=message, node_state=node_state, hostname=self._node.fqdn, pid=self._node.pid, local_id=self._node.local_id, )
def create_handler(store: Store, backend: RendezvousBackend, params: RendezvousParameters) -> DynamicRendezvousHandler: """Creates a new :py:class:`DynamicRendezvousHandler` from the specified parameters. Args: store: The C10d store to return as part of the rendezvous. backend: The backend to use to hold the rendezvous state. +-------------------+------------------------------------------------------+ | Parameter | Description | +===================+======================================================+ | join_timeout | The total time, in seconds, within which the | | | rendezvous is expected to complete. Defaults to 600 | | | seconds. | +-------------------+------------------------------------------------------+ | last_call_timeout | An additional wait amount, in seconds, before | | | completing the rendezvous once the minimum number of | | | nodes has been reached. Defaults to 30 seconds. | +-------------------+------------------------------------------------------+ | close_timeout | The time, in seconds, within which the rendezvous is | | | expected to close after a call to | | | :py:meth:`RendezvousHandler.set_closed` or | | | :py:meth:`RendezvousHandler.shutdown`. Defaults to | | | 30 seconds. | +-------------------+------------------------------------------------------+ """ try: timeout = RendezvousTimeout( _get_timeout(params, "join"), _get_timeout(params, "last_call"), _get_timeout(params, "close"), ) return DynamicRendezvousHandler.from_backend( params.run_id, store, backend, params.min_nodes, params.max_nodes, timeout, ) except Exception as e: construct_and_record_rdzv_event( message=f"{type(e).__name__}: {str(e)}", run_id=params.run_id, node_state=NodeState.FAILED, ) raise
def _create_tcp_store(params: RendezvousParameters) -> TCPStore: host, port = parse_rendezvous_endpoint(params.endpoint, default_port=29400) cfg_is_host = params.get_as_bool("is_host") # If the user has explicitly specified whether our process should host the # the store, respect it. if cfg_is_host is not None: is_host = cfg_is_host # Otherwise try to determine whether we are the host based on our hostname # and IP address. else: is_host = _matches_machine_hostname(host) # The timeout read_timeout = cast(int, params.get_as_int("read_timeout", 60)) if read_timeout <= 0: raise ValueError("The read timeout must be a positive integer.") # In specific cases we attempt to instantiate the store twice. For details # see the explanation in the except clause below. for is_server in [is_host, False]: try: store = TCPStore(host, port, is_master=is_server, timeout=timedelta(seconds=read_timeout)) if is_server: msg = f"Process {os.getpid()} hosts the TCP store for the C10d rendezvous backend." construct_and_record_rdzv_event(run_id=params.run_id, message=msg, node_state=NodeState.INIT) log.info(msg) break except (ValueError, RuntimeError, TimeoutError) as exc: # If we heuristically inferred the value of is_host as True and our # first attempt to instantiate the TCP store has failed, try it one # more time with is_host set to False. As an edge case there can be # more than one process that is part of the same rendezvous on this # machine and only one of them will eventually host the store. if not is_server or cfg_is_host is not None: raise RendezvousConnectionError( "The connection to the C10d store has failed. See inner exception for details." ) from exc return store
def create_backend( params: RendezvousParameters) -> Tuple[C10dRendezvousBackend, Store]: """Creates a new :py:class:`C10dRendezvousBackend` from the specified parameters. +--------------+-----------------------------------------------------------+ | Parameter | Description | +==============+===========================================================+ | store_type | The type of the C10d store. The currently supported types | | | are "tcp" and "file" which correspond to | | | :py:class:`torch.distributed.TCPStore` and | | | :py:class:`torch.distributed.FileStore`, respectively. | | | Defaults to "tcp". | +--------------+-----------------------------------------------------------+ | read_timeout | The read timeout, in seconds, for store operations. | | | Defaults to 60 seconds. | | | | | | Note this only applies to | | | :py:class:`torch.distributed.TCPStore`. It is not relevant| | | to :py:class:`torch.distributed.FileStore` which does not | | | take in timeout as a parameter. | +--------------+-----------------------------------------------------------+ | is_host | A boolean value indicating whether this backend instance | | | will host the C10d store. If not specified it will be | | | inferred heuristically by matching the hostname or the IP | | | address of this machine against the specified rendezvous | | | endpoint. Defaults to ``None``. | | | | | | Note that this configuration option only applies to | | | :py:class:`torch.distributed.TCPStore`. In normal | | | circumstances you can safely skip it; the only time when | | | it is needed is if its value cannot be correctly | | | determined (e.g. the rendezvous endpoint has a CNAME as | | | the hostname or does not match the FQDN of the machine). | +--------------+-----------------------------------------------------------+ """ # As of today we only support TCPStore and FileStore. Other store types do # not have the required functionality (e.g. compare_set) yet. store_type = params.get("store_type", "tcp").strip().lower() store: Store try: if store_type == "file": store = _create_file_store(params) elif store_type == "tcp": store = _create_tcp_store(params) else: raise ValueError( "Invalid store type given. Currently only supports file and tcp." ) backend = C10dRendezvousBackend(store, params.run_id) except Exception as e: construct_and_record_rdzv_event( message=f"{type(e).__name__}: {str(e)}", run_id=params.run_id, node_state=NodeState.FAILED, ) raise return backend, store