def setUp(self) -> None:
_, path = tempfile.mkstemp()
self._path = path
# Currently, filestore doesn't implement a delete_key method, so a new
# filestore has to be initialized for every test in order to have a
# clean slate.
self._store = FileStore(path)
self._backend = C10dRendezvousBackend(self._store, "dummy_run_id")
def _file_rendezvous_handler(url: str, **kwargs):
def _error(msg):
return _rendezvous_error("file:// rendezvous: " + msg)
result = urlparse(url)
path = result.path
if sys.platform == "win32":
import urllib.request
full_path = result.netloc + result.path
path = urllib.request.url2pathname(full_path)
if path:
# Normalizing an empty string produces ".", which is not expected.
path = os.path.normpath(path)
if not path:
raise _error("path missing")
query: Dict[str, str]
# mypy doesn't allow dict() to accept List of values (#257)
query = dict(
pair.split("=") for pair in filter(None, result.query.split(
"&"))) # type: ignore[misc, arg-type]
if "rank" not in query:
raise _error("rank parameter missing")
if "world_size" not in query:
raise _error("world size parameter missing")
rank = int(query["rank"])
world_size = int(query["world_size"])
store = FileStore(path, world_size)
yield (store, rank, world_size)
# If this configuration is invalidated, there is nothing we can do about it
raise RuntimeError("Unable to perform rerendezvous using file:// method")
def _file_rendezvous_handler(url: str, **kwargs):
def _error(msg):
return _rendezvous_error("file:// rendezvous: " + msg)
result = urlparse(url)
path = result.path
if sys.platform == "win32":
import urllib.request
full_path = result.netloc + result.path
path = urllib.request.url2pathname(full_path)
if path:
# Normalizing an empty string produces ".", which is not expected.
path = os.path.normpath(path)
if not path:
raise _error("path missing")
query_dict = _query_to_dict(result.query)
if "rank" not in query_dict:
raise _error("rank parameter missing")
if "world_size" not in query_dict:
raise _error("world size parameter missing")
rank = int(query_dict["rank"])
world_size = int(query_dict["world_size"])
store = FileStore(path, world_size)
yield (store, rank, world_size)
# If this configuration is invalidated, there is nothing we can do about it
raise RuntimeError("Unable to perform rerendezvous using file:// method")
class FileStoreBackendTest(TestCase, RendezvousBackendTestMixin):
_store: ClassVar[FileStore]
def setUp(self) -> None:
_, path = tempfile.mkstemp()
self._path = path
# Currently, filestore doesn't implement a delete_key method, so a new
# filestore has to be initialized for every test in order to have a
# clean slate.
self._store = FileStore(path)
self._backend = C10dRendezvousBackend(self._store, "dummy_run_id")
def tearDown(self) -> None:
os.remove(self._path)
def _corrupt_state(self) -> None:
self._store.set("torch.rendezvous.dummy_run_id", "non_base64")
def _create_file_store(params: RendezvousParameters) -> FileStore:
# If a user specifies an endpoint, we treat it as a path to a file.
if params.endpoint:
path = params.endpoint
else:
try:
# The temporary file is readable and writable only by the user of
# this process.
_, path = tempfile.mkstemp()
except OSError as exc:
raise RendezvousError(
"The file creation for C10d store has failed. See inner exception for details."
) from exc
try:
store = FileStore(path)
except (ValueError, RuntimeError) as exc:
raise RendezvousConnectionError(
"The connection to the C10d store has failed. See inner exception for details."
) from exc
return store