class HDFSTensorboardManager(base.TensorboardManager):
"""
Store and tfevents files to HDFS.
"""
@util.preserve_random_state
def __init__(
self,
hdfs_url: str,
hdfs_path: str,
user: Optional[str] = None,
*args: Any,
**kwargs: Any,
) -> None:
super().__init__(*args, **kwargs)
self.hdfs_url = hdfs_url
self.hdfs_path = hdfs_path
self.user = user
self.client = InsecureClient(self.hdfs_url,
root=self.hdfs_path,
user=self.user)
self.client.makedirs(str(self.sync_path))
@util.preserve_random_state
def sync(self) -> None:
for path in self.to_sync():
file_name = str(self.sync_path.joinpath(path.name))
logging.debug(f"Uploading {path} to {self.hdfs_path}")
self.client.upload(file_name, str(path))
def delete(self) -> None:
self.client.delete(self.sync_path, recursive=True)
class HDFSTensorboardManager(base.TensorboardManager):
"""
Store and tfevents files to HDFS.
"""
@util.preserve_random_state
def __init__(
self,
hdfs_url: str,
hdfs_path: str,
user: Optional[str] = None,
*args: Any,
**kwargs: Any,
) -> None:
super().__init__(*args, **kwargs)
self.hdfs_url = hdfs_url
self.hdfs_path = hdfs_path
self.user = user
self.client = InsecureClient(self.hdfs_url,
root=self.hdfs_path,
user=self.user)
self.client.makedirs(str(self.sync_path))
@util.preserve_random_state
def sync(
self,
selector: Callable[[pathlib.Path], bool] = lambda _: True,
mangler: Callable[[pathlib.Path, int], pathlib.Path] = lambda p, __: p,
rank: int = 0,
) -> None:
for path in self.to_sync(selector):
relative_path = path.relative_to(self.base_path)
mangled_relative_path = mangler(relative_path, rank)
mangled_path = self.sync_path.joinpath(mangled_relative_path)
file_name = str(mangled_path)
logging.debug(f"Uploading {path} to {self.hdfs_path}")
self.client.upload(file_name, str(path))
def delete(self) -> None:
self.client.delete(self.sync_path, recursive=True)