def create_s3_configuration(
access_server_port: int) -> storage.S3Configurations:
return storage.S3Configurations(
bucket="yogadl-test",
bucket_directory_path="integration-tests",
url=f"ws://localhost:{access_server_port}",
local_cache_dir="/tmp/",
)
def _configure_storage(self) -> None:
session_config = None # type: Optional[tf.compat.v1.ConfigProto]
if self._hvd_config.use:
# For multi-GPU training, we map processes to individual GPUs. TF requires
# that for each instantiation of `tf.Session`, the process is mapped
# to the same GPU.
session_config = tf.compat.v1.ConfigProto()
session_config.gpu_options.visible_device_list = str(
hvd.local_rank())
scheme = "wss" if self._env.use_tls else "ws"
rw_coordinator_url = (
f"{scheme}://{self._env.master_addr}:{self._env.master_port}/ws/data-layer/"
)
data_layer_type = self._env.experiment_config.get_data_layer_type()
if data_layer_type == StorageTypes.SHARED_FS.value:
local_cache_dir_path = self._env.experiment_config[
"data_layer"].get("container_storage_path")
local_cache_path = init_container_storage_path(
configured_storage_path=local_cache_dir_path)
storage_config = storage.LFSConfigurations(
storage_dir_path=str(local_cache_path))
self._storage = storage.LFSStorage(
storage_config, tensorflow_config=session_config)
elif data_layer_type == StorageTypes.S3.value:
local_cache_dir_path = self._env.experiment_config[
"data_layer"].get("local_cache_container_path")
local_cache_path = init_container_storage_path(
configured_storage_path=local_cache_dir_path)
storage_config = storage.S3Configurations(
bucket=self._env.experiment_config["data_layer"]["bucket"],
bucket_directory_path=self._env.experiment_config["data_layer"]
["bucket_directory_path"],
url=rw_coordinator_url,
local_cache_dir=str(local_cache_path),
access_key=self._env.experiment_config["data_layer"].get(
"access_key"),
secret_key=self._env.experiment_config["data_layer"].get(
"secret_key"),
endpoint_url=self._env.experiment_config["data_layer"].get(
"endpoint_url"),
coordinator_cert_file=self._env.master_cert_file,
coordinator_cert_name=self._env.master_cert_name,
)
self._storage = storage.S3Storage(storage_config,
tensorflow_config=session_config)
elif data_layer_type == StorageTypes.GCS.value:
local_cache_dir_path = self._env.experiment_config[
"data_layer"].get("local_cache_container_path")
local_cache_path = init_container_storage_path(
configured_storage_path=local_cache_dir_path)
storage_config = storage.GCSConfigurations(
bucket=self._env.experiment_config["data_layer"]["bucket"],
bucket_directory_path=self._env.experiment_config["data_layer"]
["bucket_directory_path"],
url=rw_coordinator_url,
local_cache_dir=str(local_cache_path),
coordinator_cert_file=self._env.master_cert_file,
coordinator_cert_name=self._env.master_cert_name,
)
self._storage = storage.GCSStorage(
storage_config, tensorflow_config=session_config)
else:
raise AssertionError(
"Please select a supported data_layer type. Supported types include: "
f"{[i.value for i in StorageTypes]}")