def _s3_open_file_with_retries(fs: s3fs.S3FileSystem, path: str,
retries: int) -> Any:
for _ in range(retries):
try:
logger.info(f"opening {path}")
file = fs.open(path)
return file
except Exception as ex:
logger.warning(f"could not open {path}: {ex}")
# if the file has just been uploaded, then it might not be visible immediatly
# but the fail to open has been cached by s3fs
# so, we invalidate the cache
fs.invalidate_cache(path)
# and we give some time to S3 to settle the file status
sleep(1)
def open(self, path, *args, **kwargs):
return S3FileSystem.open(self, get_key(path), *args, **kwargs)
def get_contents(path: str, client: S3FileSystem) -> Optional[str]:
if not client.exists(path):
return None
with client.open(path, "r") as fp:
return fp.read()
def write_contents(path: str, contents: str, client: S3FileSystem):
with client.open(path, "w") as fp:
fp.write(contents)
def open(self, path, mode='rb'):
s3_path = self._trim_filename(path)
f = S3FileSystem.open(self, s3_path, mode=mode)
return f
def open(self, path, mode='rb'):
s3_path = self._trim_filename(path)
f = S3FileSystem.open(self, s3_path, mode=mode)
return f
def open(self, path, mode='rb', **kwargs):
bucket = kwargs.pop('host', '')
s3_path = bucket + path
return S3FileSystem.open(self, s3_path, mode=mode)
