def list_prefix(self, path):
"""Lists files matching the prefix.
Args:
path: S3 file path pattern in the form s3://<bucket>/[name].
Returns:
Dictionary of file name -> size.
"""
bucket, prefix = parse_s3_path(path, object_optional=True)
request = messages.ListRequest(bucket=bucket, prefix=prefix)
file_sizes = {}
counter = 0
start_time = time.time()
logging.info("Starting the size estimation of the input")
while True:
response = self.client.list(request)
for item in response.items:
file_name = 's3://%s/%s' % (bucket, item.key)
file_sizes[file_name] = item.size
counter += 1
if counter % 10000 == 0:
logging.info("Finished computing size of: %s files", len(file_sizes))
if response.next_token:
request.continuation_token = response.next_token
else:
break
logging.info("Finished listing %s files in %s seconds.",
counter, time.time() - start_time)
return file_sizes
def list_prefix(self, path):
"""Lists files matching the prefix.
Args:
path: S3 file path pattern in the form s3://<bucket>/[name].
Returns:
Dictionary of file name -> size.
"""
bucket, prefix = parse_s3_path(path, object_optional=True)
request = messages.ListRequest(bucket=bucket, prefix=prefix)
file_sizes = {}
counter = 0
start_time = time.time()
logging.info("Starting the size estimation of the input")
while True:
#The list operation will raise an exception
#when trying to list a nonexistent S3 path.
#This should not be an issue here.
#Ignore this exception or it will break the procedure.
try:
response = self.client.list(request)
except messages.S3ClientError as e:
if e.code == 404:
break
else:
raise e
for item in response.items:
file_name = 's3://%s/%s' % (bucket, item.key)
file_sizes[file_name] = item.size
counter += 1
if counter % 10000 == 0:
logging.info("Finished computing size of: %s files", len(file_sizes))
if response.next_token:
request.continuation_token = response.next_token
else:
break
logging.info(
"Finished listing %s files in %s seconds.",
counter,
time.time() - start_time)
return file_sizes
def list_prefix(self, path, with_metadata=False):
"""Lists files matching the prefix.
Args:
path: S3 file path pattern in the form s3://<bucket>/[name].
with_metadata: Experimental. Specify whether returns file metadata.
Returns:
If ``with_metadata`` is False: dict of file name -> size; if
``with_metadata`` is True: dict of file name -> tuple(size, timestamp).
"""
bucket, prefix = parse_s3_path(path, object_optional=True)
request = messages.ListRequest(bucket=bucket, prefix=prefix)
file_info = {}
counter = 0
start_time = time.time()
if with_metadata:
logging.info("Starting the file information of the input")
else:
logging.info("Starting the size estimation of the input")
while True:
#The list operation will raise an exception
#when trying to list a nonexistent S3 path.
#This should not be an issue here.
#Ignore this exception or it will break the procedure.
try:
response = self.client.list(request)
except messages.S3ClientError as e:
if e.code == 404:
break
else:
raise e
for item in response.items:
file_name = 's3://%s/%s' % (bucket, item.key)
if with_metadata:
file_info[file_name] = (item.size,
self._updated_to_seconds(
item.last_modified))
else:
file_info[file_name] = item.size
counter += 1
if counter % 10000 == 0:
if with_metadata:
logging.info(
"Finished computing file information of: %s files",
len(file_info))
else:
logging.info("Finished computing size of: %s files",
len(file_info))
if response.next_token:
request.continuation_token = response.next_token
else:
break
logging.info("Finished listing %s files in %s seconds.", counter,
time.time() - start_time)
return file_info