def checksum(self, path):
"""Looks up the checksum of an S3 object.
Args:
path: S3 file path pattern in the form s3://<bucket>/<name>.
"""
bucket, object_path = parse_s3_path(path)
request = messages.GetRequest(bucket, object_path)
item = self.client.get_object_metadata(request)
return item.etag
def test_get_range_bad_start_end(self):
file_name = self.TEST_DATA_PATH + 'get_range'
contents = os.urandom(1024)
with self.aws.open(file_name, 'w') as f:
f.write(contents)
bucket, object = s3io.parse_s3_path(file_name)
response = self.client.get_range(messages.GetRequest(bucket, object),
-10, 20)
self.assertEqual(response, contents)
response = self.client.get_range(messages.GetRequest(bucket, object),
20, 10)
self.assertEqual(response, contents)
# Clean up
self.aws.delete(file_name)
def _s3_object(self, path):
"""Returns a S3 object metadata for the given path
This method does not perform glob expansion. Hence the given path must be
for a single S3 object.
Returns: S3 object metadata.
"""
bucket, object = parse_s3_path(path)
request = messages.GetRequest(bucket, object)
return self.client.get_object_metadata(request)
def size(self, path):
"""Returns the size of a single S3 object.
This method does not perform glob expansion. Hence the given path must be
for a single S3 object.
Returns: size of the S3 object in bytes.
"""
bucket, object_path = parse_s3_path(path)
request = messages.GetRequest(bucket, object_path)
item = self.client.get_object_metadata(request)
return item.size
def last_updated(self, path):
"""Returns the last updated epoch time of a single S3 object.
This method does not perform glob expansion. Hence the given path must be
for a single S3 object.
Returns: last updated time of the S3 object in second.
"""
bucket, object = parse_s3_path(path)
request = messages.GetRequest(bucket, object)
datetime = self.client.get_object_metadata(request).last_modified
return (time.mktime(datetime.timetuple()) - time.timezone +
datetime.microsecond / 1000000.0)
def test_get_range_nonexistent(self):
# Test nonexistent object
object = self.test_path + 'nonexistent_file_doesnt_exist'
request = messages.GetRequest(self.test_bucket, object)
self.assertRaises(messages.S3ClientError, self.client.get_range,
request, 0, 10)
try:
self.client.get_range(request, 0, 10)
except Exception as e:
self.assertIsInstance(e, messages.S3ClientError)
self.assertEqual(e.code, 404)
def test_file_mime_type(self):
if self.USE_MOCK:
self.skipTest("The boto3_client mock doesn't support mime_types")
mime_type = 'example/example'
file_name = self.TEST_DATA_PATH + 'write_file'
f = self.aws.open(file_name, 'w', mime_type=mime_type)
f.write(b'a string of binary text')
f.close()
bucket, key = s3io.parse_s3_path(file_name)
metadata = self.client.get_object_metadata(messages.GetRequest(bucket, key))
self.assertEqual(mime_type, metadata.mime_type)
# Clean up
self.aws.delete(file_name)
def exists(self, path):
"""Returns whether the given S3 object exists.
Args:
path: S3 file path pattern in the form s3://<bucket>/<name>.
"""
bucket, object = parse_s3_path(path)
request = messages.GetRequest(bucket, object)
try:
self.client.get_object_metadata(request)
return True
except messages.S3ClientError as e:
if e.code == 404:
# HTTP 404 indicates that the file did not exist
return False
else:
# We re-raise all other exceptions
raise
def __init__(self, client, path, buffer_size):
self._client = client
self._path = path
self._bucket, self._name = parse_s3_path(path)
self._buffer_size = buffer_size
# Get object state.
self._get_request = (
messages.GetRequest(bucket=self._bucket, object=self._name))
try:
metadata = self._get_object_metadata(self._get_request)
except messages.S3ClientError as e:
if e.code == 404:
raise IOError(errno.ENOENT, 'Not found: %s' % self._path)
else:
logging.error('HTTP error while requesting file %s: %s', self._path, 3)
raise
self._size = metadata.size