def check_response(self, resp, level, uri):
if resp is None:
raise InvalidUriError('\n'.join(textwrap.wrap(
'Attempt to get %s for "%s" failed. This can happen if '
'the URI refers to a non-existent object or if you meant to '
'operate on a directory (e.g., leaving off -R option on gsutil '
'cp, mv, or ls of a bucket)' % (level, uri), 80)))
def exists(self, headers=None):
"""Returns True if the object exists or False if it doesn't"""
if not self.object_name:
raise InvalidUriError('exists on object-less URI (%s)' % self.uri)
bucket = self.get_bucket()
key = bucket.get_key(self.object_name, headers=headers)
return bool(key)
def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
"""
Opens a connection to appropriate provider, depending on provider
portion of URI. Requires Credentials defined in boto config file (see
boto/pyami/config.py).
@type storage_uri: StorageUri
@param storage_uri: StorageUri specifying a bucket or a bucket+object
@rtype: L{AWSAuthConnection<fcu_boto.gs.connection.AWSAuthConnection>}
@return: A connection to storage service provider of the given URI.
"""
connection_args = dict(self.connection_args or ())
if (hasattr(self, 'suppress_consec_slashes') and
'suppress_consec_slashes' not in connection_args):
connection_args['suppress_consec_slashes'] = (
self.suppress_consec_slashes)
connection_args.update(kwargs)
if not self.connection:
if self.scheme in self.provider_pool:
self.connection = self.provider_pool[self.scheme]
elif self.scheme == 's3':
from fcu_boto.s3.connection import S3Connection
self.connection = S3Connection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'gs':
from fcu_boto.gs.connection import GSConnection
# Use OrdinaryCallingFormat instead of boto-default
# SubdomainCallingFormat because the latter changes the hostname
# that's checked during cert validation for HTTPS connections,
# which will fail cert validation (when cert validation is
# enabled).
#
# The same is not true for S3's HTTPS certificates. In fact,
# we don't want to do this for S3 because S3 requires the
# subdomain to match the location of the bucket. If the proper
# subdomain is not used, the server will return a 301 redirect
# with no Location header.
#
# Note: the following import can't be moved up to the
# start of this file else it causes a config import failure when
# run from the resumable upload/download tests.
from fcu_boto.s3.connection import OrdinaryCallingFormat
connection_args['calling_format'] = OrdinaryCallingFormat()
self.connection = GSConnection(access_key_id,
secret_access_key,
**connection_args)
self.provider_pool[self.scheme] = self.connection
elif self.scheme == 'file':
from fcu_boto.file.connection import FileConnection
self.connection = FileConnection(self)
else:
raise InvalidUriError('Unrecognized scheme "%s"' %
self.scheme)
self.connection.debug = self.debug
return self.connection
def storage_uri_for_key(key):
"""Returns a StorageUri for the given key.
:type key: :class:`fcu_boto.s3.key.Key` or subclass
:param key: URI naming bucket + optional object.
"""
if not isinstance(key, fcu_boto.s3.key.Key):
raise InvalidUriError('Requested key (%s) is not a subclass of '
'fcu_boto.s3.key.Key' % str(type(key)))
prov_name = key.bucket.connection.provider.get_provider_name()
uri_str = '%s://%s/%s' % (prov_name, key.bucket.name, key.name)
return storage_uri(uri_str)
def _set_tracker_uri(self, uri):
"""
Called when we start a new resumable upload or get a new tracker
URI for the upload. Saves URI and resets upload state.
Raises InvalidUriError if URI is syntactically invalid.
"""
parse_result = urlparse.urlparse(uri)
if (parse_result.scheme.lower() not in ['http', 'https'] or
not parse_result.netloc):
raise InvalidUriError('Invalid tracker URI (%s)' % uri)
self.tracker_uri = uri
self.tracker_uri_host = parse_result.netloc
self.tracker_uri_path = '%s?%s' % (
parse_result.path, parse_result.query)
self.server_has_bytes = 0
def add_group_email_grant(self, permission, email_address, recursive=False,
validate=False, headers=None):
self._check_bucket_uri('add_group_email_grant')
if self.scheme != 'gs':
raise ValueError('add_group_email_grant() not supported for %s '
'URIs.' % self.scheme)
if self.object_name:
if recursive:
raise ValueError('add_group_email_grant() on key-ful URI cannot '
'specify recursive=True')
key = self.get_key(validate, headers)
self.check_response(key, 'key', self.uri)
key.add_group_email_grant(permission, email_address, headers)
elif self.bucket_name:
bucket = self.get_bucket(validate, headers)
bucket.add_group_email_grant(permission, email_address, recursive,
headers)
else:
raise InvalidUriError('add_group_email_grant() on bucket-less URI '
'%s' % self.uri)
def _check_object_uri(self, function_name):
if issubclass(type(self), BucketStorageUri) and not self.object_name:
raise InvalidUriError('%s on object-less URI (%s)' %
(function_name, self.uri))
def storage_uri(uri_str,
default_scheme='file',
debug=0,
validate=True,
bucket_storage_uri_class=BucketStorageUri,
suppress_consec_slashes=True,
is_latest=False):
"""
Instantiate a StorageUri from a URI string.
:type uri_str: string
:param uri_str: URI naming bucket + optional object.
:type default_scheme: string
:param default_scheme: default scheme for scheme-less URIs.
:type debug: int
:param debug: debug level to pass in to boto connection (range 0..2).
:type validate: bool
:param validate: whether to check for bucket name validity.
:type bucket_storage_uri_class: BucketStorageUri interface.
:param bucket_storage_uri_class: Allows mocking for unit tests.
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type is_latest: bool
:param is_latest: whether this versioned object represents the
current version.
We allow validate to be disabled to allow caller
to implement bucket-level wildcarding (outside the boto library;
see gsutil).
:rtype: :class:`fcu_boto.StorageUri` subclass
:return: StorageUri subclass for given URI.
``uri_str`` must be one of the following formats:
* gs://bucket/name
* gs://bucket/name#ver
* s3://bucket/name
* gs://bucket
* s3://bucket
* filename (which could be a Unix path like /a/b/c or a Windows path like
C:\a\b\c)
The last example uses the default scheme ('file', unless overridden).
"""
version_id = None
generation = None
# Manually parse URI components instead of using urlparse because
# what we're calling URIs don't really fit the standard syntax for URIs
# (the latter includes an optional host/net location part).
end_scheme_idx = uri_str.find('://')
if end_scheme_idx == -1:
scheme = default_scheme.lower()
path = uri_str
else:
scheme = uri_str[0:end_scheme_idx].lower()
path = uri_str[end_scheme_idx + 3:]
if scheme not in ['file', 's3', 'gs']:
raise InvalidUriError('Unrecognized scheme "%s"' % scheme)
if scheme == 'file':
# For file URIs we have no bucket name, and use the complete path
# (minus 'file://') as the object name.
is_stream = False
if path == '-':
is_stream = True
return FileStorageUri(path, debug, is_stream)
else:
path_parts = path.split('/', 1)
bucket_name = path_parts[0]
object_name = ''
# If validate enabled, ensure the bucket name is valid, to avoid
# possibly confusing other parts of the code. (For example if we didn't
# catch bucket names containing ':', when a user tried to connect to
# the server with that name they might get a confusing error about
# non-integer port numbers.)
if (validate and bucket_name
and (not BUCKET_NAME_RE.match(bucket_name)
or TOO_LONG_DNS_NAME_COMP.search(bucket_name))):
raise InvalidUriError('Invalid bucket name in URI "%s"' % uri_str)
if scheme == 'gs':
match = GENERATION_RE.search(path)
if match:
md = match.groupdict()
versionless_uri_str = md['versionless_uri_str']
path_parts = versionless_uri_str.split('/', 1)
generation = int(md['generation'])
elif scheme == 's3':
match = VERSION_RE.search(path)
if match:
md = match.groupdict()
versionless_uri_str = md['versionless_uri_str']
path_parts = versionless_uri_str.split('/', 1)
version_id = md['version_id']
else:
raise InvalidUriError('Unrecognized scheme "%s"' % scheme)
if len(path_parts) > 1:
object_name = path_parts[1]
return bucket_storage_uri_class(
scheme,
bucket_name,
object_name,
debug,
suppress_consec_slashes=suppress_consec_slashes,
version_id=version_id,
generation=generation,
is_latest=is_latest)