def _get_fs(self, path):
scheme = self._get_scheme(path)
if not scheme:
raise S3FileSystemException(
'Can not figure out scheme for path "%s"' % path)
try:
return self._fs_dict[scheme]
except KeyError:
raise S3FileSystemException(
'Unknown scheme %s, available schemes: %s' %
(scheme, self._fs_dict.keys()))
def get_s3_connection(self):
"""S3 connection can actually be seen as a S3Client. A true new client would be a Boto3Client."""
kwargs = {
'aws_access_key_id': self._access_key_id,
'aws_secret_access_key': self._secret_access_key,
'security_token': self._security_token,
'is_secure': self._is_secure,
'calling_format': self._calling_format
}
# Add proxy if configured
if self._proxy_address is not None:
kwargs.update({'proxy': self._proxy_address})
if self._proxy_port is not None:
kwargs.update({'proxy_port': self._proxy_port})
if self._proxy_user is not None:
kwargs.update({'proxy_user': self._proxy_user})
if self._proxy_pass is not None:
kwargs.update({'proxy_pass': self._proxy_pass})
# Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM
try:
# Use V4 signature support by default
os.environ['S3_USE_SIGV4'] = 'True'
if self._host is not None and not aws_conf.IS_SELF_SIGNING_ENABLED.get(
):
kwargs.update({'host': self._host})
connection = boto.s3.connection.S3Connection(**kwargs)
elif self._region:
if aws_conf.IS_SELF_SIGNING_ENABLED.get():
connection = url_client_connect_to_region(
self._region, **kwargs)
else:
connection = boto.s3.connect_to_region(
self._region, **kwargs)
else:
kwargs.update({'host': 's3.amazonaws.com'})
connection = boto.s3.connection.S3Connection(**kwargs)
except Exception as e:
LOG.exception(e)
raise S3FileSystemException(
'Failed to construct S3 Connection, check configurations for aws.'
)
if connection is None:
# If no connection, attempt to fallback to IAM instance metadata
connection = boto.connect_s3()
if connection is None:
raise S3FileSystemException(
'Can not construct S3 Connection for region %s' %
self._region)
return connection
def get_s3_connection(self):
# Use V4 signature support by default
os.environ['S3_USE_SIGV4'] = 'True'
kwargs = {
'aws_access_key_id': self._access_key_id,
'aws_secret_access_key': self._secret_access_key,
'security_token': self._security_token,
'is_secure': self._is_secure,
'calling_format': self._calling_format
}
# Add proxy if configured
if self._proxy_address is not None:
kwargs.update({'proxy': self._proxy_address})
if self._proxy_port is not None:
kwargs.update({'proxy_port': self._proxy_port})
if self._proxy_user is not None:
kwargs.update({'proxy_user': self._proxy_user})
if self._proxy_pass is not None:
kwargs.update({'proxy_pass': self._proxy_pass})
# Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM
try:
if self._host is not None:
kwargs.update({'host': self._host})
connection = boto.s3.connection.S3Connection(**kwargs)
elif self._region:
connection = boto.s3.connect_to_region(self._region, **kwargs)
else:
kwargs.update({'host': 's3.amazonaws.com'})
connection = boto.s3.connection.S3Connection(**kwargs)
except Exception, e:
LOG.exception(e)
raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.')
def _get_scheme(self):
if self.destination:
dst_parts = self.destination.split('://')
if dst_parts > 0:
return dst_parts[0].upper()
else:
raise S3FileSystemException('Destination does not start with a valid scheme.')
else:
return None
def _get_scheme(self, path):
if path.lower().startswith(S3A_ROOT):
from desktop.auth.backend import rewrite_user # Avoid cyclic loop
try:
user = User.objects.get(username=self.user)
if not has_s3_access(rewrite_user(user)):
raise S3FileSystemException(
"Missing permissions for %s on %s" % (
self.user,
path,
))
except User.DoesNotExist:
raise S3FileSystemException(
"Can't check permissions for %s on %s" % (self.user, path))
split = urlparse(path)
if split.scheme:
return split.scheme
if path and path[0] == posixpath.sep:
return self._default_scheme
def get_url_request(self, action='GET', **kwargs):
LOG.debug(kwargs)
signed_url = None
try:
# http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url
signed_url = self.generate_url(self.expiration, action, **kwargs)
LOG.debug('Generated url: %s' % signed_url)
except BotoClientError as e:
LOG.error(e)
if signed_url is None:
from aws.s3.s3fs import S3FileSystemException
raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs)
return signed_url
def _check_access(self):
if not self._fs.check_access(self.destination, permission='WRITE'):
raise S3FileSystemException(
'Insufficient permissions to write to S3 path "%s".' %
self.destination)
class Client(object):
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None,
timeout=HTTP_SOCKET_TIMEOUT_S, host=None, proxy_address=None, proxy_port=None, proxy_user=None,
proxy_pass=None, calling_format=None, is_secure=True):
self._access_key_id = aws_access_key_id
self._secret_access_key = aws_secret_access_key
self._security_token = aws_security_token
self._region = region.lower() if region is not None else None
self._timeout = timeout
self._host = host
self._proxy_address = proxy_address
self._proxy_port = proxy_port
self._proxy_user = proxy_user
self._proxy_pass = proxy_pass
self._calling_format = DEFAULT_CALLING_FORMAT if calling_format is None else calling_format
self._is_secure = is_secure
if not boto.config.has_section('Boto'):
boto.config.add_section('Boto')
if not boto.config.get('Boto', 'http_socket_timeout'):
boto.config.set('Boto', 'http_socket_timeout', str(self._timeout))
@classmethod
def from_config(cls, conf):
access_key_id = conf.ACCESS_KEY_ID.get()
secret_access_key = conf.SECRET_ACCESS_KEY.get()
security_token = conf.SECURITY_TOKEN.get()
env_cred_allowed = conf.ALLOW_ENVIRONMENT_CREDENTIALS.get()
if None in (access_key_id, secret_access_key) and not env_cred_allowed and not has_iam_metadata():
raise ValueError('Can\'t create AWS client, credential is not configured')
return cls(
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
aws_security_token=security_token,
region=get_default_region(),
host=conf.HOST.get(),
proxy_address=conf.PROXY_ADDRESS.get(),
proxy_port=conf.PROXY_PORT.get(),
proxy_user=conf.PROXY_USER.get(),
proxy_pass=conf.PROXY_PASS.get(),
calling_format=conf.CALLING_FORMAT.get(),
is_secure=conf.IS_SECURE.get()
)
def get_s3_connection(self):
kwargs = {
'aws_access_key_id': self._access_key_id,
'aws_secret_access_key': self._secret_access_key,
'security_token': self._security_token,
'is_secure': self._is_secure,
'calling_format': self._calling_format
}
# Add proxy if configured
if self._proxy_address is not None:
kwargs.update({'proxy': self._proxy_address})
if self._proxy_port is not None:
kwargs.update({'proxy_port': self._proxy_port})
if self._proxy_user is not None:
kwargs.update({'proxy_user': self._proxy_user})
if self._proxy_pass is not None:
kwargs.update({'proxy_pass': self._proxy_pass})
# Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM
try:
if self._host is not None:
# Use V4 signature support by default
os.environ['S3_USE_SIGV4'] = 'True'
kwargs.update({'host': self._host})
connection = boto.s3.connection.S3Connection(**kwargs)
elif self._region:
connection = boto.s3.connect_to_region(self._region,
aws_access_key_id=self._access_key_id,
aws_secret_access_key=self._secret_access_key,
security_token=self._security_token)
else:
kwargs.update({'host': 's3.amazonaws.com'})
connection = boto.s3.connection.S3Connection(**kwargs)
except Exception, e:
LOG.exception(e)
raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.')
if connection is None:
# If no connection, attemt to fallback to IAM instance metadata
connection = boto.connect_s3()
if connection is None:
raise S3FileSystemException('Can not construct S3 Connection for region %s' % self._region)
return connection