def _get_fs(self, path): scheme = self._get_scheme(path) if not scheme: raise S3FileSystemException( 'Can not figure out scheme for path "%s"' % path) try: return self._fs_dict[scheme] except KeyError: raise S3FileSystemException( 'Unknown scheme %s, available schemes: %s' % (scheme, self._fs_dict.keys()))
def get_s3_connection(self): """S3 connection can actually be seen as a S3Client. A true new client would be a Boto3Client.""" kwargs = { 'aws_access_key_id': self._access_key_id, 'aws_secret_access_key': self._secret_access_key, 'security_token': self._security_token, 'is_secure': self._is_secure, 'calling_format': self._calling_format } # Add proxy if configured if self._proxy_address is not None: kwargs.update({'proxy': self._proxy_address}) if self._proxy_port is not None: kwargs.update({'proxy_port': self._proxy_port}) if self._proxy_user is not None: kwargs.update({'proxy_user': self._proxy_user}) if self._proxy_pass is not None: kwargs.update({'proxy_pass': self._proxy_pass}) # Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM try: # Use V4 signature support by default os.environ['S3_USE_SIGV4'] = 'True' if self._host is not None and not aws_conf.IS_SELF_SIGNING_ENABLED.get( ): kwargs.update({'host': self._host}) connection = boto.s3.connection.S3Connection(**kwargs) elif self._region: if aws_conf.IS_SELF_SIGNING_ENABLED.get(): connection = url_client_connect_to_region( self._region, **kwargs) else: connection = boto.s3.connect_to_region( self._region, **kwargs) else: kwargs.update({'host': 's3.amazonaws.com'}) connection = boto.s3.connection.S3Connection(**kwargs) except Exception as e: LOG.exception(e) raise S3FileSystemException( 'Failed to construct S3 Connection, check configurations for aws.' ) if connection is None: # If no connection, attempt to fallback to IAM instance metadata connection = boto.connect_s3() if connection is None: raise S3FileSystemException( 'Can not construct S3 Connection for region %s' % self._region) return connection
def get_s3_connection(self): # Use V4 signature support by default os.environ['S3_USE_SIGV4'] = 'True' kwargs = { 'aws_access_key_id': self._access_key_id, 'aws_secret_access_key': self._secret_access_key, 'security_token': self._security_token, 'is_secure': self._is_secure, 'calling_format': self._calling_format } # Add proxy if configured if self._proxy_address is not None: kwargs.update({'proxy': self._proxy_address}) if self._proxy_port is not None: kwargs.update({'proxy_port': self._proxy_port}) if self._proxy_user is not None: kwargs.update({'proxy_user': self._proxy_user}) if self._proxy_pass is not None: kwargs.update({'proxy_pass': self._proxy_pass}) # Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM try: if self._host is not None: kwargs.update({'host': self._host}) connection = boto.s3.connection.S3Connection(**kwargs) elif self._region: connection = boto.s3.connect_to_region(self._region, **kwargs) else: kwargs.update({'host': 's3.amazonaws.com'}) connection = boto.s3.connection.S3Connection(**kwargs) except Exception, e: LOG.exception(e) raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.')
def _get_scheme(self): if self.destination: dst_parts = self.destination.split('://') if dst_parts > 0: return dst_parts[0].upper() else: raise S3FileSystemException('Destination does not start with a valid scheme.') else: return None
def _get_scheme(self, path): if path.lower().startswith(S3A_ROOT): from desktop.auth.backend import rewrite_user # Avoid cyclic loop try: user = User.objects.get(username=self.user) if not has_s3_access(rewrite_user(user)): raise S3FileSystemException( "Missing permissions for %s on %s" % ( self.user, path, )) except User.DoesNotExist: raise S3FileSystemException( "Can't check permissions for %s on %s" % (self.user, path)) split = urlparse(path) if split.scheme: return split.scheme if path and path[0] == posixpath.sep: return self._default_scheme
def get_url_request(self, action='GET', **kwargs): LOG.debug(kwargs) signed_url = None try: # http://boto.cloudhackers.com/en/latest/ref/s3.html#boto.s3.key.Key.generate_url signed_url = self.generate_url(self.expiration, action, **kwargs) LOG.debug('Generated url: %s' % signed_url) except BotoClientError as e: LOG.error(e) if signed_url is None: from aws.s3.s3fs import S3FileSystemException raise S3FileSystemException("Resource does not exist or permission missing : '%s'" % kwargs) return signed_url
def _check_access(self): if not self._fs.check_access(self.destination, permission='WRITE'): raise S3FileSystemException( 'Insufficient permissions to write to S3 path "%s".' % self.destination)
class Client(object): def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, aws_security_token=None, region=None, timeout=HTTP_SOCKET_TIMEOUT_S, host=None, proxy_address=None, proxy_port=None, proxy_user=None, proxy_pass=None, calling_format=None, is_secure=True): self._access_key_id = aws_access_key_id self._secret_access_key = aws_secret_access_key self._security_token = aws_security_token self._region = region.lower() if region is not None else None self._timeout = timeout self._host = host self._proxy_address = proxy_address self._proxy_port = proxy_port self._proxy_user = proxy_user self._proxy_pass = proxy_pass self._calling_format = DEFAULT_CALLING_FORMAT if calling_format is None else calling_format self._is_secure = is_secure if not boto.config.has_section('Boto'): boto.config.add_section('Boto') if not boto.config.get('Boto', 'http_socket_timeout'): boto.config.set('Boto', 'http_socket_timeout', str(self._timeout)) @classmethod def from_config(cls, conf): access_key_id = conf.ACCESS_KEY_ID.get() secret_access_key = conf.SECRET_ACCESS_KEY.get() security_token = conf.SECURITY_TOKEN.get() env_cred_allowed = conf.ALLOW_ENVIRONMENT_CREDENTIALS.get() if None in (access_key_id, secret_access_key) and not env_cred_allowed and not has_iam_metadata(): raise ValueError('Can\'t create AWS client, credential is not configured') return cls( aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key, aws_security_token=security_token, region=get_default_region(), host=conf.HOST.get(), proxy_address=conf.PROXY_ADDRESS.get(), proxy_port=conf.PROXY_PORT.get(), proxy_user=conf.PROXY_USER.get(), proxy_pass=conf.PROXY_PASS.get(), calling_format=conf.CALLING_FORMAT.get(), is_secure=conf.IS_SECURE.get() ) def get_s3_connection(self): kwargs = { 'aws_access_key_id': self._access_key_id, 'aws_secret_access_key': self._secret_access_key, 'security_token': self._security_token, 'is_secure': self._is_secure, 'calling_format': self._calling_format } # Add proxy if configured if self._proxy_address is not None: kwargs.update({'proxy': self._proxy_address}) if self._proxy_port is not None: kwargs.update({'proxy_port': self._proxy_port}) if self._proxy_user is not None: kwargs.update({'proxy_user': self._proxy_user}) if self._proxy_pass is not None: kwargs.update({'proxy_pass': self._proxy_pass}) # Attempt to create S3 connection based on configured credentials and host or region first, then fallback to IAM try: if self._host is not None: # Use V4 signature support by default os.environ['S3_USE_SIGV4'] = 'True' kwargs.update({'host': self._host}) connection = boto.s3.connection.S3Connection(**kwargs) elif self._region: connection = boto.s3.connect_to_region(self._region, aws_access_key_id=self._access_key_id, aws_secret_access_key=self._secret_access_key, security_token=self._security_token) else: kwargs.update({'host': 's3.amazonaws.com'}) connection = boto.s3.connection.S3Connection(**kwargs) except Exception, e: LOG.exception(e) raise S3FileSystemException('Failed to construct S3 Connection, check configurations for aws.') if connection is None: # If no connection, attemt to fallback to IAM instance metadata connection = boto.connect_s3() if connection is None: raise S3FileSystemException('Can not construct S3 Connection for region %s' % self._region) return connection