def GetNewHttp(http_class=httplib2.Http, **kwargs):
"""Creates and returns a new httplib2.Http instance.
Args:
http_class: Optional custom Http class to use.
**kwargs: Arguments to pass to http_class constructor.
Returns:
An initialized httplib2.Http instance.
"""
##Get Proxy configuration from boto file, defaults are None, 0 and False
boto_proxy_config = {
'proxy_host': config.get('Boto', 'proxy', None),
'proxy_type': config.get('Boto', 'proxy_type', 'http'),
'proxy_port': config.getint('Boto', 'proxy_port'),
'proxy_user': config.get('Boto', 'proxy_user', None),
'proxy_pass': config.get('Boto', 'proxy_pass', None),
'proxy_rdns': config.get('Boto', 'proxy_rdns', None)
}
#Use SetProxyInfo to convert boto config to httplib2.proxyinfo object
proxy_info = SetProxyInfo(boto_proxy_config)
# Some installers don't package a certs file with httplib2, so use the
# one included with gsutil.
kwargs['ca_certs'] = GetCertsFile()
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
kwargs['timeout'] = SSL_TIMEOUT_SEC
http = http_class(proxy_info=proxy_info, **kwargs)
http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
return http
def proxy_ssl(self):
host = '%s:%d' % (self.host, self.port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.proxy, int(self.proxy_port)))
except:
raise
boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host)
sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
sock.sendall("User-Agent: %s\r\n" % UserAgent)
if self.proxy_user and self.proxy_pass:
for k, v in self.get_proxy_auth_header().items():
sock.sendall("%s: %s\r\n" % (k, v))
# See discussion about this config option at
# https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc
if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False):
sock.sendall("\r\n")
else:
sock.sendall("\r\n")
resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug)
resp.begin()
if resp.status != 200:
# Fake a socket error, use a code that make it obvious it hasn't
# been generated by the socket library
raise socket.error(-71,
"Error talking to HTTP proxy %s:%s: %s (%s)" %
(self.proxy, self.proxy_port, resp.status, resp.reason))
# We can safely close the response, it duped the original socket
resp.close()
h = httplib.HTTPConnection(host)
if self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
boto.log.debug("wrapping ssl socket for proxied connection; "
"CA certificate file=%s",
self.ca_certificates_file)
key_file = self.http_connection_kwargs.get('key_file', None)
cert_file = self.http_connection_kwargs.get('cert_file', None)
sslSock = ssl.wrap_socket(sock, keyfile=key_file,
certfile=cert_file,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.ca_certificates_file)
cert = sslSock.getpeercert()
hostname = self.host.split(':', 0)[0]
if not https_connection.ValidateCertificateHostname(cert, hostname):
raise https_connection.InvalidCertificateException(
hostname, cert, 'hostname mismatch')
else:
# Fallback for old Python without ssl.wrap_socket
if hasattr(httplib, 'ssl'):
sslSock = httplib.ssl.SSLSocket(sock)
else:
sslSock = socket.ssl(sock, None, None)
sslSock = httplib.FakeSocket(sock, sslSock)
# This is a bit unclean
h.sock = sslSock
return h
def __init__(self, logger):
"""Initializes config.
Args:
logger (logging.logger): gsutil logger.
"""
self.logger = logger
self.use_client_certificate = config.getbool('Credentials',
'use_client_certificate')
self.client_cert_path = None
self.client_cert_password = None
if not self.use_client_certificate:
# Don't spend time generating values gsutil won't need.
return
# Generates certificate and deletes it afterwards.
atexit.register(self._UnprovisionClientCert)
command_string = config.get('Credentials', 'cert_provider_command',
None)
if not command_string:
raise CertProvisionError('No cert provider detected.')
self.client_cert_path = os.path.join(gslib.GSUTIL_DIR, 'caa_cert.pem')
try:
# Certs provisioned using endpoint verification are stored as a
# single file holding both the public certificate and the private key.
self._ProvisionClientCert(command_string, self.client_cert_path)
except CertProvisionError as e:
self.logger.error('Failed to provision client certificate: %s' % e)
def GetNewHttp(http_class=httplib2.Http, **kwargs):
"""Creates and returns a new httplib2.Http instance.
Args:
http_class: Optional custom Http class to use.
**kwargs: Arguments to pass to http_class constructor.
Returns:
An initialized httplib2.Http instance.
"""
proxy_info = httplib2.ProxyInfo(
proxy_type=3,
proxy_host=boto.config.get('Boto', 'proxy', None),
proxy_port=boto.config.getint('Boto', 'proxy_port', 0),
proxy_user=boto.config.get('Boto', 'proxy_user', None),
proxy_pass=boto.config.get('Boto', 'proxy_pass', None),
proxy_rdns=boto.config.get('Boto', 'proxy_rdns', False))
# Some installers don't package a certs file with httplib2, so use the
# one included with gsutil.
kwargs['ca_certs'] = GetCertsFile()
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
kwargs['timeout'] = SSL_TIMEOUT
http = http_class(proxy_info=proxy_info, **kwargs)
http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
return http
def GetNewHttp(http_class=httplib2.Http, **kwargs):
"""Creates and returns a new httplib2.Http instance.
Args:
http_class: Optional custom Http class to use.
**kwargs: Arguments to pass to http_class constructor.
Returns:
An initialized httplib2.Http instance.
"""
proxy_info = httplib2.ProxyInfo(
proxy_type=3,
proxy_host=boto.config.get('Boto', 'proxy', None),
proxy_port=boto.config.getint('Boto', 'proxy_port', 0),
proxy_user=boto.config.get('Boto', 'proxy_user', None),
proxy_pass=boto.config.get('Boto', 'proxy_pass', None),
proxy_rdns=boto.config.get('Boto', 'proxy_rdns', False))
# Some installers don't package a certs file with httplib2, so use the
# one included with gsutil.
kwargs['ca_certs'] = GetCertsFile()
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
kwargs['timeout'] = SSL_TIMEOUT
http = http_class(proxy_info=proxy_info, **kwargs)
http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
return http
def SetProxyInfo():
"""Sets proxy info from boto and environment and converts to httplib2.ProxyInfo.
Args:
None.
Returns:
httplib2.ProxyInfo constructed from boto or environment variable string.
"""
#Defining proxy_type based on httplib2 library, accounting for None entry too.
proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3}
boto_proxy_val = config.get('Boto', 'proxy_type', None)
#proxy_type defaults to 'http (3)' for backwards compatibility
proxy_type = proxy_type_spec.get(boto_proxy_val) or proxy_type_spec['http']
proxy_host = config.get('Boto', 'proxy', None)
#For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use
#to http only
proxy_info = httplib2.ProxyInfo(
proxy_host=proxy_host,
proxy_type=proxy_type,
proxy_port=config.getint('Boto', 'proxy_port', 0),
proxy_user=config.get('Boto', 'proxy_user', None),
proxy_pass=config.get('Boto', 'proxy_pass', None),
proxy_rdns=config.getbool(
'Boto', 'proxy_rdns',
True if proxy_type == proxy_type_spec['http'] else False))
#Added to force socks proxies not to use rdns
if not (proxy_info.proxy_type == proxy_type_spec['http']):
proxy_info.proxy_rdns = False
if not (proxy_info.proxy_host and proxy_info.proxy_port):
# Fall back to using the environment variable. Use only http proxies.
for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
if proxy_env_var in os.environ and os.environ[proxy_env_var]:
proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
# Assume proxy_rnds is True if a proxy environment variable exists.
proxy_info.proxy_rdns = config.getbool('Boto', 'proxy_rdns',
True)
break
return proxy_info
def SetProxyInfo(boto_proxy_config):
"""Sets proxy info from boto and environment and converts to httplib2.ProxyInfo.
Args:
dict: Values read from the .boto file
Returns:
httplib2.ProxyInfo constructed from boto or environment variable string.
"""
#Defining proxy_type based on httplib2 library, accounting for None entry too.
proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3}
_proxy_type_val = boto_proxy_config.get('proxy_type').lower()
#proxy_type defaults to 'http (3)' for backwards compatibility
proxy_type = proxy_type_spec.get(
_proxy_type_val) or proxy_type_spec['http']
proxy_host = boto_proxy_config.get('proxy_host')
proxy_port = boto_proxy_config.get('proxy_port')
proxy_user = boto_proxy_config.get('proxy_user')
proxy_pass = boto_proxy_config.get('proxy_pass')
proxy_rdns = boto_proxy_config.get('proxy_rdns')
#For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use
#to http only
proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host,
proxy_type=proxy_type,
proxy_port=proxy_port,
proxy_user=proxy_user,
proxy_pass=proxy_pass,
proxy_rdns=proxy_rdns)
#Added to force socks proxies not to use rdns, and set default rdns for https to true
if boto_proxy_config.get('proxy_rdns') == None:
if (proxy_info.proxy_type == proxy_type_spec['http']):
proxy_info.proxy_rdns = True #Use rdns unless explicity set as False in boto
else:
proxy_info.proxy_rdns = False
#Added to force socks proxies not to use rdns
if not (proxy_info.proxy_type == proxy_type_spec['http']):
proxy_info.proxy_rdns = False
if not (proxy_info.proxy_host and proxy_info.proxy_port):
# Fall back to using the environment variable. Use only http proxies.
for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
if proxy_env_var in os.environ and os.environ[proxy_env_var]:
proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
# Assume proxy_rnds is True if a proxy environment variable exists.
proxy_info.proxy_rdns = config.getbool('Boto', 'proxy_rdns',
True)
break
return proxy_info
def _iter_storages():
directory = os.environ.get('SCCACHE_DIR')
if directory:
yield LocalStorage(directory)
bucket_name = os.environ.get('SCCACHE_BUCKET')
if bucket_name:
storage = BotoStorage(bucket_name,
dns_server=os.environ.get('SCCACHE_NAMESERVER'))
yield storage
if not isinstance(storage, S3Storage):
from boto import config
if config.getbool('s3', 'fallback', False):
yield S3Storage(bucket_name,
dns_server=os.environ.get('SCCACHE_NAMESERVER'))
def _iter_storages():
directory = os.environ.get('SCCACHE_DIR')
if directory:
yield LocalStorage(directory)
bucket_name = os.environ.get('SCCACHE_BUCKET')
print 'HEY', bucket_name
if bucket_name:
storage = BotoStorage(
bucket_name, dns_server=os.environ.get('SCCACHE_NAMESERVER'))
yield storage
if not isinstance(storage, S3Storage):
from boto import config
if config.getbool('s3', 'fallback', False):
yield S3Storage(
bucket_name,
dns_server=os.environ.get('SCCACHE_NAMESERVER'))
class TestMtls(testcase.GsUtilIntegrationTestCase):
"""Integration tests for mTLS authentication."""
@unittest.skipIf(
not config.getbool('Credentials', 'use_client_certificate'),
'mTLS requires "use_client_certificate" to be "True" in .boto config.')
@integration_testcase.SkipForXML(MTLS_AVAILABILITY_MESSAGE)
@integration_testcase.SkipForS3(MTLS_AVAILABILITY_MESSAGE)
def test_can_list_bucket_with_mtls_authentication(self):
# Cannot use self.CreateBucket because testing framework's authentication
# doesn't work with mTLS.
bucket_uri = 'gs://{}'.format(self.MakeTempName('bucket'))
self.RunGsUtil(['mb', bucket_uri])
stdout = self.RunGsUtil(['-D', 'ls'], return_stdout=True)
self.RunGsUtil(['rb', bucket_uri])
# # Check if mTLS API endpoint was hit in debug output.
self.assertIn('storage.mtls.googleapis.com', stdout)
# # If bucket was successfully listed, it implies successful authentication.
self.assertIn(bucket_uri, stdout)
def __init__(self,
bucket_name,
host=DefaultHost,
calling_format=DefaultCallingFormat,
dns_server=None):
assert bucket_name
self._bucket_name = bucket_name
from boto.s3.connection import S3Connection
from boto.utils import find_class
from boto import config
self._calling_format = find_class(calling_format)()
self._host = self._calling_format.build_host(host, self._bucket_name)
self._failed = False
# Prepare the wrapper classes to use for urllib and boto.
dns_query = dns_query_function(dns_server)
self._http_connection_class = ConnectionWrapperFactory(
httplib.HTTPConnection, dns_query)
self._https_connection_class = ConnectionWrapperFactory(
httplib.HTTPSConnection, dns_query)
self._url_opener = OpenerFactory(self._http_connection_class,
self._https_connection_class)
# Get the boto S3 bucket instance
if config.getbool('Boto', 'is_secure', True):
s3_connection = S3Connection(
host=host,
port=443,
https_connection_factory=(self._https_connection_class, ()))
else:
s3_connection = S3Connection(
host=host,
port=80,
https_connection_factory=(self._http_connection_class, ()))
self._bucket = s3_connection.get_bucket(self._bucket_name,
validate=False)
self.last_stats = {}
def GetNewHttp(http_class=httplib2.Http, **kwargs):
"""Creates and returns a new httplib2.Http instance.
Args:
http_class: Optional custom Http class to use.
**kwargs: Arguments to pass to http_class constructor.
Returns:
An initialized httplib2.Http instance.
"""
proxy_host = config.get('Boto', 'proxy', None)
proxy_info = httplib2.ProxyInfo(
proxy_type=3,
proxy_host=proxy_host,
proxy_port=config.getint('Boto', 'proxy_port', 0),
proxy_user=config.get('Boto', 'proxy_user', None),
proxy_pass=config.get('Boto', 'proxy_pass', None),
proxy_rdns=config.get('Boto',
'proxy_rdns',
True if proxy_host else False))
if not (proxy_info.proxy_host and proxy_info.proxy_port):
# Fall back to using the environment variable.
for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
if proxy_env_var in os.environ and os.environ[proxy_env_var]:
proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
# Assume proxy_rnds is True if a proxy environment variable exists.
proxy_info.proxy_rdns = config.get('Boto', 'proxy_rdns', True)
break
# Some installers don't package a certs file with httplib2, so use the
# one included with gsutil.
kwargs['ca_certs'] = GetCertsFile()
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
kwargs['timeout'] = SSL_TIMEOUT_SEC
http = http_class(proxy_info=proxy_info, **kwargs)
http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
return http
def GetNewHttp(http_class=httplib2.Http, **kwargs):
"""Creates and returns a new httplib2.Http instance.
Args:
http_class: Optional custom Http class to use.
**kwargs: Arguments to pass to http_class constructor.
Returns:
An initialized httplib2.Http instance.
"""
# Get proxy info, will get None if no proxies are set in boto or
# environment variables
proxy_info = SetProxyInfo()
# Some installers don't package a certs file with httplib2, so use the
# one included with gsutil.
kwargs['ca_certs'] = GetCertsFile()
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
kwargs['timeout'] = SSL_TIMEOUT_SEC
http = http_class(proxy_info=proxy_info, **kwargs)
http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
return http
def GetNewHttp(http_class=httplib2.Http, **kwargs):
"""Creates and returns a new httplib2.Http instance.
Args:
http_class: Optional custom Http class to use.
**kwargs: Arguments to pass to http_class constructor.
Returns:
An initialized httplib2.Http instance.
"""
proxy_info = httplib2.ProxyInfo(
proxy_type=3,
proxy_host=boto.config.get('Boto', 'proxy', None),
proxy_port=boto.config.getint('Boto', 'proxy_port', 0),
proxy_user=boto.config.get('Boto', 'proxy_user', None),
proxy_pass=boto.config.get('Boto', 'proxy_pass', None),
proxy_rdns=boto.config.get('Boto', 'proxy_rdns', False))
if not (proxy_info.proxy_host and proxy_info.proxy_port):
# Fall back to using the environment variable.
for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
if proxy_env_var in os.environ and os.environ[proxy_env_var]:
proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
# Assume proxy_rnds is True if a proxy environment variable exists.
proxy_info.proxy_rdns = boto.config.get('Boto', 'proxy_rdns', True)
break
# Some installers don't package a certs file with httplib2, so use the
# one included with gsutil.
kwargs['ca_certs'] = GetCertsFile()
# Use a non-infinite SSL timeout to avoid hangs during network flakiness.
kwargs['timeout'] = SSL_TIMEOUT
http = http_class(proxy_info=proxy_info, **kwargs)
http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
return http
def __init__(self, bucket_name, host=DefaultHost,
calling_format=DefaultCallingFormat,
dns_server=None):
assert bucket_name
self._bucket_name = bucket_name
from boto.s3.connection import S3Connection
from boto.utils import find_class
from boto import config
self._calling_format = find_class(calling_format)()
self._host = self._calling_format.build_host(host, self._bucket_name)
self._failed = False
# Prepare the wrapper classes to use for urllib and boto.
dns_query = dns_query_function(dns_server)
self._http_connection_class = ConnectionWrapperFactory(
httplib.HTTPConnection, dns_query)
self._https_connection_class = ConnectionWrapperFactory(
httplib.HTTPSConnection, dns_query)
self._url_opener = OpenerFactory(
self._http_connection_class, self._https_connection_class)
# Get the boto S3 bucket instance
if config.getbool('Boto', 'is_secure', True):
s3_connection = S3Connection(host=host, port=443,
https_connection_factory=(self._https_connection_class, ()))
else:
s3_connection = S3Connection(host=host, port=80,
https_connection_factory=(self._http_connection_class, ()))
self._bucket = s3_connection.get_bucket(self._bucket_name,
validate=False)
self.last_stats = {}
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/',
provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value('Boto',
'ca_certificates_file',
DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider, aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __init__(self, host, aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/',
provider='aws', security_token=None,
suppress_consec_slashes=True,
validate_certs=True, profile_name=None):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:keyword str security_token: The security token associated with
temporary credentials issued by STS. Optional unless using
temporary credentials. If none is specified, the environment
variable ``AWS_SECURITY_TOKEN`` is used if defined.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch. The factory should have
a similar interface to L{http_client.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
:type suppress_consec_slashes: bool
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type validate_certs: bool
:param validate_certs: Controls whether SSL certificates
will be validated or not. Defaults to True.
:type profile_name: str
:param profile_name: Override usual Credentials section in config
file to use a named set of keys instead.
"""
self.suppress_consec_slashes = suppress_consec_slashes
self.num_retries = 6
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates.
# The default is now to validate certificates. This can be
# overridden in the boto config file are by passing an
# explicit validate_certs parameter to the class constructor.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates',
validate_certs)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
certs_file = config.get_value(
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
if certs_file == 'system':
certs_file = None
self.ca_certificates_file = certs_file
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from http_client that we want to catch and retry
self.http_exceptions = (http_client.HTTPException, socket.error,
socket.gaierror, http_client.BadStatusLine)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
# if the value passed in for debug
if not isinstance(debug, six.integer_types):
debug = 0
self.debug = config.getint('Boto', 'debug', debug)
self.host_header = None
# Timeout used to tell http_client how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
# If timeout isn't defined in boto config file, use 70 second
# default as recommended by
# http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html
self.http_connection_kwargs['timeout'] = config.getint(
'Boto', 'http_socket_timeout', 70)
if isinstance(provider, Provider):
# Allow overriding Provider
self.provider = provider
else:
self._provider_type = provider
self.provider = Provider(self._provider_type,
aws_access_key_id,
aws_secret_access_key,
security_token,
profile_name)
# Allow config file to override default host, port, and host header.
if self.provider.host:
self.host = self.provider.host
if self.provider.port:
self.port = self.provider.port
if self.provider.host_header:
self.host_header = self.provider.host_header
self._pool = ConnectionPool()
self._connection = (self.host, self.port, self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
if getattr(self, 'AuthServiceName', None) is not None:
self.auth_service_name = self.AuthServiceName
self.request_hook = None
def translate_to_gcloud_storage_if_requested(self):
"""Translates the gsutil command to gcloud storage equivalent.
The translated commands get stored at
self._translated_gcloud_storage_command.
This command also translate the boto config, which gets stored as a dict
at self._translated_env_variables
Returns:
True if the command was successfully translated, else False.
"""
if self.command_name == 'version' or self.command_name == 'test':
# Running any command in debug mode will lead to calling gsutil version
# command. We don't want to translate the version command as this
# should always reflect the version that gsutil is using.
# We don't want to run any translation for the "test" command.
return False
use_gcloud_storage = config.getbool('GSUtil', 'use_gcloud_storage',
False)
try:
hidden_shim_mode = HIDDEN_SHIM_MODE(
config.get('GSUtil', 'hidden_shim_mode', 'none'))
except ValueError:
raise exception.CommandException(
'Invalid option specified for'
' GSUtil:hidden_shim_mode config setting. Should be one of: {}'
.format(' | '.join([x.value for x in HIDDEN_SHIM_MODE])))
if use_gcloud_storage:
try:
top_level_flags, env_variables = self._translate_top_level_flags(
)
header_flags = self._translate_headers()
flags_from_boto, env_vars_from_boto = self._translate_boto_config(
)
env_variables.update(env_vars_from_boto)
gcloud_binary_path = _get_gcloud_binary_path()
gcloud_storage_command = ([gcloud_binary_path] +
self.get_gcloud_storage_args() +
top_level_flags + header_flags +
flags_from_boto)
if hidden_shim_mode == HIDDEN_SHIM_MODE.DRY_RUN:
self._print_gcloud_storage_command_info(
gcloud_storage_command, env_variables, dry_run=True)
elif not os.environ.get(
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL'):
raise exception.GcloudStorageTranslationError(
'Requested to use "gcloud storage" but gsutil is not using the'
' same credentials as gcloud.'
' You can make gsutil use the same credentials by running:\n'
'{} config set pass_credentials_to_gsutil True'.format(
gcloud_binary_path))
else:
self._print_gcloud_storage_command_info(
gcloud_storage_command, env_variables)
self._translated_gcloud_storage_command = gcloud_storage_command
self._translated_env_variables = env_variables
return True
except exception.GcloudStorageTranslationError as e:
# Raise error if no_fallback mode has been requested. This mode
# should only be used for debuggling and testing purposes.
if hidden_shim_mode == HIDDEN_SHIM_MODE.NO_FALLBACK:
raise exception.CommandException(e)
# For all other cases, we want to run gsutil.
self.logger.error(
'Cannot translate gsutil command to gcloud storage.'
' Going to run gsutil command. Error: %s', e)
return False
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value(
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider,
aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
self._pool = ConnectionPool()
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __init__(self, bucket_storage_uri_class, logger, provider=None,
credentials=None, debug=0):
"""Performs necessary setup for interacting with Google Cloud Storage.
Args:
bucket_storage_uri_class: Unused.
logger: logging.logger for outputting log messages.
provider: Unused. This implementation supports only Google Cloud Storage.
credentials: Credentials to be used for interacting with Google Cloud
Storage.
debug: Debug level for the API implementation (0..3).
"""
# TODO: Plumb host_header for perfdiag / test_perfdiag.
# TODO: Add jitter to apitools' http_wrapper retry mechanism.
super(GcsJsonApi, self).__init__(bucket_storage_uri_class, logger,
provider='gs', debug=debug)
no_op_credentials = False
if not credentials:
loaded_credentials = self._CheckAndGetCredentials(logger)
if not loaded_credentials:
loaded_credentials = NoOpCredentials()
no_op_credentials = True
else:
if isinstance(credentials, NoOpCredentials):
no_op_credentials = True
self.credentials = credentials or loaded_credentials
self.certs_file = GetCertsFile()
self.http = GetNewHttp()
self.http.disable_ssl_certificate_validation = (not config.getbool(
'Boto', 'https_validate_certificates'))
self.http_base = 'https://'
gs_json_host = config.get('Credentials', 'gs_json_host', None)
self.host_base = gs_json_host or 'www.googleapis.com'
if not gs_json_host:
gs_host = config.get('Credentials', 'gs_host', None)
if gs_host:
raise ArgumentException(
'JSON API is selected but gs_json_host is not configured, '
'while gs_host is configured to %s. Please also configure '
'gs_json_host and gs_json_port to match your desired endpoint.'
% gs_host)
gs_json_port = config.get('Credentials', 'gs_json_port', None)
if not gs_json_port:
gs_port = config.get('Credentials', 'gs_port', None)
if gs_port:
raise ArgumentException(
'JSON API is selected but gs_json_port is not configured, '
'while gs_port is configured to %s. Please also configure '
'gs_json_host and gs_json_port to match your desired endpoint.'
% gs_port)
self.host_port = ''
else:
self.host_port = ':' + config.get('Credentials', 'gs_json_port')
self.api_version = config.get('GSUtil', 'json_api_version',
DEFAULT_GCS_JSON_VERSION)
self.url_base = (self.http_base + self.host_base + self.host_port + '/' +
'storage/' + self.api_version + '/')
self.credentials.set_store(
multistore_file.get_credential_storage_custom_string_key(
GetCredentialStoreFilename(), self.api_version))
log_request = (debug >= 3)
log_response = (debug >= 3)
self.api_client = apitools_client.StorageV1(
url=self.url_base, http=self.http, log_request=log_request,
log_response=log_response, credentials=self.credentials,
version=self.api_version)
if no_op_credentials:
# This API key is not secret and is used to identify gsutil during
# anonymous requests.
self.api_client.AddGlobalParam('key',
u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM')
def proxy_ssl(self, host=None, port=None):
if host and port:
host = '%s:%d' % (host, port)
else:
host = '%s:%d' % (self.host, self.port)
# Seems properly to use timeout for connect too
timeout = self.http_connection_kwargs.get("timeout")
if timeout is not None:
sock = socket.create_connection((self.proxy,
int(self.proxy_port)), timeout)
else:
sock = socket.create_connection((self.proxy, int(self.proxy_port)))
boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host)
sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
sock.sendall("User-Agent: %s\r\n" % UserAgent)
if self.proxy_user and self.proxy_pass:
for k, v in self.get_proxy_auth_header().items():
sock.sendall("%s: %s\r\n" % (k, v))
# See discussion about this config option at
# https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc
if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False):
sock.sendall("\r\n")
else:
sock.sendall("\r\n")
resp = http_client.HTTPResponse(sock, strict=True, debuglevel=self.debug)
resp.begin()
if resp.status != 200:
# Fake a socket error, use a code that make it obvious it hasn't
# been generated by the socket library
raise socket.error(-71,
"Error talking to HTTP proxy %s:%s: %s (%s)" %
(self.proxy, self.proxy_port,
resp.status, resp.reason))
# We can safely close the response, it duped the original socket
resp.close()
h = http_client.HTTPConnection(host)
if self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
msg = "wrapping ssl socket for proxied connection; "
if self.ca_certificates_file:
msg += "CA certificate file=%s" %self.ca_certificates_file
else:
msg += "using system provided SSL certs"
boto.log.debug(msg)
key_file = self.http_connection_kwargs.get('key_file', None)
cert_file = self.http_connection_kwargs.get('cert_file', None)
sslSock = ssl.wrap_socket(sock, keyfile=key_file,
certfile=cert_file,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.ca_certificates_file)
cert = sslSock.getpeercert()
hostname = self.host.split(':', 0)[0]
if not https_connection.ValidateCertificateHostname(cert, hostname):
raise https_connection.InvalidCertificateException(
hostname, cert, 'hostname mismatch')
else:
# Fallback for old Python without ssl.wrap_socket
if hasattr(http_client, 'ssl'):
sslSock = http_client.ssl.SSLSocket(sock)
else:
sslSock = socket.ssl(sock, None, None)
sslSock = http_client.FakeSocket(sock, sslSock)
# This is a bit unclean
h.sock = sslSock
return h
def __init__(
self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path="/",
provider="aws",
security_token=None,
suppress_consec_slashes=True,
validate_certs=True,
):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch. The factory should have
a similar interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
:type suppress_consec_slashes: bool
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type validate_certs: bool
:param validate_certs: Controls whether SSL certificates
will be validated or not. Defaults to True.
"""
self.suppress_consec_slashes = suppress_consec_slashes
self.num_retries = 6
# Override passed-in is_secure setting if value was defined in config.
if config.has_option("Boto", "is_secure"):
is_secure = config.getboolean("Boto", "is_secure")
self.is_secure = is_secure
# Whether or not to validate server certificates.
# The default is now to validate certificates. This can be
# overridden in the boto config file are by passing an
# explicit validate_certs parameter to the class constructor.
self.https_validate_certificates = config.getbool("Boto", "https_validate_certificates", validate_certs)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later."
)
self.ca_certificates_file = config.get_value("Boto", "ca_certificates_file", DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror, httplib.BadStatusLine)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if is_secure:
self.protocol = "https"
else:
self.protocol = "http"
self.host = host
self.path = path
# if the value passed in for debug
if not isinstance(debug, (int, long)):
debug = 0
self.debug = config.getint("Boto", "debug", debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option("Boto", "http_socket_timeout"):
timeout = config.getint("Boto", "http_socket_timeout")
self.http_connection_kwargs["timeout"] = timeout
if isinstance(provider, Provider):
# Allow overriding Provider
self.provider = provider
else:
self._provider_type = provider
self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
self._pool = ConnectionPool()
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(host, config, self.provider, self._required_auth_capability())
if getattr(self, "AuthServiceName", None) is not None:
self.auth_service_name = self.AuthServiceName
