def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ ##Get Proxy configuration from boto file, defaults are None, 0 and False boto_proxy_config = { 'proxy_host': config.get('Boto', 'proxy', None), 'proxy_type': config.get('Boto', 'proxy_type', 'http'), 'proxy_port': config.getint('Boto', 'proxy_port'), 'proxy_user': config.get('Boto', 'proxy_user', None), 'proxy_pass': config.get('Boto', 'proxy_pass', None), 'proxy_rdns': config.get('Boto', 'proxy_rdns', None) } #Use SetProxyInfo to convert boto config to httplib2.proxyinfo object proxy_info = SetProxyInfo(boto_proxy_config) # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT_SEC http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) return http
def proxy_ssl(self): host = '%s:%d' % (self.host, self.port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect((self.proxy, int(self.proxy_port))) except: raise boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host) sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) sock.sendall("User-Agent: %s\r\n" % UserAgent) if self.proxy_user and self.proxy_pass: for k, v in self.get_proxy_auth_header().items(): sock.sendall("%s: %s\r\n" % (k, v)) # See discussion about this config option at # https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False): sock.sendall("\r\n") else: sock.sendall("\r\n") resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug) resp.begin() if resp.status != 200: # Fake a socket error, use a code that make it obvious it hasn't # been generated by the socket library raise socket.error(-71, "Error talking to HTTP proxy %s:%s: %s (%s)" % (self.proxy, self.proxy_port, resp.status, resp.reason)) # We can safely close the response, it duped the original socket resp.close() h = httplib.HTTPConnection(host) if self.https_validate_certificates and HAVE_HTTPS_CONNECTION: boto.log.debug("wrapping ssl socket for proxied connection; " "CA certificate file=%s", self.ca_certificates_file) key_file = self.http_connection_kwargs.get('key_file', None) cert_file = self.http_connection_kwargs.get('cert_file', None) sslSock = ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certificates_file) cert = sslSock.getpeercert() hostname = self.host.split(':', 0)[0] if not https_connection.ValidateCertificateHostname(cert, hostname): raise https_connection.InvalidCertificateException( hostname, cert, 'hostname mismatch') else: # Fallback for old Python without ssl.wrap_socket if hasattr(httplib, 'ssl'): sslSock = httplib.ssl.SSLSocket(sock) else: sslSock = socket.ssl(sock, None, None) sslSock = httplib.FakeSocket(sock, sslSock) # This is a bit unclean h.sock = sslSock return h
def __init__(self, logger): """Initializes config. Args: logger (logging.logger): gsutil logger. """ self.logger = logger self.use_client_certificate = config.getbool('Credentials', 'use_client_certificate') self.client_cert_path = None self.client_cert_password = None if not self.use_client_certificate: # Don't spend time generating values gsutil won't need. return # Generates certificate and deletes it afterwards. atexit.register(self._UnprovisionClientCert) command_string = config.get('Credentials', 'cert_provider_command', None) if not command_string: raise CertProvisionError('No cert provider detected.') self.client_cert_path = os.path.join(gslib.GSUTIL_DIR, 'caa_cert.pem') try: # Certs provisioned using endpoint verification are stored as a # single file holding both the public certificate and the private key. self._ProvisionClientCert(command_string, self.client_cert_path) except CertProvisionError as e: self.logger.error('Failed to provision client certificate: %s' % e)
def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ proxy_info = httplib2.ProxyInfo( proxy_type=3, proxy_host=boto.config.get('Boto', 'proxy', None), proxy_port=boto.config.getint('Boto', 'proxy_port', 0), proxy_user=boto.config.get('Boto', 'proxy_user', None), proxy_pass=boto.config.get('Boto', 'proxy_pass', None), proxy_rdns=boto.config.get('Boto', 'proxy_rdns', False)) # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) return http
def SetProxyInfo(): """Sets proxy info from boto and environment and converts to httplib2.ProxyInfo. Args: None. Returns: httplib2.ProxyInfo constructed from boto or environment variable string. """ #Defining proxy_type based on httplib2 library, accounting for None entry too. proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3} boto_proxy_val = config.get('Boto', 'proxy_type', None) #proxy_type defaults to 'http (3)' for backwards compatibility proxy_type = proxy_type_spec.get(boto_proxy_val) or proxy_type_spec['http'] proxy_host = config.get('Boto', 'proxy', None) #For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use #to http only proxy_info = httplib2.ProxyInfo( proxy_host=proxy_host, proxy_type=proxy_type, proxy_port=config.getint('Boto', 'proxy_port', 0), proxy_user=config.get('Boto', 'proxy_user', None), proxy_pass=config.get('Boto', 'proxy_pass', None), proxy_rdns=config.getbool( 'Boto', 'proxy_rdns', True if proxy_type == proxy_type_spec['http'] else False)) #Added to force socks proxies not to use rdns if not (proxy_info.proxy_type == proxy_type_spec['http']): proxy_info.proxy_rdns = False if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. Use only http proxies. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. proxy_info.proxy_rdns = config.getbool('Boto', 'proxy_rdns', True) break return proxy_info
def SetProxyInfo(boto_proxy_config): """Sets proxy info from boto and environment and converts to httplib2.ProxyInfo. Args: dict: Values read from the .boto file Returns: httplib2.ProxyInfo constructed from boto or environment variable string. """ #Defining proxy_type based on httplib2 library, accounting for None entry too. proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3} _proxy_type_val = boto_proxy_config.get('proxy_type').lower() #proxy_type defaults to 'http (3)' for backwards compatibility proxy_type = proxy_type_spec.get( _proxy_type_val) or proxy_type_spec['http'] proxy_host = boto_proxy_config.get('proxy_host') proxy_port = boto_proxy_config.get('proxy_port') proxy_user = boto_proxy_config.get('proxy_user') proxy_pass = boto_proxy_config.get('proxy_pass') proxy_rdns = boto_proxy_config.get('proxy_rdns') #For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use #to http only proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host, proxy_type=proxy_type, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, proxy_rdns=proxy_rdns) #Added to force socks proxies not to use rdns, and set default rdns for https to true if boto_proxy_config.get('proxy_rdns') == None: if (proxy_info.proxy_type == proxy_type_spec['http']): proxy_info.proxy_rdns = True #Use rdns unless explicity set as False in boto else: proxy_info.proxy_rdns = False #Added to force socks proxies not to use rdns if not (proxy_info.proxy_type == proxy_type_spec['http']): proxy_info.proxy_rdns = False if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. Use only http proxies. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. proxy_info.proxy_rdns = config.getbool('Boto', 'proxy_rdns', True) break return proxy_info
def _iter_storages(): directory = os.environ.get('SCCACHE_DIR') if directory: yield LocalStorage(directory) bucket_name = os.environ.get('SCCACHE_BUCKET') if bucket_name: storage = BotoStorage(bucket_name, dns_server=os.environ.get('SCCACHE_NAMESERVER')) yield storage if not isinstance(storage, S3Storage): from boto import config if config.getbool('s3', 'fallback', False): yield S3Storage(bucket_name, dns_server=os.environ.get('SCCACHE_NAMESERVER'))
def _iter_storages(): directory = os.environ.get('SCCACHE_DIR') if directory: yield LocalStorage(directory) bucket_name = os.environ.get('SCCACHE_BUCKET') print 'HEY', bucket_name if bucket_name: storage = BotoStorage( bucket_name, dns_server=os.environ.get('SCCACHE_NAMESERVER')) yield storage if not isinstance(storage, S3Storage): from boto import config if config.getbool('s3', 'fallback', False): yield S3Storage( bucket_name, dns_server=os.environ.get('SCCACHE_NAMESERVER'))
class TestMtls(testcase.GsUtilIntegrationTestCase): """Integration tests for mTLS authentication.""" @unittest.skipIf( not config.getbool('Credentials', 'use_client_certificate'), 'mTLS requires "use_client_certificate" to be "True" in .boto config.') @integration_testcase.SkipForXML(MTLS_AVAILABILITY_MESSAGE) @integration_testcase.SkipForS3(MTLS_AVAILABILITY_MESSAGE) def test_can_list_bucket_with_mtls_authentication(self): # Cannot use self.CreateBucket because testing framework's authentication # doesn't work with mTLS. bucket_uri = 'gs://{}'.format(self.MakeTempName('bucket')) self.RunGsUtil(['mb', bucket_uri]) stdout = self.RunGsUtil(['-D', 'ls'], return_stdout=True) self.RunGsUtil(['rb', bucket_uri]) # # Check if mTLS API endpoint was hit in debug output. self.assertIn('storage.mtls.googleapis.com', stdout) # # If bucket was successfully listed, it implies successful authentication. self.assertIn(bucket_uri, stdout)
def __init__(self, bucket_name, host=DefaultHost, calling_format=DefaultCallingFormat, dns_server=None): assert bucket_name self._bucket_name = bucket_name from boto.s3.connection import S3Connection from boto.utils import find_class from boto import config self._calling_format = find_class(calling_format)() self._host = self._calling_format.build_host(host, self._bucket_name) self._failed = False # Prepare the wrapper classes to use for urllib and boto. dns_query = dns_query_function(dns_server) self._http_connection_class = ConnectionWrapperFactory( httplib.HTTPConnection, dns_query) self._https_connection_class = ConnectionWrapperFactory( httplib.HTTPSConnection, dns_query) self._url_opener = OpenerFactory(self._http_connection_class, self._https_connection_class) # Get the boto S3 bucket instance if config.getbool('Boto', 'is_secure', True): s3_connection = S3Connection( host=host, port=443, https_connection_factory=(self._https_connection_class, ())) else: s3_connection = S3Connection( host=host, port=80, https_connection_factory=(self._http_connection_class, ())) self._bucket = s3_connection.get_bucket(self._bucket_name, validate=False) self.last_stats = {}
def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ proxy_host = config.get('Boto', 'proxy', None) proxy_info = httplib2.ProxyInfo( proxy_type=3, proxy_host=proxy_host, proxy_port=config.getint('Boto', 'proxy_port', 0), proxy_user=config.get('Boto', 'proxy_user', None), proxy_pass=config.get('Boto', 'proxy_pass', None), proxy_rdns=config.get('Boto', 'proxy_rdns', True if proxy_host else False)) if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. proxy_info.proxy_rdns = config.get('Boto', 'proxy_rdns', True) break # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT_SEC http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) return http
def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ # Get proxy info, will get None if no proxies are set in boto or # environment variables proxy_info = SetProxyInfo() # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT_SEC http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) return http
def GetNewHttp(http_class=httplib2.Http, **kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. **kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ proxy_info = httplib2.ProxyInfo( proxy_type=3, proxy_host=boto.config.get('Boto', 'proxy', None), proxy_port=boto.config.getint('Boto', 'proxy_port', 0), proxy_user=boto.config.get('Boto', 'proxy_user', None), proxy_pass=boto.config.get('Boto', 'proxy_pass', None), proxy_rdns=boto.config.get('Boto', 'proxy_rdns', False)) if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. proxy_info.proxy_rdns = boto.config.get('Boto', 'proxy_rdns', True) break # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT http = http_class(proxy_info=proxy_info, **kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) return http
def __init__(self, bucket_name, host=DefaultHost, calling_format=DefaultCallingFormat, dns_server=None): assert bucket_name self._bucket_name = bucket_name from boto.s3.connection import S3Connection from boto.utils import find_class from boto import config self._calling_format = find_class(calling_format)() self._host = self._calling_format.build_host(host, self._bucket_name) self._failed = False # Prepare the wrapper classes to use for urllib and boto. dns_query = dns_query_function(dns_server) self._http_connection_class = ConnectionWrapperFactory( httplib.HTTPConnection, dns_query) self._https_connection_class = ConnectionWrapperFactory( httplib.HTTPSConnection, dns_query) self._url_opener = OpenerFactory( self._http_connection_class, self._https_connection_class) # Get the boto S3 bucket instance if config.getbool('Boto', 'is_secure', True): s3_connection = S3Connection(host=host, port=443, https_connection_factory=(self._https_connection_class, ())) else: s3_connection = S3Connection(host=host, port=80, https_connection_factory=(self._http_connection_class, ())) self._bucket = s3_connection.get_bucket(self._bucket_name, validate=False) self.last_stats = {}
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. At some point in the # future, the default should be flipped to true. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', False) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") self.ca_certificates_file = config.get_value('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append(ssl.SSLError) self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR, ) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] # Timeout used to tell httplib how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): if config.has_option('Boto', 'http_socket_timeout'): timeout = config.getint('Boto', 'http_socket_timeout') self.http_connection_kwargs['timeout'] = timeout self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host # cache up to 20 connections per host, up to 20 hosts self._pool = ConnectionPool(20, 20) self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability())
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws', security_token=None, suppress_consec_slashes=True, validate_certs=True, profile_name=None): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :keyword str security_token: The security token associated with temporary credentials issued by STS. Optional unless using temporary credentials. If none is specified, the environment variable ``AWS_SECURITY_TOKEN`` is used if defined. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{http_client.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect :type suppress_consec_slashes: bool :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type validate_certs: bool :param validate_certs: Controls whether SSL certificates will be validated or not. Defaults to True. :type profile_name: str :param profile_name: Override usual Credentials section in config file to use a named set of keys instead. """ self.suppress_consec_slashes = suppress_consec_slashes self.num_retries = 6 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. # The default is now to validate certificates. This can be # overridden in the boto config file are by passing an # explicit validate_certs parameter to the class constructor. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', validate_certs) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") certs_file = config.get_value( 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) if certs_file == 'system': certs_file = None self.ca_certificates_file = certs_file if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from http_client that we want to catch and retry self.http_exceptions = (http_client.HTTPException, socket.error, socket.gaierror, http_client.BadStatusLine) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR,) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path # if the value passed in for debug if not isinstance(debug, six.integer_types): debug = 0 self.debug = config.getint('Boto', 'debug', debug) self.host_header = None # Timeout used to tell http_client how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): # If timeout isn't defined in boto config file, use 70 second # default as recommended by # http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html self.http_connection_kwargs['timeout'] = config.getint( 'Boto', 'http_socket_timeout', 70) if isinstance(provider, Provider): # Allow overriding Provider self.provider = provider else: self._provider_type = provider self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token, profile_name) # Allow config file to override default host, port, and host header. if self.provider.host: self.host = self.provider.host if self.provider.port: self.port = self.provider.port if self.provider.host_header: self.host_header = self.provider.host_header self._pool = ConnectionPool() self._connection = (self.host, self.port, self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability()) if getattr(self, 'AuthServiceName', None) is not None: self.auth_service_name = self.AuthServiceName self.request_hook = None
def translate_to_gcloud_storage_if_requested(self): """Translates the gsutil command to gcloud storage equivalent. The translated commands get stored at self._translated_gcloud_storage_command. This command also translate the boto config, which gets stored as a dict at self._translated_env_variables Returns: True if the command was successfully translated, else False. """ if self.command_name == 'version' or self.command_name == 'test': # Running any command in debug mode will lead to calling gsutil version # command. We don't want to translate the version command as this # should always reflect the version that gsutil is using. # We don't want to run any translation for the "test" command. return False use_gcloud_storage = config.getbool('GSUtil', 'use_gcloud_storage', False) try: hidden_shim_mode = HIDDEN_SHIM_MODE( config.get('GSUtil', 'hidden_shim_mode', 'none')) except ValueError: raise exception.CommandException( 'Invalid option specified for' ' GSUtil:hidden_shim_mode config setting. Should be one of: {}' .format(' | '.join([x.value for x in HIDDEN_SHIM_MODE]))) if use_gcloud_storage: try: top_level_flags, env_variables = self._translate_top_level_flags( ) header_flags = self._translate_headers() flags_from_boto, env_vars_from_boto = self._translate_boto_config( ) env_variables.update(env_vars_from_boto) gcloud_binary_path = _get_gcloud_binary_path() gcloud_storage_command = ([gcloud_binary_path] + self.get_gcloud_storage_args() + top_level_flags + header_flags + flags_from_boto) if hidden_shim_mode == HIDDEN_SHIM_MODE.DRY_RUN: self._print_gcloud_storage_command_info( gcloud_storage_command, env_variables, dry_run=True) elif not os.environ.get( 'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL'): raise exception.GcloudStorageTranslationError( 'Requested to use "gcloud storage" but gsutil is not using the' ' same credentials as gcloud.' ' You can make gsutil use the same credentials by running:\n' '{} config set pass_credentials_to_gsutil True'.format( gcloud_binary_path)) else: self._print_gcloud_storage_command_info( gcloud_storage_command, env_variables) self._translated_gcloud_storage_command = gcloud_storage_command self._translated_env_variables = env_variables return True except exception.GcloudStorageTranslationError as e: # Raise error if no_fallback mode has been requested. This mode # should only be used for debuggling and testing purposes. if hidden_shim_mode == HIDDEN_SHIM_MODE.NO_FALLBACK: raise exception.CommandException(e) # For all other cases, we want to run gsutil. self.logger.error( 'Cannot translate gsutil command to gcloud storage.' ' Going to run gsutil command. Error: %s', e) return False
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. At some point in the # future, the default should be flipped to true. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', False) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") self.ca_certificates_file = config.get_value( 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append(ssl.SSLError) self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR,) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] # Timeout used to tell httplib how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): if config.has_option('Boto', 'http_socket_timeout'): timeout = config.getint('Boto', 'http_socket_timeout') self.http_connection_kwargs['timeout'] = timeout self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host self._pool = ConnectionPool() self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability())
def __init__(self, bucket_storage_uri_class, logger, provider=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud Storage. Args: bucket_storage_uri_class: Unused. logger: logging.logger for outputting log messages. provider: Unused. This implementation supports only Google Cloud Storage. credentials: Credentials to be used for interacting with Google Cloud Storage. debug: Debug level for the API implementation (0..3). """ # TODO: Plumb host_header for perfdiag / test_perfdiag. # TODO: Add jitter to apitools' http_wrapper retry mechanism. super(GcsJsonApi, self).__init__(bucket_storage_uri_class, logger, provider='gs', debug=debug) no_op_credentials = False if not credentials: loaded_credentials = self._CheckAndGetCredentials(logger) if not loaded_credentials: loaded_credentials = NoOpCredentials() no_op_credentials = True else: if isinstance(credentials, NoOpCredentials): no_op_credentials = True self.credentials = credentials or loaded_credentials self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) self.http_base = 'https://' gs_json_host = config.get('Credentials', 'gs_json_host', None) self.host_base = gs_json_host or 'www.googleapis.com' if not gs_json_host: gs_host = config.get('Credentials', 'gs_host', None) if gs_host: raise ArgumentException( 'JSON API is selected but gs_json_host is not configured, ' 'while gs_host is configured to %s. Please also configure ' 'gs_json_host and gs_json_port to match your desired endpoint.' % gs_host) gs_json_port = config.get('Credentials', 'gs_json_port', None) if not gs_json_port: gs_port = config.get('Credentials', 'gs_port', None) if gs_port: raise ArgumentException( 'JSON API is selected but gs_json_port is not configured, ' 'while gs_port is configured to %s. Please also configure ' 'gs_json_host and gs_json_port to match your desired endpoint.' % gs_port) self.host_port = '' else: self.host_port = ':' + config.get('Credentials', 'gs_json_port') self.api_version = config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_VERSION) self.url_base = (self.http_base + self.host_base + self.host_port + '/' + 'storage/' + self.api_version + '/') self.credentials.set_store( multistore_file.get_credential_storage_custom_string_key( GetCredentialStoreFilename(), self.api_version)) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.StorageV1( url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials, version=self.api_version) if no_op_credentials: # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam('key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM')
def proxy_ssl(self, host=None, port=None): if host and port: host = '%s:%d' % (host, port) else: host = '%s:%d' % (self.host, self.port) # Seems properly to use timeout for connect too timeout = self.http_connection_kwargs.get("timeout") if timeout is not None: sock = socket.create_connection((self.proxy, int(self.proxy_port)), timeout) else: sock = socket.create_connection((self.proxy, int(self.proxy_port))) boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host) sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) sock.sendall("User-Agent: %s\r\n" % UserAgent) if self.proxy_user and self.proxy_pass: for k, v in self.get_proxy_auth_header().items(): sock.sendall("%s: %s\r\n" % (k, v)) # See discussion about this config option at # https://groups.google.com/forum/?fromgroups#!topic/boto-dev/teenFvOq2Cc if config.getbool('Boto', 'send_crlf_after_proxy_auth_headers', False): sock.sendall("\r\n") else: sock.sendall("\r\n") resp = http_client.HTTPResponse(sock, strict=True, debuglevel=self.debug) resp.begin() if resp.status != 200: # Fake a socket error, use a code that make it obvious it hasn't # been generated by the socket library raise socket.error(-71, "Error talking to HTTP proxy %s:%s: %s (%s)" % (self.proxy, self.proxy_port, resp.status, resp.reason)) # We can safely close the response, it duped the original socket resp.close() h = http_client.HTTPConnection(host) if self.https_validate_certificates and HAVE_HTTPS_CONNECTION: msg = "wrapping ssl socket for proxied connection; " if self.ca_certificates_file: msg += "CA certificate file=%s" %self.ca_certificates_file else: msg += "using system provided SSL certs" boto.log.debug(msg) key_file = self.http_connection_kwargs.get('key_file', None) cert_file = self.http_connection_kwargs.get('cert_file', None) sslSock = ssl.wrap_socket(sock, keyfile=key_file, certfile=cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certificates_file) cert = sslSock.getpeercert() hostname = self.host.split(':', 0)[0] if not https_connection.ValidateCertificateHostname(cert, hostname): raise https_connection.InvalidCertificateException( hostname, cert, 'hostname mismatch') else: # Fallback for old Python without ssl.wrap_socket if hasattr(http_client, 'ssl'): sslSock = http_client.ssl.SSLSocket(sock) else: sslSock = socket.ssl(sock, None, None) sslSock = http_client.FakeSocket(sock, sslSock) # This is a bit unclean h.sock = sslSock return h
def __init__( self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path="/", provider="aws", security_token=None, suppress_consec_slashes=True, validate_certs=True, ): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect :type suppress_consec_slashes: bool :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type validate_certs: bool :param validate_certs: Controls whether SSL certificates will be validated or not. Defaults to True. """ self.suppress_consec_slashes = suppress_consec_slashes self.num_retries = 6 # Override passed-in is_secure setting if value was defined in config. if config.has_option("Boto", "is_secure"): is_secure = config.getboolean("Boto", "is_secure") self.is_secure = is_secure # Whether or not to validate server certificates. # The default is now to validate certificates. This can be # overridden in the boto config file are by passing an # explicit validate_certs parameter to the class constructor. self.https_validate_certificates = config.getbool("Boto", "https_validate_certificates", validate_certs) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later." ) self.ca_certificates_file = config.get_value("Boto", "ca_certificates_file", DEFAULT_CA_CERTS_FILE) self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror, httplib.BadStatusLine) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append(https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR,) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if is_secure: self.protocol = "https" else: self.protocol = "http" self.host = host self.path = path # if the value passed in for debug if not isinstance(debug, (int, long)): debug = 0 self.debug = config.getint("Boto", "debug", debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] # Timeout used to tell httplib how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): if config.has_option("Boto", "http_socket_timeout"): timeout = config.getint("Boto", "http_socket_timeout") self.http_connection_kwargs["timeout"] = timeout if isinstance(provider, Provider): # Allow overriding Provider self.provider = provider else: self._provider_type = provider self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token) # allow config file to override default host if self.provider.host: self.host = self.provider.host self._pool = ConnectionPool() self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler(host, config, self.provider, self._required_auth_capability()) if getattr(self, "AuthServiceName", None) is not None: self.auth_service_name = self.AuthServiceName