def _BotoIsSecure():
for cfg_var in ('is_secure', 'https_validate_certificates'):
if (config.has_option('Boto', cfg_var)
and not config.getboolean('Boto', cfg_var)):
return False, cfg_var
return True, ''
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value(
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider,
aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
self._pool = ConnectionPool()
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __init__(self, host, aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/',
provider='aws', security_token=None,
suppress_consec_slashes=True,
validate_certs=True, profile_name=None):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:keyword str security_token: The security token associated with
temporary credentials issued by STS. Optional unless using
temporary credentials. If none is specified, the environment
variable ``AWS_SECURITY_TOKEN`` is used if defined.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch. The factory should have
a similar interface to L{http_client.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
:type suppress_consec_slashes: bool
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type validate_certs: bool
:param validate_certs: Controls whether SSL certificates
will be validated or not. Defaults to True.
:type profile_name: str
:param profile_name: Override usual Credentials section in config
file to use a named set of keys instead.
"""
self.suppress_consec_slashes = suppress_consec_slashes
self.num_retries = 6
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates.
# The default is now to validate certificates. This can be
# overridden in the boto config file are by passing an
# explicit validate_certs parameter to the class constructor.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates',
validate_certs)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
certs_file = config.get_value(
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
if certs_file == 'system':
certs_file = None
self.ca_certificates_file = certs_file
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from http_client that we want to catch and retry
self.http_exceptions = (http_client.HTTPException, socket.error,
socket.gaierror, http_client.BadStatusLine)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
# if the value passed in for debug
if not isinstance(debug, six.integer_types):
debug = 0
self.debug = config.getint('Boto', 'debug', debug)
self.host_header = None
# Timeout used to tell http_client how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
# If timeout isn't defined in boto config file, use 70 second
# default as recommended by
# http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html
self.http_connection_kwargs['timeout'] = config.getint(
'Boto', 'http_socket_timeout', 70)
if isinstance(provider, Provider):
# Allow overriding Provider
self.provider = provider
else:
self._provider_type = provider
self.provider = Provider(self._provider_type,
aws_access_key_id,
aws_secret_access_key,
security_token,
profile_name)
# Allow config file to override default host, port, and host header.
if self.provider.host:
self.host = self.provider.host
if self.provider.port:
self.port = self.provider.port
if self.provider.host_header:
self.host_header = self.provider.host_header
self._pool = ConnectionPool()
self._connection = (self.host, self.port, self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
if getattr(self, 'AuthServiceName', None) is not None:
self.auth_service_name = self.AuthServiceName
self.request_hook = None
def _BotoIsSecure():
for cfg_var in ('is_secure', 'https_validate_certificates'):
if (config.has_option('Boto', cfg_var)
and not config.getboolean('Boto', cfg_var)):
return False, cfg_var
return True, ''
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/',
provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value('Boto',
'ca_certificates_file',
DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider, aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider,
aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/',
provider='aws'):
"""
:type host: string
:param host: The host to make the connection to
:type aws_access_key_id: string
:param aws_access_key_id: AWS Access Key ID (provided by Amazon)
:type aws_secret_access_key: string
:param aws_secret_access_key: Secret Access Key (provided by Amazon)
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:type proxy:
:param proxy:
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: string
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: string
:param proxy_pass: The password to use when connection over a proxy.
:type port: integer
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.provider = Provider(provider, aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
if self.secret_key is None:
raise BotoClientError('No credentials have been supplied')
# initialize an HMAC for signatures, make copies with each request
self.hmac = hmac.new(self.secret_key, digestmod=sha)
if sha256:
self.hmac_256 = hmac.new(self.secret_key, digestmod=sha256)
else:
self.hmac_256 = None
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
def __init__(
self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path="/",
provider="aws",
):
"""
:type host: string
:param host: The host to make the connection to
:type aws_access_key_id: string
:param aws_access_key_id: AWS Access Key ID (provided by Amazon)
:type aws_secret_access_key: string
:param aws_secret_access_key: Secret Access Key (provided by Amazon)
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:type proxy:
:param proxy:
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: string
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: string
:param proxy_pass: The password to use when connection over a proxy.
:type port: integer
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option("Boto", "is_secure"):
is_secure = config.getboolean("Boto", "is_secure")
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if is_secure:
self.protocol = "https"
else:
self.protocol = "http"
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint("Boto", "debug", debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
if self.secret_key is None:
raise BotoClientError("No credentials have been supplied")
# initialize an HMAC for signatures, make copies with each request
self.hmac = hmac.new(self.secret_key, digestmod=sha)
if sha256:
self.hmac_256 = hmac.new(self.secret_key, digestmod=sha256)
else:
self.hmac_256 = None
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
def RunCommand(self):
for cfg_var in ('is_secure', 'https_validate_certificates'):
if (config.has_option('Boto', cfg_var)
and not config.getboolean('Boto', cfg_var)):
raise CommandException(
'Your boto configuration has %s = False. The update command\n'
'cannot be run this way, for security reasons.' % cfg_var)
force_update = False
no_prompt = False
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == '-f':
force_update = True
if o == '-n':
no_prompt = True
dirs_to_remove = []
tmp_dir = tempfile.mkdtemp()
dirs_to_remove.append(tmp_dir)
os.chdir(tmp_dir)
if not no_prompt:
print 'Checking for software update...'
if self.args:
update_from_uri_str = self.args[0]
if not update_from_uri_str.endswith('.tar.gz'):
raise CommandException(
'The update command only works with tar.gz files.')
for i, result in enumerate(self.WildcardIterator(update_from_uri_str)):
if i > 0:
raise CommandException(
'Invalid update URI. Must name a single .tar.gz file.')
if result.uri.names_file():
if not force_update:
raise CommandException(
('"update" command does not support "file://" URIs without the '
'-f option.'))
elif not result.uri.names_object():
raise CommandException(
'Invalid update object URI. Must name a single .tar.gz file.')
else:
update_from_uri_str = GSUTIL_PUB_TARBALL
# Try to retrieve version info from tarball metadata; failing that; download
# the tarball and extract the VERSION file. The version lookup will fail
# when running the update system test, because it retrieves the tarball from
# a temp file rather than a cloud URI (files lack the version metadata).
tarball_version = LookUpGsutilVersion(
self.suri_builder.StorageUri(update_from_uri_str))
if tarball_version:
tf = None
else:
tf = self._FetchAndOpenGsutilTarball(update_from_uri_str)
tf.extract('./gsutil/VERSION')
with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file:
tarball_version = ver_file.read().strip()
if not force_update and self.gsutil_ver == tarball_version:
self._CleanUpUpdateCommand(tf, dirs_to_remove)
if self.args:
raise CommandException('You already have %s installed.' %
update_from_uri_str, informational=True)
else:
raise CommandException('You already have the latest gsutil release '
'installed.', informational=True)
if not no_prompt:
print(('This command will update to the "%s" version of\ngsutil at %s') %
(tarball_version, self.gsutil_bin_dir))
self._ExplainIfSudoNeeded(tf, dirs_to_remove)
if no_prompt:
answer = 'y'
else:
answer = raw_input('Proceed? [y/N] ')
if not answer or answer.lower()[0] != 'y':
self._CleanUpUpdateCommand(tf, dirs_to_remove)
raise CommandException('Not running update.', informational=True)
if not tf:
tf = self._FetchAndOpenGsutilTarball(update_from_uri_str)
# Ignore keyboard interrupts during the update to reduce the chance someone
# hitting ^C leaves gsutil in a broken state.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# self.gsutil_bin_dir lists the path where the code should end up (like
# /usr/local/gsutil), which is one level down from the relative path in the
# tarball (since the latter creates files in ./gsutil). So, we need to
# extract at the parent directory level.
gsutil_bin_parent_dir = os.path.dirname(self.gsutil_bin_dir)
# Extract tarball to a temporary directory in a sibling to gsutil_bin_dir.
old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
dirs_to_remove.append(old_dir)
dirs_to_remove.append(new_dir)
self._EnsureDirsSafeForUpdate(dirs_to_remove)
try:
tf.extractall(path=new_dir)
except Exception, e:
self._CleanUpUpdateCommand(tf, dirs_to_remove)
raise CommandException('Update failed: %s.' % e)
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.provider = Provider(provider,
aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def RunCommand(self):
for cfg_var in ("is_secure", "https_validate_certificates"):
if config.has_option("Boto", cfg_var) and not config.getboolean("Boto", cfg_var):
raise CommandException(
"Your boto configuration has %s = False. "
"The update command\ncannot be run this way, for "
"security reasons." % cfg_var
)
dirs_to_remove = []
# Retrieve gsutil tarball and check if it's newer than installed code.
# TODO: Store this version info as metadata on the tarball object and
# change this command's implementation to check that metadata instead of
# downloading the tarball to check the version info.
tmp_dir = tempfile.mkdtemp()
dirs_to_remove.append(tmp_dir)
os.chdir(tmp_dir)
print "Checking for software update..."
if len(self.args):
update_from_uri_str = self.args[0]
if not update_from_uri_str.endswith(".tar.gz"):
raise CommandException("The update command only works with tar.gz files.")
else:
update_from_uri_str = "gs://pub/gsutil.tar.gz"
self.command_runner.RunNamedCommand(
"cp", [update_from_uri_str, "file://gsutil.tar.gz"], self.headers, self.debug
)
# Note: tf is closed in _CleanUpUpdateCommand.
tf = tarfile.open("gsutil.tar.gz")
tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions.
tf.extract("./gsutil/VERSION")
ver_file = open("gsutil/VERSION", "r")
try:
latest_version_string = ver_file.read().rstrip("\n")
finally:
ver_file.close()
force_update = False
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == "-f":
force_update = True
if not force_update and self.gsutil_ver == latest_version_string:
self._CleanUpUpdateCommand(tf, dirs_to_remove)
if len(self.args):
raise CommandException("You already have %s installed." % update_from_uri_str, informational=True)
else:
raise CommandException("You already have the latest gsutil release " "installed.", informational=True)
print (
('This command will update to the "%s" version of\ngsutil at %s')
% (latest_version_string, self.gsutil_bin_dir)
)
self._ExplainIfSudoNeeded(tf, dirs_to_remove)
answer = raw_input("Proceed? [y/N] ")
if not answer or answer.lower()[0] != "y":
self._CleanUpUpdateCommand(tf, dirs_to_remove)
raise CommandException("Not running update.", informational=True)
# Ignore keyboard interrupts during the update to reduce the chance someone
# hitting ^C leaves gsutil in a broken state.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# self.gsutil_bin_dir lists the path where the code should end up (like
# /usr/local/gsutil), which is one level down from the relative path in the
# tarball (since the latter creates files in ./gsutil). So, we need to
# extract at the parent directory level.
gsutil_bin_parent_dir = os.path.dirname(self.gsutil_bin_dir)
# Extract tarball to a temporary directory in a sibling to gsutil_bin_dir.
old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
dirs_to_remove.append(old_dir)
dirs_to_remove.append(new_dir)
self._EnsureDirsSafeForUpdate(dirs_to_remove)
try:
tf.extractall(path=new_dir)
except Exception, e:
self._CleanUpUpdateCommand(tf, dirs_to_remove)
raise CommandException("Update failed: %s." % e)
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/',
provider=None):
"""
:type host: string
:param host: The host to make the connection to
:type aws_access_key_id: string
:param aws_access_key_id: AWS Access Key ID (provided by Amazon)
:type aws_secret_access_key: string
:param aws_secret_access_key: Secret Access Key (provided by Amazon)
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:type proxy:
:param proxy:
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: string
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: string
:param proxy_pass: The password to use when connection over a proxy.
:type port: integer
:param port: The port to use to connect
"""
self.provider_headers = ProviderHeaders(provider)
acl_classes = AclClasses(provider)
self.acl_class = acl_classes.acl_class
self.canned_acls = acl_classes.canned_acls
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# If credentials have been loaded with provider-dependent ids and
# secret keys, use them.
if provider:
if provider == "google":
if (config.has_option('Credentials', 'gs_access_key_id')
and config.has_option('Credentials',
'gs_secret_access_key')):
aws_access_key_id = config.get('Credentials',
'gs_access_key_id')
aws_secret_access_key = config.get('Credentials',
'gs_secret_access_key')
# allow config file to override default host
if (config.has_option('Credentials', 'gs_host')):
self.host = config.get('Credentials', 'gs_host')
elif provider == "aws":
if (config.has_option('Credentials', 'aws_access_key_id')
and config.has_option('Credentials',
'aws_secret_access_key')):
aws_access_key_id = config.get('Credentials',
'aws_access_key_id')
aws_secret_access_key = config.get(
'Credentials', 'aws_secret_access_key')
# allow config file to override default host
if (config.has_option('Credentials', 'aws_host')):
self.host = config.get('Credentials', 'aws_host')
if aws_access_key_id:
self.aws_access_key_id = aws_access_key_id
elif os.environ.has_key('AWS_ACCESS_KEY_ID'):
self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
elif config.has_option('Credentials', 'aws_access_key_id'):
self.aws_access_key_id = config.get('Credentials',
'aws_access_key_id')
if aws_secret_access_key:
self.aws_secret_access_key = aws_secret_access_key
elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'):
self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
elif config.has_option('Credentials', 'aws_secret_access_key'):
self.aws_secret_access_key = config.get('Credentials',
'aws_secret_access_key')
# initialize an HMAC for signatures, make copies with each request
self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha)
if sha256:
self.hmac_256 = hmac.new(self.aws_secret_access_key,
digestmod=sha256)
else:
self.hmac_256 = None
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider=None):
"""
:type host: string
:param host: The host to make the connection to
:type aws_access_key_id: string
:param aws_access_key_id: AWS Access Key ID (provided by Amazon)
:type aws_secret_access_key: string
:param aws_secret_access_key: Secret Access Key (provided by Amazon)
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:type proxy:
:param proxy:
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: string
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: string
:param proxy_pass: The password to use when connection over a proxy.
:type port: integer
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# If credentials have been loaded with provider-dependent ids and
# secret keys, use them.
if provider:
if provider == "google":
if (config.has_option('Credentials',
'gs_access_key_id') and
config.has_option('Credentials',
'gs_secret_access_key')):
aws_access_key_id = config.get(
'Credentials', 'gs_access_key_id')
aws_secret_access_key = config.get(
'Credentials',
'gs_secret_access_key')
# allow config file to override default host
if (config.has_option('Credentials',
'gs_host')):
self.host = config.get('Credentials',
'gs_host')
elif provider == "amazon":
if (config.has_option('Credentials',
'aws_access_key_id') and
config.has_option('Credentials',
'aws_secret_access_key')):
aws_access_key_id = config.get('Credentials',
'aws_access_key_id')
aws_secret_access_key = config.get('Credentials',
'aws_secret_access_key')
# allow config file to override default host
if (config.has_option('Credentials',
'aws_host')):
self.host = config.get('Credentials',
'aws_host')
if aws_access_key_id:
self.aws_access_key_id = aws_access_key_id
elif os.environ.has_key('AWS_ACCESS_KEY_ID'):
self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
elif config.has_option('Credentials', 'aws_access_key_id'):
self.aws_access_key_id = config.get('Credentials', 'aws_access_key_id')
if aws_secret_access_key:
self.aws_secret_access_key = aws_secret_access_key
elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'):
self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
elif config.has_option('Credentials', 'aws_secret_access_key'):
self.aws_secret_access_key = config.get('Credentials', 'aws_secret_access_key')
# initialize an HMAC for signatures, make copies with each request
self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha)
if sha256:
self.hmac_256 = hmac.new(self.aws_secret_access_key, digestmod=sha256)
else:
self.hmac_256 = None
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
def _BotoIsSecure():
for cfg_var in ("is_secure", "https_validate_certificates"):
if config.has_option("Boto", cfg_var) and not config.getboolean("Boto", cfg_var):
return False, cfg_var
return True, ""
def RunCommand(self):
for cfg_var in ('is_secure', 'https_validate_certificates'):
if (config.has_option('Boto', cfg_var)
and not config.getboolean('Boto', cfg_var)):
raise CommandException(
'Your boto configuration has %s = False. '
'The update command\ncannot be run this way, for '
'security reasons.' % cfg_var)
dirs_to_remove = []
# Retrieve gsutil tarball and check if it's newer than installed code.
# TODO: Store this version info as metadata on the tarball object and
# change this command's implementation to check that metadata instead of
# downloading the tarball to check the version info.
tmp_dir = tempfile.mkdtemp()
dirs_to_remove.append(tmp_dir)
os.chdir(tmp_dir)
print 'Checking for software update...'
if len(self.args):
update_from_uri_str = self.args[0]
if not update_from_uri_str.endswith('.tar.gz'):
raise CommandException(
'The update command only works with tar.gz files.')
else:
update_from_uri_str = 'gs://pub/gsutil.tar.gz'
self.command_runner.RunNamedCommand(
'cp', [update_from_uri_str, 'file://gsutil.tar.gz'], self.headers,
self.debug)
# Note: tf is closed in _CleanUpUpdateCommand.
tf = tarfile.open('gsutil.tar.gz')
tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions.
tf.extract('./gsutil/VERSION')
ver_file = open('gsutil/VERSION', 'r')
try:
latest_version_string = ver_file.read().rstrip('\n')
finally:
ver_file.close()
force_update = False
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == '-f':
force_update = True
if not force_update and self.gsutil_ver == latest_version_string:
self._CleanUpUpdateCommand(tf, dirs_to_remove)
if len(self.args):
raise CommandException('You already have %s installed.' %
update_from_uri_str,
informational=True)
else:
raise CommandException(
'You already have the latest gsutil release '
'installed.',
informational=True)
print(
('This command will update to the "%s" version of\ngsutil at %s') %
(latest_version_string, self.gsutil_bin_dir))
self._ExplainIfSudoNeeded(tf, dirs_to_remove)
answer = raw_input('Proceed? [y/N] ')
if not answer or answer.lower()[0] != 'y':
self._CleanUpUpdateCommand(tf, dirs_to_remove)
raise CommandException('Not running update.', informational=True)
# Ignore keyboard interrupts during the update to reduce the chance someone
# hitting ^C leaves gsutil in a broken state.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# self.gsutil_bin_dir lists the path where the code should end up (like
# /usr/local/gsutil), which is one level down from the relative path in the
# tarball (since the latter creates files in ./gsutil). So, we need to
# extract at the parent directory level.
gsutil_bin_parent_dir = os.path.dirname(self.gsutil_bin_dir)
# Extract tarball to a temporary directory in a sibling to gsutil_bin_dir.
old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir)
dirs_to_remove.append(old_dir)
dirs_to_remove.append(new_dir)
self._EnsureDirsSafeForUpdate(dirs_to_remove)
try:
tf.extractall(path=new_dir)
except Exception, e:
self._CleanUpUpdateCommand(tf, dirs_to_remove)
raise CommandException('Update failed: %s.' % e)
def __init__(
self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path="/",
provider="aws",
security_token=None,
suppress_consec_slashes=True,
validate_certs=True,
):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch. The factory should have
a similar interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
:type suppress_consec_slashes: bool
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type validate_certs: bool
:param validate_certs: Controls whether SSL certificates
will be validated or not. Defaults to True.
"""
self.suppress_consec_slashes = suppress_consec_slashes
self.num_retries = 6
# Override passed-in is_secure setting if value was defined in config.
if config.has_option("Boto", "is_secure"):
is_secure = config.getboolean("Boto", "is_secure")
self.is_secure = is_secure
# Whether or not to validate server certificates.
# The default is now to validate certificates. This can be
# overridden in the boto config file are by passing an
# explicit validate_certs parameter to the class constructor.
self.https_validate_certificates = config.getbool("Boto", "https_validate_certificates", validate_certs)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later."
)
self.ca_certificates_file = config.get_value("Boto", "ca_certificates_file", DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror, httplib.BadStatusLine)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if is_secure:
self.protocol = "https"
else:
self.protocol = "http"
self.host = host
self.path = path
# if the value passed in for debug
if not isinstance(debug, (int, long)):
debug = 0
self.debug = config.getint("Boto", "debug", debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option("Boto", "http_socket_timeout"):
timeout = config.getint("Boto", "http_socket_timeout")
self.http_connection_kwargs["timeout"] = timeout
if isinstance(provider, Provider):
# Allow overriding Provider
self.provider = provider
else:
self._provider_type = provider
self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
self._pool = ConnectionPool()
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(host, config, self.provider, self._required_auth_capability())
if getattr(self, "AuthServiceName", None) is not None:
self.auth_service_name = self.AuthServiceName
