def __init__(self, name, access_key=None, secret_key=None,
security_token=None, profile_name=None):
self.host = None
self.port = None
self.host_header = None
self.access_key = access_key
self.secret_key = secret_key
self.security_token = security_token
self.profile_name = profile_name
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self._credential_expiry_time = None
# Load shared credentials file if it exists
shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
self.shared_credentials = Config(do_load=False)
if os.path.isfile(shared_path):
self.shared_credentials.load_from_path(shared_path)
self.get_credentials(access_key, secret_key, security_token, profile_name)
self.configure_headers()
self.configure_errors()
# Allow config file to override default host and port.
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_opt_name):
self.host = config.get('Credentials', host_opt_name)
port_opt_name = '%s_port' % self.HostKeyMap[self.name]
if config.has_option('Credentials', port_opt_name):
self.port = config.getint('Credentials', port_opt_name)
host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_header_opt_name):
self.host_header = config.get('Credentials', host_header_opt_name)
def __init__(self, name, access_key=None, secret_key=None,
security_token=None, profile_name=None):
self.host = None
self.port = None
self.host_header = None
self.access_key = access_key
self.secret_key = secret_key
self.security_token = security_token
self.profile_name = profile_name
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self._credential_expiry_time = None
# Load shared credentials file if it exists
shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
self.shared_credentials = Config(do_load=False)
if os.path.isfile(shared_path):
self.shared_credentials.load_from_path(shared_path)
self.get_credentials(access_key, secret_key, security_token, profile_name)
self.configure_headers()
self.configure_errors()
# Allow config file to override default host and port.
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_opt_name):
self.host = config.get('Credentials', host_opt_name)
port_opt_name = '%s_port' % self.HostKeyMap[self.name]
if config.has_option('Credentials', port_opt_name):
self.port = config.getint('Credentials', port_opt_name)
host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_header_opt_name):
self.host_header = config.get('Credentials', host_header_opt_name)
def get_credentials(self, access_key=None, secret_key=None, security_token=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self._access_key = access_key
elif os.environ.has_key(access_key_name.upper()):
self._access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self._access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self._secret_key = secret_key
elif os.environ.has_key(secret_key_name.upper()):
self._secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self._secret_key = config.get('Credentials', secret_key_name)
security_token_name = self.HeaderInfoMap[self.name][SECURITY_TOKEN_HEADER_KEY]
security_token_name_env = security_token_name.upper().replace('-', '_')
if security_token is not None:
self._security_token = security_token
elif os.environ.has_key(security_token_name_env):
self._security_token = os.environ[security_token_name_env]
elif config.has_option('Credentials', security_token_name):
self._security_token = config.get('Credentials', security_token_name)
if isinstance(self._secret_key, unicode):
# the secret key must be bytes and not unicode to work
# properly with hmac.new (see http://bugs.python.org/issue5285)
self._secret_key = str(self._secret_key)
stsagent_command = os.environ.get('AWS_STSAGENT')
if stsagent_command:
self.stsagent = stsagent.STSAgent(stsagent_command, 60)
def __init__(self, name, access_key=None, secret_key=None,
security_token=None, profile_name=None):
self.host = None
self.port = None
self.host_header = None
self.access_key = access_key
self.secret_key = secret_key
self.security_token = security_token
self.profile_name = profile_name
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self._credential_expiry_time = None
self.get_credentials(access_key, secret_key, security_token, profile_name)
self.configure_headers()
self.configure_errors()
# Allow config file to override default host and port.
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_opt_name):
self.host = config.get('Credentials', host_opt_name)
port_opt_name = '%s_port' % self.HostKeyMap[self.name]
if config.has_option('Credentials', port_opt_name):
self.port = config.getint('Credentials', port_opt_name)
host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_header_opt_name):
self.host_header = config.get('Credentials', host_header_opt_name)
def __init__(self, name, access_key=None, secret_key=None,
security_token=None):
self.host = None
self.port = None
self.host_header = None
self.access_key = access_key
self.secret_key = secret_key
self.security_token = security_token
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self._credential_expiry_time = None
self.get_credentials(access_key, secret_key)
self.configure_headers()
self.configure_errors()
# Allow config file to override default host and port.
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_opt_name):
self.host = config.get('Credentials', host_opt_name)
port_opt_name = '%s_port' % self.HostKeyMap[self.name]
if config.has_option('Credentials', port_opt_name):
self.port = config.getint('Credentials', port_opt_name)
host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_header_opt_name):
self.host_header = config.get('Credentials', host_header_opt_name)
def get_credentials(self,
access_key=None,
secret_key=None,
security_token=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
elif os.environ.has_key(access_key_name.upper()):
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self.secret_key = secret_key
elif os.environ.has_key(secret_key_name.upper()):
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
security_token_name = self.HeaderInfoMap[
self.name][SECURITY_TOKEN_HEADER_KEY]
security_token_name_env = security_token_name.upper().replace('-', '_')
if security_token is not None:
self.security_token = security_token
elif os.environ.has_key(security_token_name_env):
self.security_token = os.environ[security_token_name_env]
elif config.has_option('Credentials', security_token_name):
self.secret_key = config.get('Credentials', security_token_name)
if isinstance(self.secret_key, unicode):
# the secret key must be bytes and not unicode to work
# properly with hmac.new (see http://bugs.python.org/issue5285)
self.secret_key = str(self.secret_key)
def _ServiceAccountCredentialsPresent(self):
# TODO: Currently, service accounts cannot be project owners (unless
# they are grandfathered). Unfortunately, setting a canned ACL other
# than project-private, the ACL that buckets get by default, removes
# project-editors access from the bucket ACL. So any canned ACL that would
# actually represent a change the bucket would also orphan the service
# account's access to the bucket. If service accounts can be owners
# in the future, remove this function and update all callers.
return (config.has_option('Credentials', 'gs_service_key_file') or
config.has_option('GoogleCompute', 'service_account'))
def _ServiceAccountCredentialsPresent(self):
# TODO: Currently, service accounts cannot be project owners (unless
# they are grandfathered). Unfortunately, setting a canned ACL other
# than project-private, the ACL that buckets get by default, removes
# project-editors access from the bucket ACL. So any canned ACL that would
# actually represent a change the bucket would also orphan the service
# account's access to the bucket. If service accounts can be owners
# in the future, remove this function and update all callers.
return (config.has_option('Credentials', 'gs_service_key_file') or
config.has_option('GoogleCompute', 'service_account'))
def get_credentials(self,
access_key=None,
secret_key=None,
security_token=None):
access_key_name, secret_key_name, security_token_name = self.CredentialMap[
self.name]
if access_key is not None:
self.access_key = access_key
boto.log.debug("Using access key provided by client.")
elif access_key_name.upper() in os.environ:
self.access_key = os.environ[access_key_name.upper()]
boto.log.debug("Using access key found in environment variable.")
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
boto.log.debug("Using access key found in config file.")
if secret_key is not None:
self.secret_key = secret_key
boto.log.debug("Using secret key provided by client.")
elif secret_key_name.upper() in os.environ:
self.secret_key = os.environ[secret_key_name.upper()]
boto.log.debug("Using secret key found in environment variable.")
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
boto.log.debug("Using secret key found in config file.")
elif config.has_option('Credentials', 'keyring'):
keyring_name = config.get('Credentials', 'keyring')
try:
import keyring
except ImportError:
boto.log.error("The keyring module could not be imported. "
"For keyring support, install the keyring "
"module.")
raise
self.secret_key = keyring.get_password(keyring_name,
self.access_key)
boto.log.debug("Using secret key found in keyring.")
if security_token is not None:
self.security_token = security_token
boto.log.debug("Using security token provided by client.")
elif security_token_name is not None:
if security_token_name.upper() in os.environ:
self.security_token = os.environ[security_token_name.upper()]
boto.log.debug("Using security token found in environment"
" variable.")
elif config.has_option('Credentials', security_token_name):
self.security_token = config.get('Credentials',
security_token_name)
boto.log.debug("Using security token found in config file.")
if ((self._access_key is None or self._secret_key is None)
and self.MetadataServiceSupport[self.name]):
self._populate_keys_from_metadata_server()
self._secret_key = self._convert_key_to_str(self._secret_key)
def confirm_aws_creds():
logger.debug("Confirming AWS creds in some section in boto.config")
akid = False
skid = False
for section in config.sections():
akid = akid or config.has_option(section, 'aws_access_key_id')
askid = skid or config.has_option(section, 'aws_secret_access_key')
if not akid and skid:
raise RuntimeError("No aws creds recognized by boto.")
logger.debug("AWS creds present")
def get_credentials(self, access_key=None, secret_key=None,
security_token=None):
access_key_name, secret_key_name, security_token_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
boto.log.debug("Using access key provided by client.")
elif access_key_name.upper() in os.environ:
self.access_key = os.environ[access_key_name.upper()]
boto.log.debug("Using access key found in environment variable.")
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
boto.log.debug("Using access key found in config file.")
if secret_key is not None:
self.secret_key = secret_key
boto.log.debug("Using secret key provided by client.")
elif secret_key_name.upper() in os.environ:
self.secret_key = os.environ[secret_key_name.upper()]
boto.log.debug("Using secret key found in environment variable.")
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
boto.log.debug("Using secret key found in config file.")
elif config.has_option('Credentials', 'keyring'):
keyring_name = config.get('Credentials', 'keyring')
try:
import keyring
except ImportError:
boto.log.error("The keyring module could not be imported. "
"For keyring support, install the keyring "
"module.")
raise
self.secret_key = keyring.get_password(
keyring_name, self.access_key)
boto.log.debug("Using secret key found in keyring.")
if security_token is not None:
self.security_token = security_token
boto.log.debug("Using security token provided by client.")
elif security_token_name is not None:
if security_token_name.upper() in os.environ:
self.security_token = os.environ[security_token_name.upper()]
boto.log.debug("Using security token found in environment"
" variable.")
elif config.has_option('Credentials', security_token_name):
self.security_token = config.get('Credentials',
security_token_name)
boto.log.debug("Using security token found in config file.")
if ((self._access_key is None or self._secret_key is None) and
self.MetadataServiceSupport[self.name]):
self._populate_keys_from_metadata_server()
self._secret_key = self._convert_key_to_str(self._secret_key)
def GetApiSelector(self, provider=None):
"""Returns a cs_api_map.ApiSelector based on input and configuration.
Args:
provider: Provider to return the ApiSelector for. If None, class-wide
default is used.
Returns:
cs_api_map.ApiSelector that will be used for calls to the delegator
for this provider.
"""
selected_provider = provider or self.provider
if not selected_provider:
raise ArgumentException('No provider selected for CloudApi')
if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP]
or self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
not in self.api_map[ApiMapConstants.API_MAP][selected_provider]
):
raise ArgumentException(
'No default api available for provider %s' % selected_provider)
if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
raise ArgumentException(
'No supported apis available for provider %s' %
selected_provider)
api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
# If we have only HMAC credentials for Google Cloud Storage, we must use
# the XML API as the JSON API does not support HMAC.
#
# Technically if we have only HMAC credentials, we should still be able to
# access public read resources via the JSON API, but the XML API can do
# that just as well. It is better to use it than inspect the credentials on
# every HTTP call.
if (provider == 'gs' and
not config.has_option('Credentials', 'gs_oauth2_refresh_token')
and
not (config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials',
'gs_service_key_file')) and
(config.has_option('Credentials', 'gs_access_key_id')
and config.has_option('Credentials', 'gs_secret_access_key'))):
api = ApiSelector.XML
# Try to force the user's preference to a supported API.
elif self.prefer_api in (
self.api_map[ApiMapConstants.SUPPORT_MAP][selected_provider]):
api = self.prefer_api
return api
def get_credentials(self, access_key=None, secret_key=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
elif os.environ.has_key(access_key_name.upper()):
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self.secret_key = secret_key
elif os.environ.has_key(secret_key_name.upper()):
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
def get_credentials(self, access_key=None, secret_key=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key:
self.access_key = access_key
elif os.environ.has_key(access_key_name.upper()):
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key:
self.secret_key = secret_key
elif os.environ.has_key(secret_key_name.upper()):
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
def __init__(
self,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
host=None,
debug=0,
https_connection_factory=None,
):
if not host:
if config.has_option("MTurk", "sandbox") and config.get("MTurk", "sandbox") == "True":
host = "mechanicalturk.sandbox.amazonaws.com"
else:
host = "mechanicalturk.amazonaws.com"
AWSQueryConnection.__init__(
self,
aws_access_key_id,
aws_secret_access_key,
is_secure,
port,
proxy,
proxy_port,
proxy_user,
proxy_pass,
host,
debug,
https_connection_factory,
)
def __init__(self,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
host=None,
debug=0,
https_connection_factory=None,
security_token=None,
profile_name=None):
if not host:
if config.has_option('MTurk', 'sandbox') and config.get(
'MTurk', 'sandbox') == 'True':
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
self.debug = debug
super(MTurkConnection, self).__init__(aws_access_key_id,
aws_secret_access_key,
is_secure,
port,
proxy,
proxy_port,
proxy_user,
proxy_pass,
host,
debug,
https_connection_factory,
security_token=security_token,
profile_name=profile_name)
def ConfigureNoOpAuthIfNeeded():
"""Sets up no-op auth handler if no boto credentials are configured."""
if not HasConfiguredCredentials():
if (config.has_option('Credentials', 'gs_service_client_id') and
not HAS_CRYPTO):
if system_util.InvokedViaCloudSdk():
raise CommandException('\n'.join(
textwrap.wrap(
'Your gsutil is configured with an OAuth2 service account, but '
'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
'Service account authentication requires one of these libraries; '
'please reactivate your service account via the gcloud auth '
'command and ensure any gcloud packages necessary for '
'service accounts are present.')))
else:
raise CommandException('\n'.join(
textwrap.wrap(
'Your gsutil is configured with an OAuth2 service account, but '
'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
'Service account authentication requires one of these libraries; '
'please install either of them to proceed, or configure a '
'different type of credentials with "gsutil config".')))
else:
# With no boto config file the user can still access publicly readable
# buckets and objects.
from gslib import no_op_auth_plugin # pylint: disable=unused-variable
def PrintTrackerDirDeprecationWarningIfNeeded():
# TODO: Remove this along with the tracker_dir config value 1 year after
# 4.6 release date. Use state_dir instead.
if config.has_option('GSUtil', 'resumable_tracker_dir'):
sys.stderr.write('Warning: you have set resumable_tracker_dir in your '
'.boto configuration file. This configuration option is '
'deprecated; please use the state_dir configuration '
'option instead.\n')
def PrintTrackerDirDeprecationWarningIfNeeded():
# TODO: Remove this along with the tracker_dir config value 1 year after
# 4.6 release date. Use state_dir instead.
if config.has_option('GSUtil', 'resumable_tracker_dir'):
sys.stderr.write('Warning: you have set resumable_tracker_dir in your '
'.boto configuration file. This configuration option is '
'deprecated; please use the state_dir configuration '
'option instead.\n')
def GetApiSelector(self, provider=None):
"""Returns a cs_api_map.ApiSelector based on input and configuration.
Args:
provider: Provider to return the ApiSelector for. If None, class-wide
default is used.
Returns:
cs_api_map.ApiSelector that will be used for calls to the delegator
for this provider.
"""
selected_provider = provider or self.provider
if not selected_provider:
raise ArgumentException('No provider selected for CloudApi')
if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP] or
self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] not in
self.api_map[ApiMapConstants.API_MAP][selected_provider]):
raise ArgumentException('No default api available for provider %s' %
selected_provider)
if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
raise ArgumentException('No supported apis available for provider %s' %
selected_provider)
api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
# If we have only HMAC credentials for Google Cloud Storage, we must use
# the XML API as the JSON API does not support HMAC.
#
# Technically if we have only HMAC credentials, we should still be able to
# access public read resources via the JSON API, but the XML API can do
# that just as well. It is better to use it than inspect the credentials on
# every HTTP call.
if (provider == 'gs' and
not config.has_option('Credentials', 'gs_oauth2_refresh_token') and
not (config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials', 'gs_service_key_file')) and
(config.has_option('Credentials', 'gs_access_key_id')
and config.has_option('Credentials', 'gs_secret_access_key'))):
api = ApiSelector.XML
# Try to force the user's preference to a supported API.
elif self.prefer_api in (self.api_map[ApiMapConstants.SUPPORT_MAP]
[selected_provider]):
api = self.prefer_api
return api
def HasConfiguredCredentials():
"""Determines if boto credential/config file exists."""
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
config.has_option('Credentials', 'gs_secret_access_key'))
has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
config.has_option('Credentials', 'aws_secret_access_key'))
has_oauth_creds = (config.has_option('Credentials',
'gs_oauth2_refresh_token'))
has_service_account_creds = (
HAS_CRYPTO and
config.has_option('Credentials', 'gs_service_client_id') and
config.has_option('Credentials', 'gs_service_key_file'))
if (has_goog_creds or has_amzn_creds or has_oauth_creds or
has_service_account_creds):
return True
valid_auth_handler = None
try:
valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost,
config,
Provider('google'),
requested_capability=['s3'])
# Exclude the no-op auth handler as indicating credentials are configured.
# Note we can't use isinstance() here because the no-op module may not be
# imported so we can't get a reference to the class type.
if 'NoOpAuth' == getattr(
getattr(valid_auth_handler, '__class__', None),
'__name__',
None): # yapf: disable
valid_auth_handler = None
except NoAuthHandlerFound:
pass
return valid_auth_handler
def HasConfiguredCredentials():
"""Determines if boto credential/config file exists."""
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
config.has_option('Credentials', 'gs_secret_access_key'))
has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
config.has_option('Credentials', 'aws_secret_access_key'))
has_oauth_creds = (
config.has_option('Credentials', 'gs_oauth2_refresh_token'))
has_service_account_creds = (
HAS_CRYPTO and
config.has_option('Credentials', 'gs_service_client_id') and
config.has_option('Credentials', 'gs_service_key_file'))
if (has_goog_creds or has_amzn_creds or has_oauth_creds or
has_service_account_creds):
return True
valid_auth_handler = None
try:
valid_auth_handler = boto.auth.get_auth_handler(
GSConnection.DefaultHost, config, Provider('google'),
requested_capability=['s3'])
# Exclude the no-op auth handler as indicating credentials are configured.
# Note we can't use isinstance() here because the no-op module may not be
# imported so we can't get a reference to the class type.
if getattr(getattr(valid_auth_handler, '__class__', None),
'__name__', None) == 'NoOpAuth':
valid_auth_handler = None
except NoAuthHandlerFound:
pass
return valid_auth_handler
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, host='ec2.amazonaws.com', debug=0,
https_connection_factory=None):
if config.has_option('Boto', 'ec2_host'):
host = config.get('Boto', 'ec2_host')
AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key,
is_secure, port, proxy, proxy_port, proxy_user, proxy_pass,
host, debug, https_connection_factory)
def get_credentials(self, access_key=None, secret_key=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
elif access_key_name.upper() in os.environ:
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self.secret_key = secret_key
elif secret_key_name.upper() in os.environ:
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
if isinstance(self.secret_key, unicode):
# the secret key must be bytes and not unicode to work
# properly with hmac.new (see http://bugs.python.org/issue5285)
self.secret_key = str(self.secret_key)
def get_credentials(self, access_key=None, secret_key=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
elif os.environ.has_key(access_key_name.upper()):
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self.secret_key = secret_key
elif os.environ.has_key(secret_key_name.upper()):
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
if isinstance(self.secret_key, unicode):
# the secret key must be bytes and not unicode to work
# properly with hmac.new (see http://bugs.python.org/issue5285)
self.secret_key = str(self.secret_key)
def get_credentials(self, access_key=None, secret_key=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
elif access_key_name.upper() in os.environ:
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self.secret_key = secret_key
elif secret_key_name.upper() in os.environ:
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
if ((self._access_key is None or self._secret_key is None) and
self.MetadataServiceSupport[self.name]):
self._populate_keys_from_metadata_server()
self._secret_key = self._convert_key_to_str(self._secret_key)
def __init__(self, name, access_key=None, secret_key=None):
self.host = None
self.access_key = None
self.secret_key = None
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self.get_credentials(access_key, secret_key)
self.configure_headers()
# allow config file to override default host
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if (config.has_option('Credentials', host_opt_name)):
self.host = config.get('Credentials', host_opt_name)
def __init__(self, name, access_key=None, secret_key=None):
self.host = None
self.access_key = None
self.secret_key = None
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self.get_credentials(access_key, secret_key)
self.configure_headers()
# allow config file to override default host
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if (config.has_option('Credentials', host_opt_name)):
self.host = config.get('Credentials', host_opt_name)
def HasConfiguredCredentials():
"""Determines if boto credential/config file exists."""
config = boto.config
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
config.has_option('Credentials', 'gs_secret_access_key'))
has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
config.has_option('Credentials', 'aws_secret_access_key'))
has_oauth_creds = (
config.has_option('Credentials', 'gs_oauth2_refresh_token'))
has_service_account_creds = (HAS_CRYPTO and
config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials', 'gs_service_key_file'))
has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
return (has_goog_creds or has_amzn_creds or has_oauth_creds
or has_auth_plugins or has_service_account_creds)
def HasConfiguredCredentials():
"""Determines if boto credential/config file exists."""
config = boto.config
has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
config.has_option('Credentials', 'gs_secret_access_key'))
has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id')
and config.has_option('Credentials',
'aws_secret_access_key'))
has_oauth_creds = (config.has_option('Credentials',
'gs_oauth2_refresh_token'))
has_service_account_creds = (
HAS_CRYPTO and config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials', 'gs_service_key_file'))
has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
return (has_goog_creds or has_amzn_creds or has_oauth_creds
or has_auth_plugins or has_service_account_creds)
def __init__(self, name, access_key=None, secret_key=None, security_token=None):
self.host = None
self.access_key = access_key
self.secret_key = secret_key
self.security_token = security_token
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self.get_credentials(access_key, secret_key)
self.configure_headers()
self.configure_errors()
# allow config file to override default host
host_opt_name = "%s_host" % self.HostKeyMap[self.name]
if config.has_option("Credentials", host_opt_name):
self.host = config.get("Credentials", host_opt_name)
def HasConfiguredCredentials():
"""Determines if boto credential/config file exists."""
config = boto.config
has_goog_creds = config.has_option("Credentials", "gs_access_key_id") and config.has_option(
"Credentials", "gs_secret_access_key"
)
has_amzn_creds = config.has_option("Credentials", "aws_access_key_id") and config.has_option(
"Credentials", "aws_secret_access_key"
)
has_oauth_creds = config.has_option("Credentials", "gs_oauth2_refresh_token")
has_service_account_creds = (
HAS_CRYPTO
and config.has_option("Credentials", "gs_service_client_id")
and config.has_option("Credentials", "gs_service_key_file")
)
has_auth_plugins = config.has_option("Plugin", "plugin_directory")
return has_goog_creds or has_amzn_creds or has_oauth_creds or has_auth_plugins or has_service_account_creds
def ConfigureNoOpAuthIfNeeded():
"""Sets up no-op auth handler if no boto credentials are configured."""
if not HasConfiguredCredentials():
if (config.has_option('Credentials', 'gs_service_client_id')
and not HAS_CRYPTO):
raise CommandException('\n'.join(textwrap.wrap(
'Your gsutil is configured with an OAuth2 service account, but you '
'do not have PyOpenSSL or PyCrypto 2.6 or later installed. Service '
'account authentication requires one of these libraries; please '
'install either of them to proceed, or configure a different type '
'of credentials with "gsutil config".')))
else:
# With no boto config file the user can still access publicly readable
# buckets and objects.
from gslib import no_op_auth_plugin # pylint: disable=unused-variable
def get_credentials(self, access_key=None, secret_key=None):
access_key_name, secret_key_name = self.CredentialMap[self.name]
if access_key is not None:
self.access_key = access_key
elif access_key_name.upper() in os.environ:
self.access_key = os.environ[access_key_name.upper()]
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
if secret_key is not None:
self.secret_key = secret_key
elif secret_key_name.upper() in os.environ:
self.secret_key = os.environ[secret_key_name.upper()]
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
if ((self.access_key is None or self.secret_key is None)
and self.MetadataServiceSupport[self.name]):
self._populate_keys_from_metadata_server()
if isinstance(self.secret_key, unicode):
# the secret key must be bytes and not unicode to work
# properly with hmac.new (see http://bugs.python.org/issue5285)
self.secret_key = str(self.secret_key)
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None,
host=None, debug=0,
https_connection_factory=None):
if not host:
if config.has_option('MTurk', 'sandbox') and config.get('MTurk', 'sandbox') == 'True':
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
AWSQueryConnection.__init__(self, aws_access_key_id,
aws_secret_access_key,
is_secure, port, proxy, proxy_port,
proxy_user, proxy_pass, host, debug,
https_connection_factory)
def ConfigureNoOpAuthIfNeeded():
"""Sets up no-op auth handler if no boto credentials are configured."""
if not HasConfiguredCredentials():
if (config.has_option('Credentials', 'gs_service_client_id')
and not HAS_CRYPTO):
raise CommandException('\n'.join(
textwrap.wrap(
'Your gsutil is configured with an OAuth2 service account, but you '
'do not have PyOpenSSL or PyCrypto 2.6 or later installed. Service '
'account authentication requires one of these libraries; please '
'install either of them to proceed, or configure a different type '
'of credentials with "gsutil config".')))
else:
# With no boto config file the user can still access publicly readable
# buckets and objects.
from gslib import no_op_auth_plugin # pylint: disable=unused-variable
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=False, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None,
host=None, debug=0,
https_connection_factory=None):
if not host:
if config.has_option('MTurk', 'sandbox') and config.get('MTurk', 'sandbox') == 'True':
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
AWSQueryConnection.__init__(self, aws_access_key_id,
aws_secret_access_key,
is_secure, port, proxy, proxy_port,
proxy_user, proxy_pass, host, debug,
https_connection_factory)
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None,
host=None, debug=0,
https_connection_factory=None, security_token=None,
profile_name=None, provider='aws'):
if not host:
if config.has_option('MTurk', 'sandbox') and config.get('MTurk', 'sandbox') == 'True':
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
self.debug = debug
super(MTurkConnection, self).__init__(aws_access_key_id,
aws_secret_access_key,
is_secure, port, proxy, proxy_port,
proxy_user, proxy_pass, host, debug,
https_connection_factory,
security_token=security_token,
profile_name=profile_name, provider=provider)
def __init__(
self,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
host=None,
debug=0,
https_connection_factory=None,
security_token=None,
profile_name=None,
):
if not host:
if config.has_option("MTurk", "sandbox") and config.get("MTurk", "sandbox") == "True":
host = "mechanicalturk.sandbox.amazonaws.com"
else:
host = "mechanicalturk.amazonaws.com"
self.debug = debug
super(MTurkConnection, self).__init__(
aws_access_key_id,
aws_secret_access_key,
is_secure,
port,
proxy,
proxy_port,
proxy_user,
proxy_pass,
host,
debug,
https_connection_factory,
security_token=security_token,
profile_name=profile_name,
)
def ConfigureNoOpAuthIfNeeded():
"""Sets up no-op auth handler if no boto credentials are configured."""
if not HasConfiguredCredentials():
if (config.has_option('Credentials', 'gs_service_client_id')
and not HAS_CRYPTO):
if os.environ.get('CLOUDSDK_WRAPPER') == '1':
raise CommandException('\n'.join(textwrap.wrap(
'Your gsutil is configured with an OAuth2 service account, but '
'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
'Service account authentication requires one of these libraries; '
'please reactivate your service account via the gcloud auth '
'command and ensure any gcloud packages necessary for '
'service accounts are present.')))
else:
raise CommandException('\n'.join(textwrap.wrap(
'Your gsutil is configured with an OAuth2 service account, but '
'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. '
'Service account authentication requires one of these libraries; '
'please install either of them to proceed, or configure a '
'different type of credentials with "gsutil config".')))
else:
# With no boto config file the user can still access publicly readable
# buckets and objects.
from gslib import no_op_auth_plugin # pylint: disable=unused-variable
def _BotoIsSecure():
for cfg_var in ('is_secure', 'https_validate_certificates'):
if (config.has_option('Boto', cfg_var)
and not config.getboolean('Boto', cfg_var)):
return False, cfg_var
return True, ''
def __init__(self, host, aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/',
provider='aws', security_token=None,
suppress_consec_slashes=True,
validate_certs=True, profile_name=None):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:keyword str security_token: The security token associated with
temporary credentials issued by STS. Optional unless using
temporary credentials. If none is specified, the environment
variable ``AWS_SECURITY_TOKEN`` is used if defined.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch. The factory should have
a similar interface to L{http_client.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
:type suppress_consec_slashes: bool
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type validate_certs: bool
:param validate_certs: Controls whether SSL certificates
will be validated or not. Defaults to True.
:type profile_name: str
:param profile_name: Override usual Credentials section in config
file to use a named set of keys instead.
"""
self.suppress_consec_slashes = suppress_consec_slashes
self.num_retries = 6
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates.
# The default is now to validate certificates. This can be
# overridden in the boto config file are by passing an
# explicit validate_certs parameter to the class constructor.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates',
validate_certs)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
certs_file = config.get_value(
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
if certs_file == 'system':
certs_file = None
self.ca_certificates_file = certs_file
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from http_client that we want to catch and retry
self.http_exceptions = (http_client.HTTPException, socket.error,
socket.gaierror, http_client.BadStatusLine)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
# if the value passed in for debug
if not isinstance(debug, six.integer_types):
debug = 0
self.debug = config.getint('Boto', 'debug', debug)
self.host_header = None
# Timeout used to tell http_client how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
# If timeout isn't defined in boto config file, use 70 second
# default as recommended by
# http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html
self.http_connection_kwargs['timeout'] = config.getint(
'Boto', 'http_socket_timeout', 70)
if isinstance(provider, Provider):
# Allow overriding Provider
self.provider = provider
else:
self._provider_type = provider
self.provider = Provider(self._provider_type,
aws_access_key_id,
aws_secret_access_key,
security_token,
profile_name)
# Allow config file to override default host, port, and host header.
if self.provider.host:
self.host = self.provider.host
if self.provider.port:
self.port = self.provider.port
if self.provider.host_header:
self.host_header = self.provider.host_header
self._pool = ConnectionPool()
self._connection = (self.host, self.port, self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
if getattr(self, 'AuthServiceName', None) is not None:
self.auth_service_name = self.AuthServiceName
self.request_hook = None
def _HasGceCreds():
return config.has_option('GoogleCompute', 'service_account')
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/',
provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value('Boto',
'ca_certificates_file',
DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider, aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def get_credentials(self,
access_key=None,
secret_key=None,
security_token=None,
profile_name=None):
access_key_name, secret_key_name, security_token_name, \
profile_name_name = self.CredentialMap[self.name]
# Load profile from shared environment variable if it was not
# already passed in and the environment variable exists
if profile_name is None and profile_name_name is not None and \
profile_name_name.upper() in os.environ:
profile_name = os.environ[profile_name_name.upper()]
shared = self.shared_credentials
if access_key is not None:
self.access_key = access_key
boto.log.debug("Using access key provided by client.")
elif access_key_name.upper() in os.environ:
self.access_key = os.environ[access_key_name.upper()]
boto.log.debug("Using access key found in environment variable.")
elif profile_name is not None:
if shared.has_option(profile_name, access_key_name):
self.access_key = shared.get(profile_name, access_key_name)
boto.log.debug("Using access key found in shared credential "
"file for profile %s." % profile_name)
elif config.has_option("profile %s" % profile_name,
access_key_name):
self.access_key = config.get("profile %s" % profile_name,
access_key_name)
boto.log.debug("Using access key found in config file: "
"profile %s." % profile_name)
else:
raise ProfileNotFoundError('Profile "%s" not found!' %
profile_name)
elif shared.has_option('default', access_key_name):
self.access_key = shared.get('default', access_key_name)
boto.log.debug("Using access key found in shared credential file.")
elif config.has_option('Credentials', access_key_name):
self.access_key = config.get('Credentials', access_key_name)
boto.log.debug("Using access key found in config file.")
if secret_key is not None:
self.secret_key = secret_key
boto.log.debug("Using secret key provided by client.")
elif secret_key_name.upper() in os.environ:
self.secret_key = os.environ[secret_key_name.upper()]
boto.log.debug("Using secret key found in environment variable.")
elif profile_name is not None:
if shared.has_option(profile_name, secret_key_name):
self.secret_key = shared.get(profile_name, secret_key_name)
boto.log.debug("Using secret key found in shared credential "
"file for profile %s." % profile_name)
elif config.has_option("profile %s" % profile_name,
secret_key_name):
self.secret_key = config.get("profile %s" % profile_name,
secret_key_name)
boto.log.debug("Using secret key found in config file: "
"profile %s." % profile_name)
else:
raise ProfileNotFoundError('Profile "%s" not found!' %
profile_name)
elif shared.has_option('default', secret_key_name):
self.secret_key = shared.get('default', secret_key_name)
boto.log.debug("Using secret key found in shared credential file.")
elif config.has_option('Credentials', secret_key_name):
self.secret_key = config.get('Credentials', secret_key_name)
boto.log.debug("Using secret key found in config file.")
elif config.has_option('Credentials', 'keyring'):
keyring_name = config.get('Credentials', 'keyring')
try:
import keyring
except ImportError:
boto.log.error("The keyring module could not be imported. "
"For keyring support, install the keyring "
"module.")
raise
self.secret_key = keyring.get_password(keyring_name,
self.access_key)
boto.log.debug("Using secret key found in keyring.")
if security_token is not None:
self.security_token = security_token
boto.log.debug("Using security token provided by client.")
elif ((security_token_name is not None) and (access_key is None)
and (secret_key is None)):
# Only provide a token from the environment/config if the
# caller did not specify a key and secret. Otherwise an
# environment/config token could be paired with a
# different set of credentials provided by the caller
if security_token_name.upper() in os.environ:
self.security_token = os.environ[security_token_name.upper()]
boto.log.debug("Using security token found in environment"
" variable.")
elif shared.has_option(profile_name or 'default',
security_token_name):
self.security_token = shared.get(profile_name or 'default',
security_token_name)
boto.log.debug("Using security token found in shared "
"credential file.")
elif config.has_option('Credentials', security_token_name):
self.security_token = config.get('Credentials',
security_token_name)
boto.log.debug("Using security token found in config file.")
if ((self._access_key is None or self._secret_key is None)
and self.MetadataServiceSupport[self.name]):
self._populate_keys_from_metadata_server()
self._secret_key = self._convert_key_to_str(self._secret_key)
def _HasOauth2UserAccountCreds():
return config.has_option('Credentials', 'gs_oauth2_refresh_token')
def _HasOauth2ServiceAccountCreds():
return config.has_option('Credentials', 'gs_service_key_file')
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
# Setup a database connection to be used in the rest of the code
DB_HOST = os.environ.get('DB_HOST', 'localhost:27017')
client = MongoClient(DB_HOST)
DB_NAME = os.environ.get('DB_NAME', 'dealsdb')
db1 = client[DB_NAME]
if 'DB_USER' in os.environ:
db1.authenticate(os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD'))
on_aws = False #"ON_AWS" in os.environ
if on_aws:
if not botoconfig.has_section('Credentials'):
botoconfig.add_section('Credentials')
if not botoconfig.has_option('Credentials', 'aws_access_key_id'):
botoconfig.set('Credentials', 'aws_access_key_id',
os.environ.get('AWS_KEY'))
if not botoconfig.has_option('Credentials', 'aws_secret_access_key'):
botoconfig.set('Credentials', 'aws_secret_access_key',
os.environ.get('AWS_SECRET'))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'je+95#mqpv5%c2(sdp69q)5(o0_-5%*j-a(s0w%q9@spxaipkv'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
def GetApiSelector(self, provider=None):
"""Returns a cs_api_map.ApiSelector based on input and configuration.
Args:
provider: Provider to return the ApiSelector for. If None, class-wide
default is used.
Returns:
cs_api_map.ApiSelector that will be used for calls to the delegator
for this provider.
"""
selected_provider = provider or self.provider
if not selected_provider:
raise ArgumentException('No provider selected for CloudApi')
if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP]
or self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
not in self.api_map[ApiMapConstants.API_MAP][selected_provider]
):
raise ArgumentException(
'No default api available for provider %s' % selected_provider)
if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
raise ArgumentException(
'No supported apis available for provider %s' %
selected_provider)
api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
using_gs_hmac = (
provider == 'gs'
and not config.has_option('Credentials', 'gs_oauth2_refresh_token')
and
not (config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials', 'gs_service_key_file'))
and (config.has_option('Credentials', 'gs_access_key_id')
and config.has_option('Credentials', 'gs_secret_access_key')))
configured_encryption = (
provider == 'gs'
and (config.has_option('GSUtil', 'encryption_key')
or config.has_option('GSUtil', 'decryption_key1')))
if using_gs_hmac and configured_encryption:
raise CommandException(
'gsutil does not support HMAC credentials with customer-supplied '
'encryption keys (CSEK) or customer-managed KMS encryption keys '
'(CMEK). Please generate and include non-HMAC credentials '
'in your .boto configuration file, or to access public encrypted '
'objects, remove your HMAC credentials.')
# If we have only HMAC credentials for Google Cloud Storage, we must use
# the XML API as the JSON API does not support HMAC.
#
# Technically if we have only HMAC credentials, we should still be able to
# access public read resources via the JSON API, but the XML API can do
# that just as well. It is better to use it than inspect the credentials on
# every HTTP call.
elif using_gs_hmac:
api = ApiSelector.XML
# CSEK and CMEK encryption keys are currently only supported in the
# JSON API implementation (GcsJsonApi). We can't stop XML API users from
# interacting with encrypted objects, since we don't know the object is
# encrypted until after the API call is made, but if they specify
# configuration values we will use JSON.
elif configured_encryption:
api = ApiSelector.JSON
# Try to force the user's preference to a supported API.
elif self.prefer_api in (
self.api_map[ApiMapConstants.SUPPORT_MAP][selected_provider]):
api = self.prefer_api
if (api == ApiSelector.XML and context_config.get_context_config() and
context_config.get_context_config().use_client_certificate):
raise ArgumentException(
'User enabled mTLS by setting "use_client_certificate", but mTLS'
' is not supported for the selected XML API. Try configuring for '
' the GCS JSON API or setting "use_client_certificate" to "False" in'
' the Boto config.')
return api
def GetApiSelector(self, provider=None):
"""Returns a cs_api_map.ApiSelector based on input and configuration.
Args:
provider: Provider to return the ApiSelector for. If None, class-wide
default is used.
Returns:
cs_api_map.ApiSelector that will be used for calls to the delegator
for this provider.
"""
selected_provider = provider or self.provider
if not selected_provider:
raise ArgumentException("No provider selected for CloudApi")
if (
selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP]
or self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
not in self.api_map[ApiMapConstants.API_MAP][selected_provider]
):
raise ArgumentException("No default api available for provider %s" % selected_provider)
if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
raise ArgumentException("No supported apis available for provider %s" % selected_provider)
api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
using_gs_hmac = (
provider == "gs"
and not config.has_option("Credentials", "gs_oauth2_refresh_token")
and not (
config.has_option("Credentials", "gs_service_client_id")
and config.has_option("Credentials", "gs_service_key_file")
)
and (
config.has_option("Credentials", "gs_access_key_id")
and config.has_option("Credentials", "gs_secret_access_key")
)
)
configured_encryption = provider == "gs" and (
config.has_option("GSUtil", "encryption_key") or config.has_option("GSUtil", "decryption_key1")
)
if using_gs_hmac and configured_encryption:
raise CommandException(
"gsutil does not support HMAC credentials with customer-supplied "
"encryption keys. Please generate and include non-HMAC credentials "
"in your .boto configuration file, or to access public encrypted "
"objects, remove your HMAC credentials."
)
# If we have only HMAC credentials for Google Cloud Storage, we must use
# the XML API as the JSON API does not support HMAC.
#
# Technically if we have only HMAC credentials, we should still be able to
# access public read resources via the JSON API, but the XML API can do
# that just as well. It is better to use it than inspect the credentials on
# every HTTP call.
elif using_gs_hmac:
api = ApiSelector.XML
# Customer-supplied encryption keys are only supported in the JSON API.
# We can't stop XML API users from interacting with encrypted objects,
# since we don't know the object is encrypted until after the API call is
# made, but if they specify configuration values we will use JSON.
elif configured_encryption:
api = ApiSelector.JSON
# Try to force the user's preference to a supported API.
elif self.prefer_api in (self.api_map[ApiMapConstants.SUPPORT_MAP][selected_provider]):
api = self.prefer_api
return api
# Setup a database connection to be used in the rest of the code
DB_HOST = os.environ.get('DB_HOST', 'localhost:27017')
client = MongoClient(DB_HOST)
DB_NAME = os.environ.get('DB_NAME', 'autotest')
AWS_KEY = os.environ.get('AWS_KEY',None)
AWS_SECRET = os.environ.get('AWS_SECRET', None)
db1 = client[DB_NAME]
if 'DB_USER' in os.environ:
db1.authenticate(os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD'))
on_aws = True #"ON_AWS" in os.environ
if on_aws:
if not botoconfig.has_section('Credentials'):
botoconfig.add_section('Credentials')
if not botoconfig.has_option('Credentials', 'aws_access_key_id'):
botoconfig.set('Credentials', 'aws_access_key_id',
os.environ.get('AWS_KEY'))
if not botoconfig.has_option('Credentials', 'aws_secret_access_key'):
botoconfig.set('Credentials', 'aws_secret_access_key',
os.environ.get('AWS_SECRET'))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'z%h7pr=_$j%^l54+xcjco8e+*%y)%j7q^&0w_+j8nfsh=ra_n('
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value(
'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider,
aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
self._pool = ConnectionPool()
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/'):
"""
:type host: string
:param host: The host to make the connection to
:type aws_access_key_id: string
:param aws_access_key_id: AWS Access Key ID (provided by Amazon)
:type aws_secret_access_key: string
:param aws_secret_access_key: Secret Access Key (provided by Amazon)
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:type proxy:
:param proxy:
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: string
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: string
:param proxy_pass: The password to use when connection over a proxy.
:type port: integer
:param port: The port to use to connect
"""
self.num_retries = 5
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
if aws_access_key_id:
self.aws_access_key_id = aws_access_key_id
elif os.environ.has_key('AWS_ACCESS_KEY_ID'):
self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
elif config.has_option('Credentials', 'aws_access_key_id'):
self.aws_access_key_id = config.get('Credentials',
'aws_access_key_id')
if aws_secret_access_key:
self.aws_secret_access_key = aws_secret_access_key
elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'):
self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
elif config.has_option('Credentials', 'aws_secret_access_key'):
self.aws_secret_access_key = config.get('Credentials',
'aws_secret_access_key')
# initialize an HMAC for signatures, make copies with each request
self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha)
if sha256:
self.hmac_256 = hmac.new(self.aws_secret_access_key,
digestmod=sha256)
else:
self.hmac_256 = None
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
def _HasOauth2ServiceAccountCreds():
return config.has_option('Credentials', 'gs_service_key_file')
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/', provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR,)
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
self.provider = Provider(provider,
aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def _HasOauth2UserAccountCreds():
return config.has_option('Credentials', 'gs_oauth2_refresh_token')
def _BotoIsSecure():
for cfg_var in ('is_secure', 'https_validate_certificates'):
if (config.has_option('Boto', cfg_var)
and not config.getboolean('Boto', cfg_var)):
return False, cfg_var
return True, ''
def _HasGceCreds():
return config.has_option('GoogleCompute', 'service_account')
