def has_google_credentials(): global _HAS_GOOGLE_CREDENTIALS if _HAS_GOOGLE_CREDENTIALS is None: provider = Provider('google') if (provider.get_access_key() is None or provider.get_secret_key() is None): _HAS_GOOGLE_CREDENTIALS = False else: _HAS_GOOGLE_CREDENTIALS = True return _HAS_GOOGLE_CREDENTIALS
def _get_session_token(self): if self._need_session_token: boto.log.debug('Creating new Session Token') sts = boto.connect_sts(self._passed_access_key, self._passed_secret_key) token = sts.get_session_token() self.provider = Provider(self._provider_type, token.access_key, token.secret_key, token.session_token) else: self.provider = Provider(self._provider_type) self._auth_handler.update_provider(self.provider)
def cleanup(uri): (unused, regionPath) = uri.split(':') (region, tableName) = regionPath.split('/') provider = Provider('aws') conn = boto.dynamodb.connect_to_region(region, aws_access_key_id=provider.get_access_key(), aws_secret_access_key=provider.get_secret_key()) table = conn.get_table(tableName) for item in table.scan(attributes_to_get=["name", "path"]): if item["path"] == "/" and item["name"] == "/": continue if item["path"] == "global" and item["name"] == "counter": continue if item["path"] == '/' and item['name'] == DELETED_LINKS: continue item.delete()
def HasConfiguredCredentials(): """Determines if boto credential/config file exists.""" has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and config.has_option('Credentials', 'aws_secret_access_key')) has_oauth_creds = (config.has_option('Credentials', 'gs_oauth2_refresh_token')) has_service_account_creds = ( HAS_CRYPTO and config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) if (has_goog_creds or has_amzn_creds or has_oauth_creds or has_service_account_creds): return True valid_auth_handler = None try: valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost, config, Provider('google'), requested_capability=['s3']) # Exclude the no-op auth handler as indicating credentials are configured. # Note we can't use isinstance() here because the no-op module may not be # imported so we can't get a reference to the class type. if 'NoOpAuth' == getattr( getattr(valid_auth_handler, '__class__', None), '__name__', None): # yapf: disable valid_auth_handler = None except NoAuthHandlerFound: pass return valid_auth_handler
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, region=None, path='/', converter=None, validate_certs=True, anon=False, security_token=None, profile_name=None, provider='aws'): """ :type anon: boolean :param anon: If this parameter is True, the ``STSConnection`` object will make anonymous requests, and it will not use AWS Credentials or even search for AWS Credentials to make these requests. """ if not region: region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint, connection_cls=STSConnection, provider=provider) self.region = region self.anon = anon self._mutex = threading.Semaphore() # If an anonymous request is sent, do not try to look for credentials. # So we pass in dummy values for the access key id, secret access # key, and session token. It does not matter that they are # not actual values because the request is anonymous. if self.anon: provider = Provider('aws', NO_CREDENTIALS_PROVIDED, NO_CREDENTIALS_PROVIDED, NO_CREDENTIALS_PROVIDED) super(STSConnection, self).__init__(aws_access_key_id, aws_secret_access_key, is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, self.region.endpoint, debug, https_connection_factory, path, validate_certs=validate_certs, security_token=security_token, profile_name=profile_name, provider=provider)
def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): """Callback to use with `async_aws_sts`. The 'provider' arg is a bit misleading, it is a relic from boto and should probably be left to its default. This will take the new Credentials obj from `async_aws_sts.get_session_token()` and use it to update self.provider, and then will clear the deque of pending requests. A callback is optional. If provided, it must be callable without any arguments. """ def raise_error(): # get out of locked state self.provider.security_token = None if callable(callback): return callback(error=error) else: logging.error(error) raise error if error: if isinstance(error, InvalidClientTokenIdError): # no need to retry if error is due to bad tokens raise_error() else: if attempts > self.max_sts_attempts: raise_error() else: seconds_to_wait = (0.1 * (2**attempts)) logging.warning( "Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) ioloop.IOLoop.current().add_timeout( time.time() + seconds_to_wait, functools.partial(self._update_session_token, attempts=attempts + 1, callback=callback, bypass_lock=True)) return else: self.provider = Provider(provider, creds.access_key, creds.secret_key, creds.session_token) # force the correct auth, with the new provider self._auth_handler = HmacAuthV3HTTPHandler(self.host, None, self.provider) while self.pending_requests: request = self.pending_requests.pop() request() if callable(callback): return callback()
def test_pickle_works(self): provider = Provider('aws', access_key='access_key', secret_key='secret_key') auth = HmacAuthV4Handler('queue.amazonaws.com', None, provider) # Pickle it! pickled = pickle.dumps(auth) # Now restore it auth2 = pickle.loads(pickled) self.assertEqual(auth.host, auth2.host)
def __init__(self, uri): (unused, regionPath) = uri.split(':') (region, tableName) = regionPath.split('/') self.log = logging.getLogger("dynamo-fuse-oper ") self.tableName = tableName self.region = region for reg in boto.dynamodb2.regions(): if reg.name == region: self.regionv2 = reg break provider = Provider('aws') self.conn = boto.dynamodb.connect_to_region(region, aws_access_key_id=provider.get_access_key(), aws_secret_access_key=provider.get_secret_key()) connection = DynamoDBConnection(aws_access_key_id=provider.get_access_key(), aws_secret_access_key=provider.get_secret_key(), region=self.regionv2) try: self.table = self.conn.get_table(tableName) self.tablev2 = Table(tableName, connection=connection) self.blockTable = self.conn.get_table(self.tableName + "Blocks") self.blockTablev2 = Table(self.tableName + "Blocks", connection=connection) except: self.createTable() self.counter = itertools.count() self.counter.next() # start from 1 self.__createRoot() print "Ready"
def createTable(self): provider = Provider('aws') connection = DynamoDBConnection(aws_access_key_id=provider.get_access_key(), aws_secret_access_key=provider.get_secret_key(), region=self.regionv2) self.blockTablev2 = Table.create(self.tableName + "Blocks", schema=[ HashKey('blockId'), RangeKey('blockNum', data_type=NUMBER) ], throughput={'read': 30, 'write': 10}, connection=connection ) self.tablev2 = Table.create(self.tableName, schema=[ HashKey('path'), RangeKey('name') ], throughput={'read': 30, 'write': 10}, indexes=[ KeysOnlyIndex("Links", parts=[ HashKey('path'), RangeKey('link') ]) ], connection=connection ) description = connection.describe_table(self.tableName) iter = 0 while description["Table"]["TableStatus"] != "ACTIVE": print "Waiting for %s to create %d..." % (self.tableName, iter) iter += 1 sleep(1) description = connection.describe_table(self.tableName) self.table = self.conn.get_table(self.tableName) self.blockTable = self.conn.get_table(self.tableName + "Blocks")
from boto.provider import Provider from apscheduler.scheduler import Scheduler from apscheduler.jobstores.shelve_store import ShelveJobStore from shove import Shove import logging SHOVE_BUCKET = 'my-bucket' class ShoveJobStore(ShelveJobStore): def __init__(self, path): self.jobs = [] self.path = path self.store = Shove(path, optimize=False) class S3JobStore(ShoveJobStore): def __init__(self, access_key, secret_key, bucket, prefix='job_'): self.prefix = prefix if prefix[-1] == '/' else (prefix + '/') path = 's3://{}:{}@{}'.format(access_key, secret_key, bucket) super(S3JobStore, self).__init__(path) logging.basicConfig() PROVIDER = Provider('aws') JOB_STORE = S3JobStore(PROVIDER.get_access_key(), PROVIDER.get_secret_key(), SHOVE_BUCKET) SCHEDULER = Scheduler(misfire_grace_time=1000) SCHEDULER.add_jobstore(JOB_STORE, 's3')
def __init__(self, host, **kwargs): provider = Provider('aws', access_key=kwargs.get('access_key'), secret_key=kwargs.get('secret_key')) super(BotoConnection, self).__init__(provider=provider, host=host)
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws', security_token=None, suppress_consec_slashes=True, validate_certs=True, profile_name=None): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :keyword str security_token: The security token associated with temporary credentials issued by STS. Optional unless using temporary credentials. If none is specified, the environment variable ``AWS_SECURITY_TOKEN`` is used if defined. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{http_client.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect :type suppress_consec_slashes: bool :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type validate_certs: bool :param validate_certs: Controls whether SSL certificates will be validated or not. Defaults to True. :type profile_name: str :param profile_name: Override usual Credentials section in config file to use a named set of keys instead. """ self.suppress_consec_slashes = suppress_consec_slashes self.num_retries = 6 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. # The default is now to validate certificates. This can be # overridden in the boto config file are by passing an # explicit validate_certs parameter to the class constructor. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', validate_certs) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") certs_file = config.get_value( 'Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) if certs_file == 'system': certs_file = None self.ca_certificates_file = certs_file if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from http_client that we want to catch and retry self.http_exceptions = (http_client.HTTPException, socket.error, socket.gaierror, http_client.BadStatusLine) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR,) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path # if the value passed in for debug if not isinstance(debug, six.integer_types): debug = 0 self.debug = config.getint('Boto', 'debug', debug) self.host_header = None # Timeout used to tell http_client how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): # If timeout isn't defined in boto config file, use 70 second # default as recommended by # http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html self.http_connection_kwargs['timeout'] = config.getint( 'Boto', 'http_socket_timeout', 70) if isinstance(provider, Provider): # Allow overriding Provider self.provider = provider else: self._provider_type = provider self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token, profile_name) # Allow config file to override default host, port, and host header. if self.provider.host: self.host = self.provider.host if self.provider.port: self.port = self.provider.port if self.provider.host_header: self.host_header = self.provider.host_header self._pool = ConnectionPool() self._connection = (self.host, self.port, self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability()) if getattr(self, 'AuthServiceName', None) is not None: self.auth_service_name = self.AuthServiceName self.request_hook = None
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR,) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host # cache up to 20 connections per host, up to 20 hosts self._pool = ConnectionPool(20, 20) self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability())
def _HasS3Credentials(): provider = Provider('aws') if not provider.access_key or not provider.secret_key: return False return True
def get_access_key(): from boto.provider import Provider provider = Provider('aws') return None if provider._credential_expiry_time is None else provider.get_access_key( )
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure # Whether or not to validate server certificates. At some point in the # future, the default should be flipped to true. self.https_validate_certificates = config.getbool( 'Boto', 'https_validate_certificates', False) if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION: raise BotoClientError( "SSL server certificate validation is enabled in boto " "configuration, but Python dependencies required to " "support this feature are not available. Certificate " "validation is only supported when running under Python " "2.6 or later.") self.ca_certificates_file = config.get_value('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define subclasses of the above that are not retryable. self.http_unretryable_exceptions = [] if HAVE_HTTPS_CONNECTION: self.http_unretryable_exceptions.append(ssl.SSLError) self.http_unretryable_exceptions.append( https_connection.InvalidCertificateException) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR, ) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] # Timeout used to tell httplib how long to wait for socket timeouts. # Default is to leave timeout unchanged, which will in turn result in # the socket's default global timeout being used. To specify a # timeout, set http_socket_timeout in Boto config. Regardless, # timeouts will only be applied if Python is 2.6 or greater. self.http_connection_kwargs = {} if (sys.version_info[0], sys.version_info[1]) >= (2, 6): if config.has_option('Boto', 'http_socket_timeout'): timeout = config.getint('Boto', 'http_socket_timeout') self.http_connection_kwargs['timeout'] = timeout self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host # cache up to 20 connections per host, up to 20 hosts self._pool = ConnectionPool(20, 20) self._connection = (self.server_name(), self.is_secure) self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability())
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws'): """ :type host: string :param host: The host to make the connection to :type aws_access_key_id: string :param aws_access_key_id: AWS Access Key ID (provided by Amazon) :type aws_secret_access_key: string :param aws_secret_access_key: Secret Access Key (provided by Amazon) :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{httplib.HTTPSConnection}. :type proxy: :param proxy: :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: string :param proxy_user: The username to connect with on the proxy :type proxy_pass: string :param proxy_pass: The password to use when connection over a proxy. :type port: integer :param port: The port to use to connect """ self.num_retries = 5 # Override passed-in is_secure setting if value was defined in config. if config.has_option('Boto', 'is_secure'): is_secure = config.getboolean('Boto', 'is_secure') self.is_secure = is_secure self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) # define exceptions from httplib that we want to catch and retry self.http_exceptions = (httplib.HTTPException, socket.error, socket.gaierror) # define values in socket exceptions we don't want to catch self.socket_exception_values = (errno.EINTR, ) if https_connection_factory is not None: self.https_connection_factory = https_connection_factory[0] self.http_exceptions += https_connection_factory[1] else: self.https_connection_factory = None if (is_secure): self.protocol = 'https' else: self.protocol = 'http' self.host = host self.path = path if debug: self.debug = debug else: self.debug = config.getint('Boto', 'debug', debug) if port: self.port = port else: self.port = PORTS_BY_SECURITY[is_secure] self.provider = Provider(provider, aws_access_key_id, aws_secret_access_key) # allow config file to override default host if self.provider.host: self.host = self.provider.host if self.secret_key is None: raise BotoClientError('No credentials have been supplied') # initialize an HMAC for signatures, make copies with each request self.hmac = hmac.new(self.secret_key, digestmod=sha) if sha256: self.hmac_256 = hmac.new(self.secret_key, digestmod=sha256) else: self.hmac_256 = None # cache up to 20 connections per host, up to 20 hosts self._pool = ConnectionPool(20, 20) self._connection = (self.server_name(), self.is_secure) self._last_rs = None
def _get_session_token(self): self.provider = Provider(self._provider_type) self._auth_handler.update_provider(self.provider)
def _update_provider(self): self.provider = Provider('aws', self.creds.access_key, self.creds.secret_key, self.creds.session_token) self._auth_handler.update_provider(self.provider)