def greengrass_hello_world_run(): # Create the green grass client so that we can send messages to IoT console client = greengrasssdk.client('iot-data') iot_topic = '$aws/things/{}/infer'.format(os.environ['AWS_IOT_THING_NAME']) # Stream configuration, name and retention # Note that the name will appear as deeplens-myStream stream_name = 'myStream' retention = 2 #hours # Amount of time to stream wait_time = 60 * 60 * 5 #seconds # Use the boto session API to grab credentials session = Session() creds = session.get_credentials() # Create producer and stream. producer = dkv.createProducer(creds.access_key, creds.secret_key, creds.token, "us-east-1") client.publish(topic=iot_topic, payload="Producer created") kvs_stream = producer.createStream(stream_name, retention) client.publish(topic=iot_topic, payload="Stream {} created".format(stream_name)) # Start putting data into the KVS stream kvs_stream.start() client.publish(topic=iot_topic, payload="Stream started") time.sleep(wait_time) # Stop putting data into the KVS stream kvs_stream.stop() client.publish(topic=iot_topic, payload="Stream stopped")
def test_credential_process_returns_error(self): config = ( '[profile processcreds]\n' 'credential_process = %s --raise-error\n' ) config = config % self.credential_process with temporary_file('w') as f: f.write(config) f.flush() self.environ['AWS_CONFIG_FILE'] = f.name session = Session(profile='processcreds') # This regex validates that there is no substring: b' # The reason why we want to validate that is that we want to # make sure that stderr is actually decoded so that in # exceptional cases the error is properly formatted. # As for how the regex works: # `(?!b').` is a negative lookahead, meaning that it will only # match if it is not followed by the pattern `b'`. Since it is # followed by a `.` it will match any character not followed by # that pattern. `((?!hede).)*` does that zero or more times. The # final pattern adds `^` and `$` to anchor the beginning and end # of the string so we can know the whole string is consumed. # Finally `(?s)` at the beginning makes dots match newlines so # we can handle a multi-line string. reg = r"(?s)^((?!b').)*$" with self.assertRaisesRegexp(CredentialRetrievalError, reg): session.get_credentials()
def call_lambda(function, data): s = Session() clientLambda = s.create_client("lambda", config=Config(retries={'max_attempts': 0})) clientLambda.invoke(FunctionName=function, InvocationType="Event", Payload=json.dumps(data))
def setUp(self): self.env_original = os.environ.copy() self.environ_copy = os.environ.copy() super(TestAssumeRoleCredentials, self).setUp() os.environ = self.environ_copy self.parent_session = Session() self.iam = self.parent_session.create_client('iam') self.sts = self.parent_session.create_client('sts') self.tempdir = tempfile.mkdtemp() self.config_file = os.path.join(self.tempdir, 'config') # A role trust policy that allows the current account to call assume # role on itself. account_id = self.sts.get_caller_identity()['Account'] self.role_policy = { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "arn:aws:iam::%s:root" % account_id }, "Action": "sts:AssumeRole" }] }
def test_credential_process_returns_error(self): config = ('[profile processcreds]\n' 'credential_process = %s --raise-error\n') config = config % self.credential_process with temporary_file('w') as f: f.write(config) f.flush() self.environ['AWS_CONFIG_FILE'] = f.name session = Session(profile='processcreds') # This regex validates that there is no substring: b' # The reason why we want to validate that is that we want to # make sure that stderr is actually decoded so that in # exceptional cases the error is properly formatted. # As for how the regex works: # `(?!b').` is a negative lookahead, meaning that it will only # match if it is not followed by the pattern `b'`. Since it is # followed by a `.` it will match any character not followed by # that pattern. `((?!hede).)*` does that zero or more times. The # final pattern adds `^` and `$` to anchor the beginning and end # of the string so we can know the whole string is consumed. # Finally `(?s)` at the beginning makes dots match newlines so # we can handle a multi-line string. reg = r"(?s)^((?!b').)*$" with self.assertRaisesRegex(CredentialRetrievalError, reg): session.get_credentials()
def get_credentials(profile: str = "default", environment: Optional[Environment] = None) -> Environment: """ Use session cache so users don't need to use MFA while there are valid session tokens. This is the behavior of the AWSCLI and what we are trying to emulate. Modified with support for profiles from: https://github.com/boto/botocore/pull/1338/#issuecomment-368472031 """ # By default the cache path is ~/.aws/boto/cache cli_cache = (Path.home() / ".aws" / "cli" / "cache").absolute() # Construct botocore session with cache session = Session(profile=profile) session.get_component("credential_provider").get_provider( "assume-role").cache = credentials.JSONFileCache(cli_cache) # return credentials from session creds = boto3.Session(botocore_session=session, profile_name=profile).get_credentials() return { "AWS_ACCESS_KEY_ID": creds.access_key, "AWS_SECRET_ACCESS_KEY": creds.secret_key, "AWS_SESSION_TOKEN": creds.token, }
def setUp(self): self.env_original = os.environ.copy() self.environ_copy = os.environ.copy() super(TestAssumeRoleCredentials, self).setUp() os.environ = self.environ_copy # The tests rely on manipulating AWS_CONFIG_FILE, # but we also need to make sure we don't accidentally # pick up the ~/.aws/credentials file either. os.environ['AWS_SHARED_CREDENTIALS_FILE'] = str(uuid4()) self.parent_session = Session() self.iam = self.parent_session.create_client('iam') self.sts = self.parent_session.create_client('sts') self.tempdir = tempfile.mkdtemp() self.config_file = os.path.join(self.tempdir, 'config') # A role trust policy that allows the current account to call assume # role on itself. account_id = self.sts.get_caller_identity()['Account'] self.role_policy = { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "AWS": "arn:aws:iam::%s:root" % account_id }, "Action": "sts:AssumeRole" }] }
def load(cls, profile: str) -> Optional["AwsConfig"]: session = Session(profile=profile) try: c = session.get_scoped_config() except ProfileNotFound: raise MissingAwsConfigException(session.profile) def get(key: str, func: Callable = None): val = c.get(key) if val is None or val == "None": return None else: return func(val) if func else val try: return cls( profile=profile, azure_tenant_id=c["awsad-azure_tenant_id"], azure_app_id=c["awsad-azure_app_id"], azure_app_title=c["awsad-azure_app_title"], aws_default_role_arn=get("awsad-aws_default_role_arn"), aws_session_duration=get("awsad-aws_session_duration", int), aws_access_key_id=get("aws_access_key_id"), aws_secret_access_key=get("aws_secret_access_key"), aws_session_token=get("aws_session_token"), aws_expiration_time=get("awsad-aws_expiration_time", datetime.datetime.fromisoformat)) except KeyError: raise MissingAwsConfigException(session.profile)
def test_old_model_continues_to_work(): # This test ensures that botocore can load the service models as they exist # today. There's a directory in tests/functional/models that is a # snapshot of a service model. This test ensures that we can continue # to stub an API call using this model. That way if the models ever # change we have a mechanism to ensure that the existing models continue # to work with botocore. The test should not change (with the exception # of potential changes to the ClientHTTPStubber), and the files in # tests/functional/models should not change! session = Session() loader = session.get_component('data_loader') # We're adding our path to the existing search paths so we don't have to # copy additional data files such as _retry.json to our TEST_MODELS_DIR. # We only care about the service model and endpoints file not changing. # This also prevents us from having to make any changes to this models dir # if we end up adding a new data file that's needed to create clients. # We're adding our TEST_MODELS_DIR as the first element in the list to # ensure we load the endpoints.json file from TEST_MODELS_DIR. For the # service model we have an extra safety net where we can choose a custom # client name. loader.search_paths.insert(0, TEST_MODELS_DIR) # The model dir we copied was renamed to 'custom-lambda' # to ensure we're loading our version of the model and not # the built in one. client = session.create_client( 'custom-acm', region_name='us-west-2', aws_access_key_id='foo', aws_secret_access_key='bar', ) with ClientHTTPStubber(client) as stubber: stubber.add_response(url='https://acm.us-west-2.amazonaws.com/', headers={ 'x-amzn-RequestId': 'abcd', 'Date': 'Fri, 26 Oct 2018 01:46:30 GMT', 'Content-Length': '29', 'Content-Type': 'application/x-amz-json-1.1' }, body=b'{"CertificateSummaryList":[]}') response = client.list_certificates() assert response == { 'CertificateSummaryList': [], 'ResponseMetadata': { 'HTTPHeaders': { 'content-length': '29', 'content-type': 'application/x-amz-json-1.1', 'date': 'Fri, 26 Oct 2018 01:46:30 GMT', 'x-amzn-requestid': 'abcd' }, 'HTTPStatusCode': 200, 'RequestId': 'abcd', 'RetryAttempts': 0 } } # Also verify we can use the paginators as well. assert client.can_paginate('list_certificates') is True assert client.waiter_names == ['certificate_validated']
def create_botocore_session(profile=None, debug=False): # type: (str, bool) -> Session s = Session(profile=profile) _add_chalice_user_agent(s) if debug: s.set_debug_logger('') _inject_large_request_body_filter() return s
def assume_profile_role(role_profile, session_name="", session_duration=0): """Assume role described by role_profile and return the auth response.""" # Get local profile config config = Session(profile=role_profile).get_scoped_config() # Construct assume role request assert "role_arn" in config, f"{role_profile} does not have role_arn." role_arn = config["role_arn"] rq = { "RoleArn": role_arn, "RoleSessionName": session_name or get_default_session_name(), # "DurationSeconds": 28800 # get_max_duration(role_arn) } # Specify duration if one was given if not session_duration: best_max = get_max_duration(role_arn) dt = timedelta(seconds=best_max) log.debug(f"Using duration of {humanize.naturaldelta(dt)} based on cache.") session_duration = best_max if session_duration: rq["DurationSeconds"] = session_duration else: log.debug(f"No session duration specified, letting AWS choose a default.") # Add MFA token if needed if "mfa_serial" in config: rq["SerialNumber"] = config["mfa_serial"] rq["TokenCode"] = questionary.text("Enter MFA code:").ask() # Log request before making it log.debug(f"Auth request:\n{json.dumps(rq, indent=4)}") # If source_profile is given, we should use it instead of the default profile source_profile = config.get("source_profile") log.info(f"Using source profile: {source_profile}") # Get auth token session = boto3.Session(profile_name=source_profile) sts = session.client("sts") response = sts.assume_role(**rq) # Cache session duration if session_duration: cache_max_duration(role_arn, session_duration) # Log auth token resp_str = json.dumps(response, indent=4, default=lambda o: str(o)) log.debug(f"Auth response:\n{resp_str}") # Log expiration date local_exp = response["Credentials"]["Expiration"].astimezone() remaining = humanize.naturaldelta(local_exp - datetime.now(pytz.utc)) log.info(f"The token will expire after {remaining} on {local_exp}") return response
def call_lambda_sync(function, data): s = Session() clientLambda = s.create_client("lambda", config=Config(retries={'max_attempts': 0})) response = clientLambda.invoke(FunctionName=function, InvocationType="RequestResponse", Payload=json.dumps(data)) body = json.loads(response['Payload'].read()) return body
def __init__(self): """Set up default client suppliers for identity silos.""" self._china_supplier = DefaultClientSupplier(botocore_session=Session( profile="china")) self._middle_east_supplier = DefaultClientSupplier( botocore_session=Session(profile="middle-east")) self._hong_kong_supplier = DefaultClientSupplier( botocore_session=Session(profile="hong-kong")) self._default_supplier = DefaultClientSupplier()
def __init__(self) -> None: current_session = Session() region = current_session.get_config_variable('region') creds = current_session.get_credentials() self.signer = SigV4Auth(creds, 'execute-api', region) analysis_api_fqdn = os.environ.get('ANALYSIS_API_FQDN') analysis_api_path = os.environ.get('ANALYSIS_API_PATH') self.url = 'https://' + analysis_api_fqdn + '/' + analysis_api_path
def run(self): print('Running test...') client = Session().create_client('lambda', region_name=self.region) ret = client.invoke(FunctionName=self.name, InvocationType='RequestResponse', Payload=self.testdata.encode()) print('HTTP Status Code: %s' % ret['ResponseMetadata']['HTTPStatusCode']) print(ret['Payload'].read())
def create_session(): boto_session = Session() boto_session.lazy_register_component('data_loader', lambda: create_loader()) if 'REQUESTS_CA_BUNDLE' not in os.environ: ca_bundle_path = extract_file_from_jar( "botocore/vendored/requests/cacert.pem") os.environ['REQUESTS_CA_BUNDLE'] = ca_bundle_path return boto_session
def test_old_model_continues_to_work(): # This test ensures that botocore can load the service models as they exist # today. There's a directory in tests/functional/models that is a # snapshot of a service model. This test ensures that we can continue # to stub an API call using this model. That way if the models ever # change we have a mechanism to ensure that the existing models continue # to work with botocore. The test should not change (with the exception # of potential changes to the ClientHTTPStubber), and the files in # tests/functional/models should not change! session = Session() loader = session.get_component('data_loader') # We're adding our path to the existing search paths so we don't have to # copy additional data files such as _retry.json to our FIXED_MODELS_DIR. # We only care about the service model and endpoints file not changing. # This also prevents us from having to make any changes to this models dir # if we end up adding a new data file that's needed to create clients. # We're adding our FIXED_MODELS_DIR as the first element in the list to # ensure we load the endpoints.json file from FIXED_MODELS_DIR. For the # service model we have an extra safety net where we can choose a custom # client name. loader.search_paths.insert(0, FIXED_MODELS_DIR) # The model dir we copied was renamed to 'custom-lambda' # to ensure we're loading our version of the model and not # the built in one. client = session.create_client( 'custom-acm', region_name='us-west-2', aws_access_key_id='foo', aws_secret_access_key='bar', ) with ClientHTTPStubber(client) as stubber: stubber.add_response( url='https://acm.us-west-2.amazonaws.com/', headers={'x-amzn-RequestId': 'abcd', 'Date': 'Fri, 26 Oct 2018 01:46:30 GMT', 'Content-Length': '29', 'Content-Type': 'application/x-amz-json-1.1'}, body=b'{"CertificateSummaryList":[]}') response = client.list_certificates() assert_equal( response, {'CertificateSummaryList': [], 'ResponseMetadata': { 'HTTPHeaders': { 'content-length': '29', 'content-type': 'application/x-amz-json-1.1', 'date': 'Fri, 26 Oct 2018 01:46:30 GMT', 'x-amzn-requestid': 'abcd'}, 'HTTPStatusCode': 200, 'RequestId': 'abcd', 'RetryAttempts': 0} } ) # Also verify we can use the paginators as well. assert_equal(client.can_paginate('list_certificates'), True) assert_equal(client.waiter_names, ['certificate_validated'])
def __init__(self, collectionId): self.collectionId = collectionId self.faceId = None self.allThreadStarted = False self._finished = False self.threads = [] self.resultsQ = Queue.Queue() config = Config(connect_timeout=1, read_timeout=2) session = Session() self.rekClient = session.create_client('rekognition', config=config)
def moto_server(): # Start moto server proc = Popen(["moto_server", "s3", "-p", "8082"], stderr=DEVNULL, stdout=DEVNULL) time.sleep(1) with mock_s3(), proc: # Make sure bucket exists session = Session() client = session.create_client("s3", endpoint_url=f"http://{s3_netloc}") yield lambda: s3_reset(client) proc.kill()
def run(): "Entry proint" obj = BotocoreClientHandler(service='ecr') parser = obj._parse_args() print(parser.profile) session = Session(profile=parser.profile) print(dir(session)) print(session.get_config_variable('region')) print(session.get_credentials(), session.profile)
def write_image_to_s3(img): session = Session() s3 = session.create_client('s3') # File name matches file name in s3-image-processing file_name = 'DeepLens/face.jpg' # You can contorl the size and quality of the image encode_param=[int(cv2.IMWRITE_JPEG_QUALITY),90] _, jpg_data = cv2.imencode('.jpg', img, encode_param) response = s3.put_object(ACL='public-read-write', Body=jpg_data.tostring(),Bucket='deeplens-fd-family',Key=file_name) image_url = 'https://s3.amazonaws.com/deeplens-fd-family/'+file_name return image_url
def s3(s3_base): from botocore.session import Session os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") # NB: we use the sync botocore client for setup session = Session() client = session.create_client("s3", endpoint_url=endpoint_uri) client.create_bucket(Bucket=BUCKET_NAME) yield
def create_botocore_session(profile=None, debug=False, connection_timeout=None): # type: (str, bool, int) -> Session s = Session(profile=profile) _add_chalice_user_agent(s) if debug: s.set_debug_logger('') _inject_large_request_body_filter() if connection_timeout is not None: config = BotocoreConfig(connect_timeout=connection_timeout) s.set_default_client_config(config) return s
def get_creds(): global access_key, secret_key, token session = Session() creds = session.get_credentials() if creds is None: logger.info("no credentials found: " + str(creds)) return None access_key = creds.access_key secret_key = creds.secret_key token = creds.token
def _public_apis(): session = Session() # Mimic the scenario that user does not have aws credentials setup session.get_credentials = mock.Mock(return_value=None) for service_name in PUBLIC_API_TESTS: client = session.create_client(service_name, REGIONS[service_name]) for operation_name in PUBLIC_API_TESTS[service_name]: kwargs = PUBLIC_API_TESTS[service_name][operation_name] method = getattr(client, xform_name(operation_name)) yield client, method, kwargs
def test_honors_aws_shared_credentials_file_env_var(self): with temporary_file('w') as f: f.write('[default]\n' 'aws_access_key_id=custom1\n' 'aws_secret_access_key=custom2\n') f.flush() os.environ['AWS_SHARED_CREDENTIALS_FILE'] = f.name s = Session() credentials = s.get_credentials() self.assertEqual(credentials.access_key, 'custom1') self.assertEqual(credentials.secret_key, 'custom2')
def setUp(self): self.session = Session() self.client = self.session.create_client( 'logs', region_name='us-west-2') self.stubber = Stubber(self.client) self.group_name = 'groupName' self.start = '1970-01-01T00:00:01.000000' self.expected_start_as_milli_epoch = 1000 self.filter_pattern = 'mypattern' self.log_timestamp = 1000 self.expected_log_timestamp_as_datetime = datetime( 1970, 1, 1, 0, 0, 1, tzinfo=tz.tzutc())
def setUp(self): self.region = 'us-west-2' self.session = Session() self.session.set_config_variable('region', self.region) self.request_serializer = s3transfer.crt.BotocoreCRTRequestSerializer( self.session) self.bucket = "test_bucket" self.key = "test_key" self.files = FileCreator() self.filename = self.files.create_file('myfile', 'my content') self.expected_path = "/" + self.bucket + "/" + self.key self.expected_host = "s3.%s.amazonaws.com" % (self.region)
def write_image_to_s3(img): print("writing image to s3") session = Session() s3 = session.create_client('s3') file_name = 'DeepLens/image-'+time.strftime("%Y%m%d-%H%M%S")+'.jpg' # You can contorl the size and quality of the image encode_param=[int(cv2.IMWRITE_JPEG_QUALITY),90] _, jpg_data = cv2.imencode('.jpg', img, encode_param) response = s3.put_object(ACL='public-read', Body=jpg_data.tostring(),Bucket='rekog-face',Key=file_name) image_url = 'https://s3.amazonaws.com/<BUCKET_NAME>/'+file_name return image_url
def test_public_apis_will_not_be_signed(): session = Session() # Mimic the scenario that user does not have aws credentials setup session.get_credentials = mock.Mock(return_value=None) for service_name in PUBLIC_API_TESTS: client = session.create_client(service_name, REGIONS[service_name]) for operation_name in PUBLIC_API_TESTS[service_name]: kwargs = PUBLIC_API_TESTS[service_name][operation_name] method = getattr(client, xform_name(operation_name)) yield _test_public_apis_will_not_be_signed, client, method, kwargs
def test_tagged_union_member_name_does_not_coincide_with_unknown_key(self): # This test ensures that operation models do not use SDK_UNKNOWN_MEMBER # as a member name. Thereby reserving SDK_UNKNOWN_MEMBER for the parser to # set as a key on the reponse object. This is necessary when the client # encounters a member that it is unaware of or not modeled. session = Session() for service_name in session.get_available_services(): service_model = session.get_service_model(service_name) for shape_name in service_model.shape_names: shape = service_model.shape_for(shape_name) if hasattr(shape, 'is_tagged_union') and shape.is_tagged_union: self.assertNotIn('SDK_UNKNOWN_MEMBER', shape.members)
async def build_region_list(self, service: str, chosen_regions=None, partition_name='aws'): service = 'ec2containerservice' if service == 'ecs' else service available_services = await run_concurrently(lambda: Session().get_available_services()) if service not in available_services: raise Exception('Service ' + service + ' is not available.') regions = await run_concurrently(lambda: Session().get_available_regions(service, partition_name)) if chosen_regions: return list((Counter(regions) & Counter(chosen_regions)).elements()) else: return regions
def write_image_to_s3(img): session = Session() s3 = session.create_client('s3') file_name = 'DeepLens/image-' + time.strftime("%Y%m%d-%H%M%S") + '.jpg' # You can contorl the size and quality of the image encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 100] _, jpg_data = cv2.imencode('.jpg', img, encode_param) response = s3.put_object( ACL='public-read', Body=jpg_data.tostring(), Bucket='deeplens-sagemaker-f2ede581-36fb-41ce-b61c-881bb384ee61', Key=file_name) image_url = 'https://s3.amazonaws.com/deeplens-sagemaker-f2ede581-36fb-41ce-b61c-881bb384ee61/' + file_name return image_url
def setUp(self): self.global_args = mock.Mock() self._session = Session() self.sso_client = self._session.create_client( 'sso', region_name='us-west-2', ) self.sso_stub = Stubber(self.sso_client) self.profile = 'a-profile' self.scoped_config = {} self.full_config = { 'profiles': { self.profile: self.scoped_config } } self.mock_session = mock.Mock(spec=Session) self.mock_session.get_scoped_config.return_value = self.scoped_config self.mock_session.full_config = self.full_config self.mock_session.create_client.return_value = self.sso_client self.mock_session.profile = self.profile self.config_path = '/some/path' self.session_config = { 'config_file': self.config_path, } self.mock_session.get_config_variable = self.session_config.get self.mock_session.get_available_regions.return_value = ['us-east-1'] self.token_cache = {} self.writer = mock.Mock(spec=ConfigFileWriter) self.prompter = mock.Mock(spec=PTKPrompt) self.selector = mock.Mock(spec=select_menu) self.configure_sso = ConfigureSSOCommand( self.mock_session, prompter=self.prompter, selector=self.selector, config_writer=self.writer, sso_token_cache=self.token_cache, ) self.region = 'us-west-2' self.output = 'json' self.sso_region = 'us-east-1' self.start_url = 'https://d-92671207e4.awsapps.com/start' self.account_id = '0123456789' self.role_name = 'roleA' self.cached_token_key = '13f9d35043871d073ab260e020f0ffde092cb14b' self.expires_at = datetime.now(tzlocal()) + timedelta(hours=24) self.access_token = { 'accessToken': 'access.token.string', 'expiresAt': self.expires_at, } self.token_cache[self.cached_token_key] = self.access_token
def setUp(self): self.env_original = os.environ.copy() self.environ_copy = os.environ.copy() super(TestAssumeRoleCredentials, self).setUp() os.environ = self.environ_copy # The tests rely on manipulating AWS_CONFIG_FILE, # but we also need to make sure we don't accidentally # pick up the ~/.aws/credentials file either. os.environ['AWS_SHARED_CREDENTIALS_FILE'] = str(uuid4()) self.parent_session = Session() self.iam = self.parent_session.create_client('iam') self.sts = self.parent_session.create_client('sts') self.tempdir = tempfile.mkdtemp() self.config_file = os.path.join(self.tempdir, 'config') # A role trust policy that allows the current account to call assume # role on itself. account_id = self.sts.get_caller_identity()['Account'] self.role_policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "arn:aws:iam::%s:root" % account_id }, "Action": "sts:AssumeRole" } ] }
def setUp(self): self.environ = { 'AWS_DATA_PATH': os.environ['AWS_DATA_PATH'], 'AWS_DEFAULT_REGION': 'us-east-1', 'AWS_ACCESS_KEY_ID': 'access_key', 'AWS_SECRET_ACCESS_KEY': 'secret_key', 'AWS_CONFIG_FILE': '', } self.environ_patch = mock.patch('os.environ', self.environ) self.environ_patch.start() emitter = HierarchicalEmitter() session = Session(EnvironmentVariables, emitter) session.register_component('data_loader', _LOADER) load_plugins({}, event_hooks=emitter) driver = CLIDriver(session=session) self.session = session self.driver = driver
def create_session(self, profile=None): session = Session(profile=profile) # We have to set bogus credentials here or otherwise we'll trigger # an early credential chain resolution. sts = session.create_client( 'sts', aws_access_key_id='spam', aws_secret_access_key='eggs', ) stubber = Stubber(sts) stubber.activate() assume_role_provider = AssumeRoleProvider( load_config=lambda: session.full_config, client_creator=lambda *args, **kwargs: sts, cache={}, profile_name=profile, credential_sourcer=CanonicalNameCredentialSourcer([ self.env_provider, self.container_provider, self.metadata_provider ]) ) component_name = 'credential_provider' resolver = session.get_component(component_name) available_methods = [p.METHOD for p in resolver.providers] replacements = { 'env': self.env_provider, 'iam-role': self.metadata_provider, 'container-role': self.container_provider, 'assume-role': assume_role_provider } for name, provider in replacements.items(): try: index = available_methods.index(name) except ValueError: # The provider isn't in the session continue resolver.providers[index] = provider session.register_component( 'credential_provider', resolver ) return session, stubber
class TestResetStreamOnRetry(unittest.TestCase): def setUp(self): super(TestResetStreamOnRetry, self).setUp() self.total_calls = 0 self.auth = Mock() self.session = Session(include_builtin_handlers=False) self.service = Mock() self.service.endpoint_prefix = 's3' self.service.session = self.session self.endpoint = RestEndpoint( self.service, 'us-east-1', 'https://s3.amazonaws.com/', auth=self.auth) self.http_session = Mock() self.endpoint.http_session = self.http_session self.get_response_patch = patch('botocore.response.get_response') self.get_response = self.get_response_patch.start() self.retried_on_exception = None def tearDown(self): self.get_response_patch.stop() def max_attempts_retry_handler(self, attempts, **kwargs): # Simulate a max requests of 3. self.total_calls += 1 if attempts == 3: return None else: # Returning anything non-None will trigger a retry, # but 0 here is so that time.sleep(0) happens. return 0 def test_reset_stream_on_retry(self): # It doesn't really matter what the operation is, we will # check in general if we're self.session.register('needs-retry.s3.PutObject', self.max_attempts_retry_handler) op = Mock() payload = Payload() payload.literal_value = RecordStreamResets('foobar') op.name = 'PutObject' op.http = {'uri': '', 'method': 'POST'} self.endpoint.make_request(op, {'headers': {}, 'payload': payload}) self.assertEqual(self.total_calls, 3) self.assertEqual(payload.literal_value.total_resets, 2)
def factory(context, request): """ :type context: object :type request: pyramid.request.Request :rtype: boto3.Session """ session = None if cache is not None: session = getattr(cache, session_name, None) if session is None: core_session = None if core_settings: core_session = CoreSession() for k, v in core_settings.items(): core_session.set_config_variable(k, v) session = Session(botocore_session=core_session, **settings) if cache is not None: setattr(cache, session_name, session) return session
def create_botocore_session(profile=None, debug=False, connection_timeout=None, read_timeout=None, max_retries=None): # type: (OptStr, bool, OptInt, OptInt, OptInt) -> Session s = Session(profile=profile) _add_chalice_user_agent(s) if debug: _inject_large_request_body_filter() config_args = {} # type: Dict[str, Any] if connection_timeout is not None: config_args['connect_timeout'] = connection_timeout if read_timeout is not None: config_args['read_timeout'] = read_timeout if max_retries is not None: config_args['retries'] = {'max_attempts': max_retries} if config_args: config = BotocoreConfig(**config_args) s.set_default_client_config(config) return s
def test_assume_role_uses_correct_region(self): config = ( '[profile A]\n' 'role_arn = arn:aws:iam::123456789:role/RoleA\n' 'source_profile = B\n\n' '[profile B]\n' 'aws_access_key_id = abc123\n' 'aws_secret_access_key = def456\n' ) self.write_config(config) session = Session(profile='A') # Verify that when we configure the session with a specific region # that we use that region when creating the sts client. session.set_config_variable('region', 'cn-north-1') create_client, expected_creds = self.create_stubbed_sts_client(session) session.create_client = create_client resolver = create_credential_resolver(session) provider = resolver.get_provider('assume-role') creds = provider.load() self.assert_creds_equal(creds, expected_creds) self.assertEqual(self.actual_client_region, 'cn-north-1')
def setUp(self): super(TestRetryInterface, self).setUp() self.total_calls = 0 self.auth = Mock() self.session = Session(include_builtin_handlers=False) self.service = Mock() self.service.endpoint_prefix = "ec2" self.service.session = self.session self.endpoint = QueryEndpoint(self.service, "us-west-2", "https://ec2.us-west-2.amazonaws.com/", auth=self.auth) self.http_session = Mock() self.endpoint.http_session = self.http_session self.get_response_patch = patch("botocore.response.get_response") self.get_response = self.get_response_patch.start() self.retried_on_exception = None
def setUp(self): super(TestResetStreamOnRetry, self).setUp() self.total_calls = 0 self.auth = Mock() self.session = Session(include_builtin_handlers=False) self.service = Mock() self.service.endpoint_prefix = 's3' self.service.session = self.session self.endpoint = RestEndpoint( self.service, 'us-east-1', 'https://s3.amazonaws.com/', auth=self.auth) self.http_session = Mock() self.endpoint.http_session = self.http_session self.get_response_patch = patch('botocore.response.get_response') self.get_response = self.get_response_patch.start() self.retried_on_exception = None
def setUp(self): super(TestAssumeRoleCredentials, self).setUp() self.environ = os.environ.copy() self.parent_session = Session() self.iam = self.parent_session.create_client('iam') self.sts = self.parent_session.create_client('sts') self.tempdir = tempfile.mkdtemp() self.config_file = os.path.join(self.tempdir, 'config') # A role trust policy that allows the current account to call assume # role on itself. account_id = self.sts.get_caller_identity()['Account'] self.role_policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "arn:aws:iam::%s:root" % account_id }, "Action": "sts:AssumeRole" } ] }
def getEnvironment(self, profile=None): """Return environment variables that should be set for the profile.""" eventHooks = HierarchicalEmitter() session = Session(event_hooks=eventHooks) if profile: session.set_config_variable('profile', profile) eventHooks.register('session-initialized', inject_assume_role_provider_cache, unique_id='inject_assume_role_cred_provider_cache') session.emit('session-initialized', session=session) creds = session.get_credentials() env = {} def set(key, value): if value: env[key] = value set('AWS_ACCESS_KEY_ID', creds.access_key) set('AWS_SECRET_ACCESS_KEY', creds.secret_key) # AWS_SESSION_TOKEN is the ostensibly the standard: # http://blogs.aws.amazon.com/security/post/Tx3D6U6WSFGOK2H/A-New-and-Standardized-Way-to-Manage-Credentials-in-the-AWS-SDKs # http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-environment set('AWS_SESSION_TOKEN', creds.token) # ...but boto expects AWS_SECURITY_TOKEN. Set both for compatibility. # https://github.com/boto/boto/blob/b016c07d834df5bce75141c4b9d2f3d30352e1b8/boto/connection.py#L438 set('AWS_SECURITY_TOKEN', creds.token) set('AWS_DEFAULT_REGION', session.get_config_variable('region')) return env
def _get_session(): session = Session() session.set_credentials('foo', 'bar') session.set_config_variable('region', 'us-west-2') session.config_filename = 'no-exist-foo' return session
class TestAssumeRoleCredentials(unittest.TestCase): def setUp(self): super(TestAssumeRoleCredentials, self).setUp() self.environ = os.environ.copy() self.parent_session = Session() self.iam = self.parent_session.create_client('iam') self.sts = self.parent_session.create_client('sts') self.tempdir = tempfile.mkdtemp() self.config_file = os.path.join(self.tempdir, 'config') # A role trust policy that allows the current account to call assume # role on itself. account_id = self.sts.get_caller_identity()['Account'] self.role_policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "AWS": "arn:aws:iam::%s:root" % account_id }, "Action": "sts:AssumeRole" } ] } def tearDown(self): super(TestAssumeRoleCredentials, self).tearDown() shutil.rmtree(self.tempdir) def random_name(self): return 'clitest-' + random_chars(10) def create_role(self, policy_document, policy_arn=None): name = self.random_name() response = self.iam.create_role( RoleName=name, AssumeRolePolicyDocument=json.dumps(policy_document) ) self.addCleanup(self.iam.delete_role, RoleName=name) if policy_arn: self.iam.attach_role_policy(RoleName=name, PolicyArn=policy_arn) self.addCleanup( self.iam.detach_role_policy, RoleName=name, PolicyArn=policy_arn ) return response['Role'] def create_user(self, policy_arns): name = self.random_name() user = self.iam.create_user(UserName=name)['User'] self.addCleanup(self.iam.delete_user, UserName=name) for arn in policy_arns: self.iam.attach_user_policy( UserName=name, PolicyArn=arn ) self.addCleanup( self.iam.detach_user_policy, UserName=name, PolicyArn=arn ) return user def create_creds(self, user_name): creds = self.iam.create_access_key(UserName=user_name)['AccessKey'] self.addCleanup( self.iam.delete_access_key, UserName=user_name, AccessKeyId=creds['AccessKeyId'] ) return creds def wait_for_assume_role(self, role_arn, access_key, secret_key, token=None, attempts=30, delay=10): # "Why not use the policy simulator?" you might ask. The answer is # that the policy simulator will return success far before you can # actually make the calls. client = self.parent_session.create_client( 'sts', aws_access_key_id=access_key, aws_secret_access_key=secret_key, aws_session_token=token ) attempts_remaining = attempts role_session_name = random_chars(10) while attempts_remaining > 0: attempts_remaining -= 1 try: result = client.assume_role( RoleArn=role_arn, RoleSessionName=role_session_name) return result['Credentials'] except ClientError as e: code = e.response.get('Error', {}).get('Code') if code in ["InvalidClientTokenId", "AccessDenied"]: time.sleep(delay) else: raise raise Exception("Unable to assume role %s" % role_arn) def create_assume_policy(self, role_arn): policy_document = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Resource": role_arn, "Action": "sts:AssumeRole" } ] } name = self.random_name() response = self.iam.create_policy( PolicyName=name, PolicyDocument=json.dumps(policy_document) ) self.addCleanup( self.iam.delete_policy, PolicyArn=response['Policy']['Arn'] ) return response['Policy']['Arn'] def assert_s3_read_only_profile(self, profile_name): # Calls to S3 should succeed command = 's3api list-buckets --profile %s' % profile_name result = aws(command, env_vars=self.environ) self.assertEqual(result.rc, 0, result.stderr) # Calls to other services should not command = 'iam list-groups --profile %s' % profile_name result = aws(command, env_vars=self.environ) self.assertNotEqual(result.rc, 0, result.stdout) self.assertIn('AccessDenied', result.stderr) def test_recursive_assume_role(self): # Create the final role, the one that will actually have access to s3 final_role = self.create_role(self.role_policy, S3_READ_POLICY_ARN) # Create the role that can assume the final role middle_policy_arn = self.create_assume_policy(final_role['Arn']) middle_role = self.create_role(self.role_policy, middle_policy_arn) # Create a user that can only assume the middle-man role, and then get # static credentials for it. user_policy_arn = self.create_assume_policy(middle_role['Arn']) user = self.create_user([user_policy_arn]) user_creds = self.create_creds(user['UserName']) # Setup the config file with the profiles we'll be using. For # convenience static credentials are placed here instead of putting # them in the credentials file. config = ( '[default]\n' 'aws_access_key_id = %s\n' 'aws_secret_access_key = %s\n' '[profile middle]\n' 'source_profile = default\n' 'role_arn = %s\n' '[profile final]\n' 'source_profile = middle\n' 'role_arn = %s\n' ) config = config % ( user_creds['AccessKeyId'], user_creds['SecretAccessKey'], middle_role['Arn'], final_role['Arn'] ) with open(self.config_file, 'w') as f: f.write(config) # Wait for IAM permissions to propagate middle_creds = self.wait_for_assume_role( role_arn=middle_role['Arn'], access_key=user_creds['AccessKeyId'], secret_key=user_creds['SecretAccessKey'], ) self.wait_for_assume_role( role_arn=final_role['Arn'], access_key=middle_creds['AccessKeyId'], secret_key=middle_creds['SecretAccessKey'], token=middle_creds['SessionToken'], ) # Configure our credentials file to be THE credentials file self.environ['AWS_CONFIG_FILE'] = self.config_file self.assert_s3_read_only_profile(profile_name='final') def test_assume_role_with_credential_source(self): # Create a role with read access to S3 role = self.create_role(self.role_policy, S3_READ_POLICY_ARN) # Create a user that can assume the role and get static credentials # for it. user_policy_arn = self.create_assume_policy(role['Arn']) user = self.create_user([user_policy_arn]) user_creds = self.create_creds(user['UserName']) # Setup the config file with the profile we'll be using. config = ( '[profile assume]\n' 'role_arn = %s\n' 'credential_source = Environment\n' ) config = config % role['Arn'] with open(self.config_file, 'w') as f: f.write(config) # Wait for IAM permissions to propagate self.wait_for_assume_role( role_arn=role['Arn'], access_key=user_creds['AccessKeyId'], secret_key=user_creds['SecretAccessKey'], ) # Setup the environment so that our new config file is THE config # file and add the expected credentials since we're using the # environment as our credential source. self.environ['AWS_CONFIG_FILE'] = self.config_file self.environ['AWS_SECRET_ACCESS_KEY'] = user_creds['SecretAccessKey'] self.environ['AWS_ACCESS_KEY_ID'] = user_creds['AccessKeyId'] self.assert_s3_read_only_profile(profile_name='assume')
def profile_name(self, new_name): """set profile_name and refresh_boto_connections""" self._profile_name = new_name if new_name is not None: self.config = Session(profile=new_name).get_scoped_config() self.setup_session_and_refresh_connections()
class BotoConnections(object): """Central Management of boto3 client and resource connection objects.""" def __init__(self, region_name=None, profile_name=None): """ Optionally pass region_name and profile_name. Setup boto3 session. Attach boto3 client and resource connection objects. """ # defaults. self.config = {} self._region_name = self._profile_name = None # trigger region_name.setter self.region_name = region_name # trigger profile_name.setter self.profile_name = profile_name @property def profile_name(self): return self._profile_name @profile_name.setter def profile_name(self, new_name): """set profile_name and refresh_boto_connections""" self._profile_name = new_name if new_name is not None: self.config = Session(profile=new_name).get_scoped_config() self.setup_session_and_refresh_connections() @property def region_name(self): if self._region_name is None: return self.config.get('region', None) return self._region_name @region_name.setter def region_name(self, new_name): """set region_name and refresh_boto_connections""" self._region_name = new_name self.setup_session_and_refresh_connections() def setup_session_and_refresh_connections(self): if self.profile_name and self.region_name: boto3.setup_default_session( profile_name = self.profile_name, region_name = self.region_name, ) elif self.profile_name: boto3.setup_default_session(profile_name = self.profile_name) elif self.region_name: boto3.setup_default_session(region_name = self.region_name) else: return None self.refresh_boto_connections() def refresh_boto_connections(self): """Attach related Boto3 clients and resources.""" self.iam = boto3.resource('iam') self.ec2 = boto3.resource('ec2') self.ec2_client = boto3.client('ec2') self.rds = boto3.client('rds') self.elasticache = boto3.client('elasticache') self.elb = boto3.client('elb') self.autoscaling = boto3.client('autoscaling') self.route53 = boto3.client('route53') @property def azones(self): """Return a list of available AZ names for active AWS profile/region.""" az_filter = make_filter('state', 'available') azs = self.ec2_client.describe_availability_zones(Filters=az_filter) return map(lambda az : az['ZoneName'], azs['AvailabilityZones'])
class TestRetryInterface(BaseSessionTest): def setUp(self): super(TestRetryInterface, self).setUp() self.total_calls = 0 self.auth = Mock() self.session = Session(include_builtin_handlers=False) self.service = Mock() self.service.endpoint_prefix = 'ec2' self.service.session = self.session self.endpoint = QueryEndpoint( self.service, 'us-west-2', 'https://ec2.us-west-2.amazonaws.com/', auth=self.auth) self.http_session = Mock() self.endpoint.http_session = self.http_session self.get_response_patch = patch('botocore.response.get_response') self.get_response = self.get_response_patch.start() self.retried_on_exception = None def tearDown(self): self.get_response_patch.stop() def max_attempts_retry_handler(self, attempts, **kwargs): # Simulate a max requests of 3. self.total_calls += 1 if attempts == 3: return None else: # Returning anything non-None will trigger a retry, # but 0 here is so that time.sleep(0) happens. return 0 def connection_error_handler(self, attempts, caught_exception, **kwargs): self.total_calls += 1 if attempts == 3: return None elif isinstance(caught_exception, ConnectionError): # Returning anything non-None will trigger a retry, # but 0 here is so that time.sleep(0) happens. return 0 else: return None def test_retry_events_are_emitted(self): emitted_events = [] self.session.register('needs-retry.ec2.DescribeInstances', lambda **kwargs: emitted_events.append(kwargs)) op = Mock() op.name = 'DescribeInstances' self.endpoint.make_request(op, {}) self.assertEqual(len(emitted_events), 1) self.assertEqual(emitted_events[0]['event_name'], 'needs-retry.ec2.DescribeInstances') def test_retry_events_can_alter_behavior(self): self.session.register('needs-retry.ec2.DescribeInstances', self.max_attempts_retry_handler) op = Mock() op.name = 'DescribeInstances' self.endpoint.make_request(op, {}) self.assertEqual(self.total_calls, 3) def test_retry_on_socket_errors(self): self.session.register('needs-retry.ec2.DescribeInstances', self.connection_error_handler) op = Mock() op.name = 'DescribeInstances' self.http_session.send.side_effect = ConnectionError() self.endpoint.make_request(op, {}) self.assertEqual(self.total_calls, 3)