def _elasticsearch_connect(): """ Connect to configured Elasticsearch domain. :return: An Elasticsearch connection object. """ es_url = config("ELASTICSEARCH_URL", default="localhost") es_port = config("ELASTICSEARCH_PORT", default=9200, cast=int) es_aws_region = config("ELASTICSEARCH_AWS_REGION", default="us-east-1") auth = AWSRequestsAuth( aws_access_key=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, aws_host=es_url, aws_region=es_aws_region, aws_service="es", ) auth.encode = lambda x: bytes(x.encode("utf-8")) _es = Elasticsearch( host=es_url, port=es_port, connection_class=RequestsHttpConnection, timeout=10, max_retries=1, retry_on_timeout=True, http_auth=auth, wait_for_status="yellow", ) _es.info() return _es
def __init__(self, es_host, region='us-east-1', is_lambda=True, index='iocs'): AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] if is_lambda: aws_token = os.environ["AWS_SESSION_TOKEN"] self.auth = AWSRequestsAuth( aws_access_key=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_token=aws_token, aws_host=es_host, aws_region=region, aws_service='es') else: self.auth = AWSRequestsAuth( aws_access_key=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_host=es_host, aws_region=region, aws_service='es') self.session = requests.session() self.session.auth = self.auth self.es_host = es_host
def __call__(self, r): try: # Host used in signature *MUST* always match with Host HTTP header. host = r.headers.get('Host') if not host: _, _, host, _, _, _, _ = parse_url(r.url) r.headers['Host'] = host if self.domain is not None: aws_params = self._parse_url(self.domain) else: aws_params = self._parse_url(host) except ValueError: aws_params = {"region": "us-west-2", "service": "svs"} except Exception as error: aws_params = {"region": "us-west-2", "service": "svs"} aws_request = AWSRequestsAuth( aws_access_key=self.aws_access_key, aws_secret_access_key=self.aws_secret_access_key, aws_host=host, aws_region=aws_params['region'], aws_service=aws_params['service'], aws_token=self.aws_token) return aws_request.__call__(r)
def _elasticsearch_connect() -> Elasticsearch: """ Connect to an Elasticsearch indices at the configured domain. This method also handles AWS authentication using the AWS access key ID and the secret access key. :return: an Elasticsearch client """ log.info( f"Connecting to {ELASTICSEARCH_URL}:{ELASTICSEARCH_PORT} with AWS auth" ) auth = AWSRequestsAuth( aws_access_key=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_host=ELASTICSEARCH_URL, aws_region=AWS_REGION, aws_service="es", ) auth.encode = lambda x: bytes(x.encode("utf-8")) es = Elasticsearch( host=ELASTICSEARCH_URL, port=ELASTICSEARCH_PORT, connection_class=RequestsHttpConnection, http_auth=auth, timeout=TWELVE_HOURS_SEC, ) es.info() return es
def _elasticsearch_connect(timeout=300): """ Connect to configured Elasticsearch domain. :param timeout: How long to wait before ANY request to Elasticsearch times out. Because we use parallel bulk uploads (which sometimes wait long periods of time before beginning execution), a value of at least 30 seconds is recommended. :return: An Elasticsearch connection object. """ log.info('Connecting to %s %s with AWS auth', ELASTICSEARCH_URL, ELASTICSEARCH_PORT) auth = AWSRequestsAuth(aws_access_key=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_host=ELASTICSEARCH_URL, aws_region=AWS_REGION, aws_service='es') auth.encode = lambda x: bytes(x.encode('utf-8')) es = Elasticsearch(host=ELASTICSEARCH_URL, port=ELASTICSEARCH_PORT, connection_class=RequestsHttpConnection, timeout=timeout, max_retries=10, retry_on_timeout=True, http_auth=auth, wait_for_status='yellow') es.info() return es
def __call__(self, r): try: # Host used in signature *MUST* always match with Host HTTP header. host = r.headers.get('Host') if not host: _, _, host, _, _, _, _ = parse_url(r.url) r.headers['Host'] = host if self.domain is not None: aws_params = self._parse_url(self.domain) else: aws_params = self._parse_url(host) except ValueError: print("ERROR: Could not parse neccessary information from URL.") raise except Exception as error: print("Error parsing URL: %s" % error) raise if isinstance(r.body, io.RawIOBase): r.body = r.body.readall() elif isinstance(r.body, (io.BufferedIOBase, io.TextIOBase)): r.body = r.body.read() aws_request = AWSRequestsAuth( aws_access_key=self.aws_access_key, aws_secret_access_key=self.aws_secret_access_key, aws_host=host, aws_region=aws_params['region'], aws_service=aws_params['service'], aws_token=self.aws_token) return aws_request.__call__(r)
def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest: if self.access_key is None: fatal_plugin_error(f"ERROR: Missing {AWS_ACCESS_KEY} in environment") if self.secret_key is None: fatal_plugin_error(f"ERROR: Missing {AWS_SECRET_KEY} in environment") try: host = ( r.headers.get("Host").decode("utf-8") if "Host" in r.headers else urlparse(r.url).netloc ) if self.region is None or self.service is None: p = parse( r"(?P<region>[^\.]+)\.(?P<service>.+)\.api\.nifcloud\.com", host ) self.region = p["region"] self.service = p["service"] except Exception as error: fatal_plugin_error(f"ERROR: {error}") raise auth = AWSRequestsAuth( aws_access_key=self.access_key, aws_secret_access_key=self.secret_key, aws_host=host, aws_region=self.region, aws_service=self.service, ) return auth.__call__(r)
def _elasticsearch_connect(): """ Connect to configured Elasticsearch domain. :return: An Elasticsearch connection object. """ auth = AWSRequestsAuth( aws_access_key=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, aws_host=settings.ELASTICSEARCH_URL, aws_region=settings.ELASTICSEARCH_AWS_REGION, aws_service='es' ) auth.encode = lambda x: bytes(x.encode('utf-8')) _es = Elasticsearch( host=settings.ELASTICSEARCH_URL, port=settings.ELASTICSEARCH_PORT, connection_class=RequestsHttpConnection, timeout=10, max_retries=99, retry_on_timeout=True, http_auth=auth, wait_for_status='yellow' ) _es.info() return _es
def test_path_with_querystring(self): """ Assert we generate the 'correct' cannonical query string and path for request that includes a query stirng """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/my_index/?pretty=True' mock_request = mock.Mock() mock_request.url = url self.assertEqual('/my_index/', AWSRequestsAuth.get_caononical_path(mock_request)) self.assertEqual('pretty=True', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_characters_escaped_in_path(self): """ Assert we generate the 'correct' cannonical query string and path a request with characters that need to be escaped """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/+foo.*/_stats' mock_request = mock.Mock() mock_request.url = url self.assertEqual('/%2Bfoo.%2A/_stats', AWSRequestsAuth.get_caononical_path(mock_request)) self.assertEqual('', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_characters_escaped_in_path(self): """ Assert we generate the 'correct' cannonical query string and path a request with characters that need to be escaped """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/+foo.*/_stats' mock_request = mock.Mock() mock_request.url = url self.assertEqual('/%2Bfoo.%2A/_stats', AWSRequestsAuth.get_canonical_path(mock_request)) self.assertEqual( '', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_no_query_params(self): """ Assert we generate the 'correct' cannonical query string and canonical path for a request with no query params Correct is relative here b/c 'correct' simply means what the AWS Elasticsearch service expects """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/' mock_request = mock.Mock() mock_request.url = url self.assertEqual('/', AWSRequestsAuth.get_caononical_path(mock_request)) self.assertEqual('', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_path_with_querystring(self): """ Assert we generate the 'correct' cannonical query string and path for request that includes a query stirng """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/my_index/?pretty=True' mock_request = mock.Mock() mock_request.url = url self.assertEqual('/my_index/', AWSRequestsAuth.get_canonical_path(mock_request)) self.assertEqual( 'pretty=True', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_no_query_params(self): """ Assert we generate the 'correct' cannonical query string and canonical path for a request with no query params Correct is relative here b/c 'correct' simply means what the AWS Elasticsearch service expects """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/' mock_request = mock.Mock() mock_request.url = url self.assertEqual('/', AWSRequestsAuth.get_canonical_path(mock_request)) self.assertEqual( '', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_auth_for_post(self): auth = AWSRequestsAuth( aws_access_key='YOURKEY', aws_secret_access_key='YOURSECRET', aws_host='search-foo.us-east-1.es.amazonaws.com', aws_region='us-east-1', aws_service='es') url = 'http://search-foo.us-east-1.es.amazonaws.com:80/' mock_request = mock.Mock() mock_request.url = url mock_request.method = "POST" mock_request.body = b'foo=bar' mock_request.headers = { 'Content-Type': 'application/x-www-form-urlencoded', } frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5) with mock.patch('datetime.datetime') as mock_datetime: mock_datetime.utcnow.return_value = frozen_datetime auth(mock_request) self.assertEqual( { 'Authorization': 'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, ' 'SignedHeaders=host;x-amz-date, ' 'Signature=a6fd88e5f5c43e005482894001d9b05b43f6710e96be6098bcfcfccdeb8ed812', 'Content-Type': 'application/x-www-form-urlencoded', 'x-amz-date': '20160618T220405Z', }, mock_request.headers)
def test_auth_for_post_with_unicode_body_python3(self): auth = AWSRequestsAuth( aws_access_key='YOURKEY', aws_secret_access_key='YOURSECRET', aws_host='search-foo.us-east-1.es.amazonaws.com', aws_region='us-east-1', aws_service='es') url = 'http://search-foo.us-east-1.es.amazonaws.com:80/' mock_request = mock.Mock() mock_request.url = url mock_request.method = "POST" mock_request.body = 'foo=bar\xc3' mock_request.headers = { 'Content-Type': 'application/x-www-form-urlencoded', } frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5) with mock.patch('datetime.datetime') as mock_datetime: mock_datetime.utcnow.return_value = frozen_datetime auth(mock_request) self.assertEqual( { 'Authorization': 'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, ' 'SignedHeaders=host;x-amz-date, ' 'Signature=0836dae4bce95c1bcdbd3751c84c0c7e589ba7c81331bab92d0e1acb94adcdd9', 'Content-Type': 'application/x-www-form-urlencoded', 'x-amz-date': '20160618T220405Z', }, mock_request.headers)
def login(self, username, password): """ Retrieve the aws credentials used by the apis. The login is staged, p1 we login to gigya, p2 we use p1 credentials in irbt login api, that finally return aws credentials to use in all the subsequent calls (mqtt and api gateway) """ # discover cloud settings self._disc() # login to gigya (get uid, timestamp and signature) gigya_login = self._gigya_login(username, password) # get aws keys through irbt login api try: (self.access_key_id, self.secret_key, self.session_token) = self._irbt_login_api( gigya_login['UIDSignature'], gigya_login['signatureTimestamp'], gigya_login['UID']) except KeyError: logger.error('Authentication failed, wrong login/password') return -1 # use AWSRequestsAuth module to set the aws authentication headers # (Signature Version 4 Signing Process) self.auth = AWSRequestsAuth( aws_access_key=self.access_key_id, aws_secret_access_key=self.secret_key, aws_token=self.session_token, aws_host=self._aws_host, aws_region=self._aws_region, aws_service='execute-api') return True
def handler(event, context): # For this function, we don't care about 'event' and 'context', # but they need to be in the function signature anyway. esinfo = {} esendpoint = 'vpc-shop-v5-f4mgyu2gin24qdpouzn3icyz7m.eu-west-1.es.amazonaws.com' session = boto3.session.Session() credentials = session.get_credentials().get_frozen_credentials() awsauth = AWSRequestsAuth( aws_access_key=credentials.access_key, aws_secret_access_key=credentials.secret_key, aws_token=credentials.token, aws_host=esendpoint, aws_region=session.region_name, aws_service='es' ) es = Elasticsearch( hosts=[{'host': esendpoint, 'port': 443}], http_auth=awsauth, use_ssl=True, verify_certs=True, ca_certs=certifi.where(), connection_class=RequestsHttpConnection ) esinfo = es.info(); print(esinfo)
def __init__(self): self.session = requests.Session() self.session.headers.update({ 'X-Cuenca-Api-Version': API_VERSION, 'User-Agent': f'cuenca-python/{CLIENT_VERSION}', }) # basic auth api_key = os.getenv('CUENCA_API_KEY', '') api_secret = os.getenv('CUENCA_API_SECRET', '') self.basic_auth = (api_key, api_secret) # IAM auth aws_access_key = os.getenv('AWS_ACCESS_KEY_ID', '') aws_secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY', '') aws_region = os.getenv('AWS_DEFAULT_REGION', AWS_DEFAULT_REGION) if aws_access_key and aws_secret_access_key: self.iam_auth = AWSRequestsAuth( aws_access_key=aws_access_key, aws_secret_access_key=aws_secret_access_key, aws_host=self.host, aws_region=aws_region, aws_service=AWS_SERVICE, )
def __init__(self, json_auth): with open(json_auth) as f: auth_info = json.load(f) self.url = auth_info['apiEndpoint'] self.user_name = auth_info['userId'] host = self.url.split('/')[2] self.navi_auth = AWSRequestsAuth( aws_access_key=auth_info['accessKey'], aws_secret_access_key=auth_info['secretKey'], aws_host=host, aws_region=auth_info['region'], aws_service='execute-api') #endpoints self.status_endpoint = "api/v0/status" self.robots_endpoint = "api/v0/users/{1}/robots" self.robot_endpoint = "api/v0/users/{1}/robots/{2}" self.sessions_endpoint = "api/v0/users/{1}/robots/{2}/sessions" self.session_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}" self.nodes_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes" self.node_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}" self.node_labelled_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/labelled/{4}" self.odo_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/odometry" self.big_data_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data" self.big_data_element_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data/{5}" self.big_data_raw_element_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/nodes/{4}/data/{5}/raw" self.variable_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/variables/{4}" self.factors_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/factors" self.factor_endpoint = "$factorsEndpoint/{4}" self.bearing_range_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/factors/bearingrange" self.session_ready_endpoint = "api/v0/users/{1}/robots/{2}/sessions/{3}/ready/{4}" # context self.sessionId = "" self.robotId = "" self.userId = ""
def _get_auth(self) -> AWSRequestsAuth: """Generate an AWSRequestsAuth object using a boto session for the current/local account. Returns: AWSRequestsAuth object """ if datetime.now() >= self._auth_expiration: session = boto3.Session() credentials = session.get_credentials() region = ( session.region_name if self._neptune_endpoint.region is None else self._neptune_endpoint.region ) auth = AWSRequestsAuth( aws_access_key=credentials.access_key, aws_secret_access_key=credentials.secret_key, aws_token=credentials.token, aws_host=f"{self._neptune_endpoint.host}:{self._neptune_endpoint.port}", aws_region=region, aws_service="neptune-db", ) self._auth = auth self._auth_expiration = datetime.now() + timedelta(minutes=SESSION_LIFETIME_MINUTES) return self._auth
def connectES(esEndPoint): print ('Connecting to the ES Endpoint {0}'.format(esEndPoint)) try: access_key = os.getenv('AWS_ACCESS_KEY_ID') secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY') session_token = os.getenv('AWS_SESSION_TOKEN') region = os.getenv('AWS_REGION') auth = AWSRequestsAuth(aws_access_key=access_key, aws_secret_access_key=secret_access_key, aws_token=session_token, aws_host=esEndPoint, aws_region=region, aws_service='es') esClient = Elasticsearch(host=esEndPoint, port=443, use_ssl=True, connection_class=RequestsHttpConnection, http_auth=auth) return esClient except Exception as E: print("Unable to connect to {0}".format(esEndPoint)) print(E) exit(3)
def get_aws_auth(): """Get HTTP authentication for our Elastic Search AWS user""" return AWSRequestsAuth(aws_access_key=private.ES_ACCESS_KEY_ID, aws_secret_access_key=private.ES_SECRET_ACCESS_KEY, aws_host=private.ES_HOST, aws_region=private.ES_REGION, aws_service="es")
def amazonCall(url): sts_client = boto3.client('sts') # Call the assume_role method of the STSConnection object and pass the role # ARN and a role session name. assumedRoleObject = sts_client.assume_role( RoleArn="arn:aws:iam::543303400543:role/MyApiCall", RoleSessionName="mySession") # From the response that contains the assumed role, get the temporary # credentials that can be used to make subsequent API calls credentials = assumedRoleObject['Credentials'] # Use the temporary credentials that AssumeRole returns to make a # connection to Amazon S3 # s3_resource = boto3.resource( # 's3', # aws_access_key_id = , # aws_secret_access_key = , # aws_session_token = credentials['SessionToken'], # ) # print("s3 resource:",s3_resource) # # let's talk to our AWS Elasticsearch cluster auth = AWSRequestsAuth( aws_access_key=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_host='ec2-18-220-161-116.us-east-2.compute.amazonaws.com', aws_region='us-east-2', aws_service='compute') response = requests.get(url, auth=auth) return response
def lambda_handler(event, context): print('Event Received: {}'.format(json.dumps(event))) bucket_name = event['Records'][0]['s3']['bucket']['name'] # Wait for other objects to finish being uploaded time.sleep(30) resp = s3.list_objects_v2(Bucket=bucket_name)['Contents'] all_keys = [d['Key'] for d in resp] final_keys = [ key for key in all_keys if key.startswith('gisaid/') and key.endswith('vcf.gz') ] locations = [f's3://{bucket_name}/{d}' for d in final_keys] payload = { "id": "gisaid", "vcfLocations": locations, } api = BEACON_URL.split('/')[2] url = BEACON_URL + "/submit" auth = AWSRequestsAuth( aws_access_key=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_token=aws_session_token, aws_host=api, aws_region='ap-southeast-2', aws_service='execute-api', ) response = requests.patch(url, data=json.dumps(payload), auth=auth) print(dir(response))
def elasticsearch_client(): kwargs = { 'host': ES_HOST, 'port': ES_PORT, } if ES_SSL: kwargs.update({ 'use_ssl': True, 'ca_certs': certifi.where(), }) if ES_SIGNING: kwargs.update({ 'connection_class': RequestsHttpConnection, 'http_auth': AWSRequestsAuth(aws_host=ES_HOST, aws_region=ES_REGION, aws_service='es', **boto_utils.get_credentials()), }) return Elasticsearch(**kwargs)
def test_auth_for_get(self): auth = AWSRequestsAuth( aws_access_key='YOURKEY', aws_secret_access_key='YOURSECRET', aws_host='search-foo.us-east-1.es.amazonaws.com', aws_region='us-east-1', aws_service='es') url = 'http://search-foo.us-east-1.es.amazonaws.com:80/' mock_request = mock.Mock() mock_request.url = url mock_request.method = "GET" mock_request.body = None mock_request.headers = {} frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5) with mock.patch('datetime.datetime') as mock_datetime: mock_datetime.utcnow.return_value = frozen_datetime auth(mock_request) self.assertEqual( { 'Authorization': 'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, ' 'SignedHeaders=host;x-amz-date, ' 'Signature=ca0a856286efce2a4bd96a978ca6c8966057e53184776c0685169d08abd74739', 'x-amz-date': '20160618T220405Z', }, mock_request.headers)
def _make_elastic(self): """create elasticsearch client""" elastic_host = os.environ["ES_HOST"] session = boto3.session.Session() credentials = session.get_credentials().get_frozen_credentials() awsauth = AWSRequestsAuth( # These environment variables are automatically set by Lambda aws_access_key=credentials.access_key, aws_secret_access_key=credentials.secret_key, aws_token=credentials.token, aws_host=elastic_host, aws_region=session.region_name, aws_service="es" ) return Elasticsearch( hosts=[{"host": elastic_host, "port": 443}], http_auth=awsauth, max_backoff=get_time_remaining(self.context) if self.context else MAX_BACKOFF, # Give ES time to respond when under load timeout=ELASTIC_TIMEOUT, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection )
def get_elasticsearch_connection(): "gets an aws-authenticated elasticsearch connection" session = boto3.session.Session() credentials = session.get_credentials().get_frozen_credentials() es_host = ( "search-sciwiki-search-0-f7ntx2mpc5g6yohp6dtiwzsdiy.us-west-2.es.amazonaws.com" ) awsauth = AWSRequestsAuth( aws_access_key=credentials.access_key, aws_secret_access_key=credentials.secret_key, aws_token=credentials.token, aws_host=es_host, aws_region=session.region_name, aws_service="es", ) return Elasticsearch( hosts=[{ "host": es_host, "port": 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, )
async def async_process_db_change(event, context): aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID'] aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY'] aws_session_token = os.environ['AWS_SESSION_TOKEN'] auth = AWSRequestsAuth( aws_access_key=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_token=aws_session_token, aws_host='0ppn9ycig5.execute-api.us-east-1.amazonaws.com', aws_region='us-east-1', aws_service='execute-api') latest_record = {} for i, record in enumerate(event['Records']): latest_record[record['dynamodb']['Keys']['id']['S']] = i all_tasks = [] for game_id, ind in latest_record.items(): if 'NewImage' not in event['Records'][ind]['dynamodb']: continue if not event['Records'][ind]['dynamodb']['NewImage']['id'][ 'S'].startswith('game'): continue record = convert_record(event['Records'][ind]['dynamodb']['NewImage']) try: all_tasks.append(asyncio.create_task(handle_record(record, auth))) except ValueError as e: return { 'statusCode': 409, 'body': json.dumps('Game state not recognized') } await asyncio.gather(*all_tasks) return {'statusCode': 200, 'body': json.dumps('success')}
def test_auth_for_post_with_unicode_body_python2(self): auth = AWSRequestsAuth( aws_access_key='YOURKEY', aws_secret_access_key='YOURSECRET', aws_host='search-foo.us-east-1.es.amazonaws.com', aws_region='us-east-1', aws_service='es') url = 'http://search-foo.us-east-1.es.amazonaws.com:80/' mock_request = mock.Mock() mock_request.url = url mock_request.method = "POST" mock_request.body = 'foo=bar\xc3' mock_request.headers = { 'Content-Type': 'application/x-www-form-urlencoded', } frozen_datetime = datetime.datetime(2016, 6, 18, 22, 4, 5) with mock.patch('datetime.datetime') as mock_datetime: mock_datetime.utcnow.return_value = frozen_datetime auth(mock_request) self.assertEqual( { 'Authorization': 'AWS4-HMAC-SHA256 Credential=YOURKEY/20160618/us-east-1/es/aws4_request, ' 'SignedHeaders=host;x-amz-date, ' 'Signature=88046be72423b267de5e7e604aaffb2c5668c3fd9022ef4aac8287b82ab71124', 'Content-Type': 'application/x-www-form-urlencoded', 'x-amz-date': '20160618T220405Z', }, mock_request.headers)
def lambda_handler(event, context): es_host = os.environ['ELASTIC_SEARCH_HOSTNAME'] num_indexes_to_keep = int(os.environ['NUM_INDEXES_TO_KEEP']) index_prefix = os.environ['INDEX_PREFIX'] auth = AWSRequestsAuth(aws_access_key=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'], aws_token=os.environ['AWS_SESSION_TOKEN'], aws_region=os.environ['AWS_REGION'], aws_service='es', aws_host=es_host) indexResponse = requests.get(f"https://{es_host}/*", auth=auth) if (indexResponse.status_code != 200): raise Exception(f"failed to retrieve indexes: {indexResponse.text}") indexData = indexResponse.json() indexNames = sorted([x for x in indexData.keys() if x.startswith(index_prefix)]) indexesToDelete = indexNames[0 : max(0, len(indexNames) - num_indexes_to_keep)] print(f"number of selected indexes: {len(indexNames)}, number to delete: {len(indexesToDelete)}") for idx in indexesToDelete: deleteResponse = requests.delete(f"https://{es_host}/{idx}", auth=auth) if deleteResponse.status_code == 200: print(f"deleted {idx}") else: raise Exception(f"failed to delete {idx}: {deleteResponse.text}")
def __create_connection(self, config): kwargs = { 'host': config['HOST'], 'port': config.get('PORT', 9200), 'use_ssl': config.get('USE_SSL', False), 'verify_certs': True, 'ca_certs': certifi.where() } if 'AWS_ACCESS_KEY' in config and \ 'AWS_SECRET_KEY' in config and \ 'AWS_REGION' in config: kwargs['connection_class'] = RequestsHttpConnection kwargs['http_auth'] = AWSRequestsAuth( aws_access_key=config['AWS_ACCESS_KEY'], aws_secret_access_key=config['AWS_SECRET_KEY'], aws_host=config['HOST'], aws_region=config['AWS_REGION'], aws_service='es') es = Elasticsearch(**kwargs) es._index = config['INDEX_NAME'] es._settings = config.get('INDEX_SETTINGS', DEFAULT_INDEX_SETTINGS) return es
def test_multiple_get_params(self): """ Assert we generate the 'correct' cannonical query string for request that includes more than one query parameter """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/index/type/_search?scroll=5m&search_type=scan' mock_request = mock.Mock() mock_request.url = url self.assertEqual('scroll=5m&search_type=scan', AWSRequestsAuth.get_canonical_querystring(mock_request))
def test_post_request_with_get_param(self): """ Assert we generate the 'correct' cannonical query string for a post request that includes GET-parameters """ url = 'http://search-foo.us-east-1.es.amazonaws.com:80/index/type/1/_update?version=1' mock_request = mock.Mock() mock_request.url = url mock_request.method = "POST" self.assertEqual('version=1', AWSRequestsAuth.get_canonical_querystring(mock_request))