def validate_delivery(order: dict) -> Tuple[bool, str]:
"""
Validate the delivery price
"""
# Gather the domain name and AWS region
url = urlparse(DELIVERY_API_URL)
region = boto3.session.Session().region_name
# Create the signature helper
iam_auth = BotoAWSRequestsAuth(aws_host=url.netloc,
aws_region=region,
aws_service='execute-api')
# Send a POST request
response = requests.post(DELIVERY_API_URL + "/backend/pricing",
json={
"products": order["products"],
"address": order["address"]
},
auth=iam_auth)
logger.debug({
"message": "Response received from delivery",
"body": response.json()
})
body = response.json()
if response.status_code != 200 or "pricing" not in body:
logger.warning({
"message": "Failure to contact the delivery service",
"statusCode": response.status_code,
"body": body
})
return (False, "Failure to contact the delivery service")
if body["pricing"] != order["deliveryPrice"]:
logger.info({
"message":
"Wrong delivery price: got {}, expected {}".format(
order["deliveryPrice"], body["pricing"]),
"orderPrice":
order["deliveryPrice"],
"deliveryPrice":
body["pricing"]
})
return (False, "Wrong delivery price: got {}, expected {}".format(
order["deliveryPrice"], body["pricing"]))
return (True, "The delivery price is valid")
def main():
parsed = urlparse(NOTIFICATION_ENDPOINT)
auth = BotoAWSRequestsAuth(aws_host=parsed.netloc,
aws_region=API_REGION,
aws_service='execute-api')
print(PAYLOAD)
response = requests.post(NOTIFICATION_ENDPOINT,
json=PAYLOAD,
auth=auth,
timeout=25)
print(response.json())
if response.status_code != 200:
return 1
else:
return 0
def invoke_endpoint(payload):
"""
We get credentials from the IAM role of the notebook instance,
then use them to create a signed request to the API Gateway
"""
auth = BotoAWSRequestsAuth(
aws_host="{}.execute-api.{}.amazonaws.com".format(
config.REST_API_GATEWAY, config.AWS_REGION),
aws_region=config.AWS_REGION,
aws_service='execute-api')
invoke_url = "https://{}.execute-api.{}.amazonaws.com/prod/invocations".format(
config.REST_API_GATEWAY, config.AWS_REGION)
requests.post(invoke_url, json=payload, auth=auth)
def invoke_endpoint(payload):
# Gets credentials from the IAM role of the notebook instance,
# then uses them to create a signed request to the API Gateway
auth = BotoAWSRequestsAuth(
aws_host="1bd3rm2so6.execute-api.{}.amazonaws.com".format(region),
aws_region=region,
aws_service='execute-api')
invoke_url = "https://1bd3rm2so6.execute-api.{}.amazonaws.com/prod/invocations".format(
region)
response = requests.post(invoke_url, json=payload, auth=auth)
return response
def _get_from_kyivmural_api(resource, params=None):
"""Calls kyivmural api"""
region = current_app.config["AWS_REGION"]
api_url = f"{current_app.config['KYIVMURAL_API_ENDPOINT']}/{resource}"
api_host = urlparse(api_url).netloc
auth = BotoAWSRequestsAuth(aws_host=api_host,
aws_region=region,
aws_service="execute-api")
requests_session = _requests_retry_session()
response = requests_session.get(api_url, auth=auth, params=params)
response.raise_for_status()
if response.text:
return response.json()
return {}
def test_server(self):
# read the XML document used by the server
xml_file = open(SERVER_API_BODY_FILE, 'r')
xml = xml_file.read()
xml_file.close()
# create a signature for the call
auth = BotoAWSRequestsAuth(aws_host=API_HOST,
aws_region='us-east-1',
aws_service='execute-api')
# call the api
response = requests.post(
SERVER_API, headers=API_HEADERS, data=xml, auth=auth)
# get the XML response
root_element = ET.fromstring(response.text)
# check the key
key = root_element.find(
".//{urn:ietf:params:xml:ns:keyprov:pskc}PlainValue")
self.assertEqual(EXPECTED_SERVER_KEY, base64.b64decode(key.text))
def _get_es_conn():
conn = None
if settings.ELASTICSEARCH_AWS_HOST:
auth = BotoAWSRequestsAuth(
aws_host=settings.ELASTICSEARCH_AWS_HOST,
aws_region=settings.ELASTICSEARCH_AWS_REGION,
aws_service='es')
conn = Elasticsearch(host=settings.ELASTICSEARCH_AWS_HOST,
connection_class=RequestsHttpConnection,
port=80,
http_auth=auth)
else:
conn = Elasticsearch(hosts=settings.ELASTICSEARCH_HOSTS,
use_ssl=settings.ELASTICSEARCH_SSL,
http_auth=settings.ELASTICSEARCH_AUTH,
verify_certs=settings.ELASTICSEARCH_SSL,
ca_certs=certifi.where())
return conn
def validate_payment(order: dict) -> Tuple[bool, str]:
"""
Validate the payment token
"""
# Gather the domain name and AWS region
url = urlparse(PAYMENT_API_URL)
region = boto3.session.Session().region_name
# Create the signature helper
iam_auth = BotoAWSRequestsAuth(aws_host=url.netloc,
aws_region=region,
aws_service='execute-api')
# Send a POST request
response = requests.post(PAYMENT_API_URL + "/backend/validate",
json={
"paymentToken": order["paymentToken"],
"total": order["total"]
},
auth=iam_auth)
logger.debug({
"message": "Response received from payment",
"body": response.json()
})
body = response.json()
if response.status_code != 200 or "ok" not in body:
logger.warning({
"message": "Failure to contact the payment service",
"statusCode": response.status_code,
"body": body
})
return (False, "Failure to contact the payment service")
if not body["ok"]:
logger.info({
"message": "Wrong payment token",
"paymentToken": order["paymentToken"],
"total": order["total"]
})
return (False, "Wrong payment token")
return (True, "The payment token is valid")
def __init__(self):
self.scriptExecutionLocation = os.getcwd()
# Config
print("Reading configuration file")
self.config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
self.config.read(
os.path.join(self.scriptExecutionLocation, 'config.ini'))
# Blockchain RPC
self.blockchainRpc = self.config['blockchain']['rpc']
print("Blockchain RPC: %s" % self.blockchainRpc)
# Master index
self.masterIndex = self.config['masterindex']['all']
print("Master index: %s" % self.masterIndex)
# Elasticsearch endpoint
self.elasticSearchEndpoint = self.config['elasticSearch']['endpoint']
print("ElasticSearch Endpoint: %s" % self.elasticSearchEndpoint)
# Elasticsearch AWS region
self.elasticSearchAwsRegion = self.config['elasticSearch'][
'aws_region']
# Web 3 init
self.web3 = Web3(HTTPProvider(str(self.blockchainRpc)))
# AWS Boto
self.auth = BotoAWSRequestsAuth(aws_host=self.elasticSearchEndpoint,
aws_region=self.elasticSearchAwsRegion,
aws_service='es')
self.es = Elasticsearch(hosts=[{
'host': self.elasticSearchEndpoint,
'port': 443
}],
region=self.elasticSearchAwsRegion,
use_ssl=True,
verify_certs=True,
http_auth=self.auth,
connection_class=RequestsHttpConnection)
def step_impl(context):
# Execute the lambada with Boto and
auth = BotoAWSRequestsAuth(aws_host=context.env_vars['awsApiHost'],
aws_region=context.env_vars['region'],
aws_service='execute-api')
params = {'force': 'true', 'offset': '1'}
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(context.env_vars['dynamoDbName'])
response = table.update_item(
Key=context.env_vars['dynamoDBPrimaryKey'],
UpdateExpression='set lang=:l',
ExpressionAttributeValues={
':l': "en"
},
ReturnValues="UPDATED_NEW"
)
print("Response: {}".format(response))
def get_order(order_id: str) -> Optional[dict]:
"""
Retrieve the order from ther Orders service
"""
logger.debug({
"message": "Retreiving order {}".format(order_id),
"orderId": order_id
})
# Create IAM signature helper
request_url = ORDERS_API_URL + order_id
url = urlparse(request_url)
region = boto3.session.Session().region_name
auth = BotoAWSRequestsAuth(aws_host=url.netloc,
aws_region=region,
aws_service='execute-api')
# Send request to order service
response = requests.get(request_url, auth=auth)
if response.status_code != 200:
logger.error({
"message": "Failed to retrieve order {}".format(order_id),
"orderId": order_id,
"statusCode": response.status_code,
"response": response.json()
})
return None
order = response.json()
logger.info({
"message":
"Retrieved order {} from Orders service".format(order_id),
"orderId":
order_id,
"order":
order
})
return order
def lambda_handler(event, context):
auth = BotoAWSRequestsAuth(aws_host=os.environ['ES_HOST'],
aws_region=os.environ['ES_REGION'],
aws_service='es')
es_client = Elasticsearch(host=os.environ['ES_HOST'],
port=443,
use_ssl=True,
connection_class=RequestsHttpConnection,
http_auth=auth)
s3_client = boto3.client('s3')
event_bucket = event['Records'][0]['s3']['bucket']['name']
event_key = event['Records'][0]['s3']['object']['key']
downloaded_file_path = '/tmp/cloudtrail_log.gz'
s3_client.download_file(event_bucket, event_key, downloaded_file_path)
record_set = parse_log('/tmp/cloudtrail_log.gz')
resp = write_bulk(record_set, es_client)
print(resp)
def get_es_client(config):
global ES_CLIENT
aws_es = config.get('AWS_ES', False)
aws_region = config.get('AWS_REGION')
es_host = config.get('ES_HOST', '127.0.0.1')
es_url = config.get('ES_URL', 'http://127.0.0.1:9200')
if ES_CLIENT is None:
if aws_es is True:
ES_CLIENT = Elasticsearch(
hosts=[es_url],
http_auth=BotoAWSRequestsAuth(aws_host=es_host, aws_region=aws_region, aws_service='es'),
use_ssl=True,
connection_class=RequestsHttpConnection,
verify_certs=False,
ssl_show_warn=False
)
else:
ES_CLIENT = Elasticsearch(hosts=[es_url])
return ES_CLIENT
def post_DocSim(language, data):
stage = os.environ['STAGE']
if stage == 'prod':
host = 'api.codetekt.org'
else:
host = 'api.{}.codetekt.org'.format(stage)
if language == "de":
url = "https://" + host + "/ml_model_service/models/DocSim"
else:
logger.error("Language not supported!")
raise Exception('Language not supported by DocSim!')
headers = {"content-type": "text/csv", "Accept": "text/csv"}
auth = BotoAWSRequestsAuth(aws_host=host,
aws_region="eu-central-1",
aws_service="execute-api")
response = requests.post(url,
headers=headers,
data=data.encode('utf-8'),
auth=auth)
return response
def _create_es(search_endpoint, aws_region):
"""
search_endpoint: url for search endpoint
aws_region: name of aws region endpoint is hosted in
"""
es_url = urlparse(search_endpoint)
auth = BotoAWSRequestsAuth(aws_host=es_url.hostname,
aws_region=aws_region,
aws_service='es')
port = es_url.port or (443 if es_url.scheme == 'https' else 80)
es_client = Elasticsearch(hosts=[{
'host': es_url.hostname,
'port': port
}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
return es_client
def _post_audit_info(audit_api_url: str,
path: str,
start_time: int,
exit_code: int = None,
stdout: List[str] = None,
update: bool = False):
root = get_git_root(path)
path = path.replace(root, '')
status = Status.IN_PROGRESS if exit_code is None else (
Status.SUCCESS if exit_code == 0 else Status.FAILED)
logger.info('Attempting to send data to Audit API: %s - %s', path, status)
url = (audit_api_url + AUDIT_UPDATE_PATH) if update else (audit_api_url +
AUDIT_POST_PATH)
auth = BotoAWSRequestsAuth(
aws_host='terraform-audit-api.devops.amplify.com',
aws_region='us-west-2',
aws_service='execute-api')
stdout_str = ''.join(stdout) if stdout else ''
try:
requests.post(url=url,
auth=auth,
json={
'directory': path,
'start_time': start_time,
'status': status,
'output': stdout_str
})
logger.info('Successfully posted data to provided url: %s',
audit_api_url)
except requests.exceptions.RequestException:
logger.error("Unable to post data to provided url: %s", audit_api_url)
def get_grq_es(logger=None):
global GRQ_ES
if GRQ_ES is None:
aws_es = app.conf.get('GRQ_AWS_ES', False)
es_host = app.conf['GRQ_ES_HOST']
es_url = app.conf['GRQ_ES_URL']
region = app.conf['AWS_REGION']
if aws_es is True:
aws_auth = BotoAWSRequestsAuth(aws_host=es_host,
aws_region=region,
aws_service='es')
GRQ_ES = ElasticsearchUtility(
es_url=es_url,
logger=logger,
http_auth=aws_auth,
connection_class=RequestsHttpConnection,
use_ssl=True,
verify_certs=False,
ssl_show_warn=False)
else:
GRQ_ES = ElasticsearchUtility(es_url, logger)
return GRQ_ES
def validate_products(order: dict) -> Tuple[bool, str]:
"""
Validate the products in the order
"""
# Gather the domain name and AWS region
url = urlparse(PRODUCTS_API_URL)
region = boto3.session.Session().region_name
# Create the signature helper
iam_auth = BotoAWSRequestsAuth(aws_host=url.netloc,
aws_region=region,
aws_service='execute-api')
# Send a POST request
response = requests.post(PRODUCTS_API_URL + "/backend/validate",
json={"products": order["products"]},
auth=iam_auth)
logger.debug({
"message": "Response received from products",
"body": response.json()
})
body = response.json()
return (len(body.get("products", [])) == 0, body.get("message", ""))
def _create_es(search_endpoint, aws_region):
"""
search_endpoint: url for search endpoint
aws_region: name of aws region endpoint is hosted in
"""
es_url = urlparse(search_endpoint)
credentials = create_botocore_session().get_credentials()
if credentials:
# use registry-provided credentials
creds = credentials.get_frozen_credentials()
auth = AWSRequestsAuth(
aws_access_key=creds.access_key,
aws_secret_access_key=creds.secret_key,
aws_host=es_url.hostname,
aws_region=aws_region,
aws_service='es',
aws_token=creds.token,
)
else:
auth = BotoAWSRequestsAuth(aws_host=es_url.hostname,
aws_region=aws_region,
aws_service='es')
port = es_url.port or (443 if es_url.scheme == 'https' else 80)
es_client = Elasticsearch(hosts=[{
'host': es_url.hostname,
'port': port
}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
return es_client
def _load_auth(self):
return BotoAWSRequestsAuth(aws_host=self.es_domain,
aws_region=self.region,
aws_service='es')
# Blockchain RPC
blockchainRpc = config['blockchain']['rpc']
print("Blockchain RPC: %s" % blockchainRpc)
# Elasticsearch endpoint
elasticSearchEndpoint = config['elasticSearch']['endpoint']
print("ElasticSearch Endpoint: %s" % elasticSearchEndpoint)
# Elasticsearch AWS region
elasticSearchAwsRegion = config['elasticSearch']['aws_region']
# Web 3 init
web3 = Web3(HTTPProvider(str(blockchainRpc)))
# AWS Boto
auth = BotoAWSRequestsAuth(aws_host=elasticSearchEndpoint, aws_region=elasticSearchAwsRegion, aws_service='es')
es = Elasticsearch(
hosts=[{'host': elasticSearchEndpoint, 'port': 443}],
region=elasticSearchAwsRegion,
use_ssl=True,
verify_certs=True,
http_auth=auth,
connection_class=RequestsHttpConnection
)
# New data
doc = {}
outerData = {}
outerData["indexed"] = "false"
doc["doc"] = outerData
# Match all to access all records
def signed_api_call(service: str, path: str = "/", method: str = 'GET',
configuration: Configuration = None,
secrets: Secrets = None,
params: Dict[str, Any] = None) -> requests.Response:
"""
Perform an API call against an AWS service.
This should only be used when boto does not already implement the service
itself. See https://boto3.readthedocs.io/en/latest/reference/services/index.html
for a list of supported services by boto. This function does not claim
being generic enough to support the whole range of AWS API.
The `configuration` object should look like this:
```json
{
"aws_region": "us-east-1",
"aws_host": "amazonaws.com"
}
```
While both are optional, and default to the values shown in this snippet,
you should make sure to be explicit about them to avoid confusion.
The endpoint being called is built from the given `service` name, the
given region and host as well as the `path` of the action being called on
the service. By default, the call is made over `HTTPS` but this can be
changed by setting `aws_endpoint_scheme` in the configuration dictionary.
Pass any parameters of the API itself as part of the remaining `params`
paramater is a dictionary. It should match the signature of the service
you are trying to call and will be sent as a query-string when `method` is
`"GET"` or `"DELETE"`, or as a JSON payload otherwise. Refer to the AWS
documentation for each service type.
This function does not support profile names so you must provide the
credentials in secrets.
""" # noqa: E501
configuration = configuration or {}
region = configuration.get("aws_region", "us-east-1") or ""
host = configuration.get("aws_host", "amazonaws.com")
scheme = configuration.get("aws_endpoint_scheme", "https")
host = "{s}.{r}.{h}".format(s=service, r=region, h=host)
endpoint = configuration.get(
"aws_endpoint", '{scheme}://{h}'.format(
scheme=scheme, h=host)).replace('..', '.')
endpoint = "{e}{p}".format(e=endpoint, p=path)
creds = get_credentials(secrets)
# when creds weren't provided via secrets, we let boto search for them
# from the process environment
if creds["aws_access_key_id"] and creds["aws_secret_access_key"]:
auth = AWSRequestsAuth(
aws_access_key=creds["aws_access_key_id"],
aws_secret_access_key=creds["aws_secret_access_key"],
aws_host=host,
aws_region=region,
aws_service=service)
else:
auth = BotoAWSRequestsAuth(
aws_host=host,
aws_region=region,
aws_service=service)
headers = {
"Accept": "application/json"
}
if method in ('DELETE', 'GET'):
return requests.request(
method, endpoint, headers=headers, auth=auth, params=params)
return requests.request(
method, endpoint, headers=headers, auth=auth, json=params)
parser = argparse.ArgumentParser(description='Ingest jsonl data into an Elasticsearch instance.')
parser.add_argument("-e","--host",help="ip address of Elasticsearch host. Defaults to localhost:9200",default=None)
parser.add_argument("-a","--aws",help="Using Amazon Web Services: requires boto3",action="store_true")
parser.add_argument("-r","--aws_region",help="AWS region of Elasticsearch instance",default="us-east-2")
parser.add_argument("index",help="Elasticsearch index to be used.")
return parser
if __name__ == "__main__":
es=None
args = form_parser().parse_args()
if args.aws:
if not(args.aws and args.aws_region and args.host):
print("Error:AWS region and AWS elasticsearch host needed with AWS flag")
sys.exit(1)
from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
auth = BotoAWSRequestsAuth(aws_host=args.host,
aws_region=args.aws_region,
aws_service='es')
es = Elasticsearch(host=args.host,
port=80,
connection_class=RequestsHttpConnection,
http_auth=auth,
timeout=30, max_retries=10, retry_on_timeout=True)
elif args.host:
es = Elasticsearch(args.host,timeout=30, max_retries=10, retry_on_timeout=True)
else:
es = Elasticsearch(timeout=30, max_retries=10, retry_on_timeout=True)
print(helpers.bulk(es, form_query(args.index,es)))
def boto_auth():
return BotoAWSRequestsAuth(aws_host='s3-us-west-2.amazonaws.com',
aws_region='us-west-2',
aws_service='s3')
def lambda_handler(request):
"""
Proxy the request to the elastic search.
"""
action = request.args.get('action')
indexes = request.args.get('index')
if action == 'search':
query = request.args.get('query', '')
body = {
"query": {
"simple_query_string": {
"query": query,
"fields": ['content', 'comment', 'key_text', 'meta_text']
}
}
}
# TODO: should be user settable; we should proably forbid `content` (can be huge)
_source = [
'key', 'version_id', 'updated', 'last_modified', 'size',
'user_meta'
]
size = 1000
elif action == 'stats':
body = {
"query": {
"match_all": {}
},
"aggs": {
"totalBytes": {
"sum": {
"field": 'size'
}
},
"exts": {
"terms": {
"field": 'ext'
},
"aggs": {
"size": {
"sum": {
"field": 'size'
}
}
},
},
"updated": {
"max": {
"field": 'updated'
}
},
}
}
size = 0
_source = []
else:
return make_json_response(400, {"title": "Invalid action"})
es_host = os.environ['ES_HOST']
region = os.environ['AWS_REGION']
auth = BotoAWSRequestsAuth(aws_host=es_host,
aws_region=region,
aws_service='es')
es_client = Elasticsearch(hosts=[{
'host': es_host,
'port': 443
}],
http_auth=auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
to_search = f"{indexes},{INDEX_OVERRIDES}" if INDEX_OVERRIDES else indexes
result = es_client.search(to_search,
body,
_source=_source,
size=size,
timeout=MAX_QUERY_DURATION)
return make_json_response(200, result)
def main(argv=None):
"""Run when invoked from the command-line."""
colorama.init(autoreset=True)
# Create an arparse argument parser for parsing command-line arguments
desc = 'A command line HTTP client for AWS services with an intuitive UI, XML support and syntax highlighting.'
epilog = 'See the AWS Documentation for API references for each service: https://docs.aws.amazon.com'
parser = argparse.ArgumentParser(description=desc,
epilog=epilog,
prog='http_aws')
parser.add_argument(
'-r',
'--region',
help='The region to use. Overrides config/env settings.')
parser.add_argument('-s',
'--service',
help='AWS service - e.g. ec2, s3, etc.',
default='ec2')
parser.add_argument(
'-e',
'--endpoint',
help=
"Override command's default URL with the given URL - e.g. ec2.us-east-1.amazonaws.com"
)
parser.add_argument(
'-c',
'--creds',
help=
"Override AWS Access Key Id and AWS Secret Access Key - i.e. <Access_Key>:<Secret_Key>"
)
parser.add_argument('-v',
'--version',
help='API version to use for the service',
default='2015-10-01')
parser.add_argument('-p',
'--paginate',
action='store_true',
help='Paginate long output')
parser.add_argument(
'-w',
'--wrap',
action='store_true',
help='Wrap long lines in paginated output instead of chopping them off'
)
parser.add_argument(
'api',
help='Name of the API to call - e.g. "DescribeVpcs" (for ec2 service)')
args = parser.parse_args(argv)
# --- Configure AWS basics ---
# Configure AWS region
aws_region = 'us-east-1' # Default if not overriden on command-line or specified in config file
if args.region:
aws_region = args.region
else:
import os
# Read the region from the ~/.aws/config file if it exists
try:
with open(os.path.expanduser('~/.aws/config')) as f:
data = f.read()
lines = data.splitlines()
for line in lines:
if line.startswith('region = '):
aws_region = line.split('region = ')[1]
break
except (FileNotFoundError, PermissionError):
pass
# Configure AWS service
if args.service:
aws_service = args.service
else:
aws_service = 'ec2' # Default if not overriden on command-line
# Configure AWS endpoint
if args.endpoint:
aws_endpoint = args.endpoint
else:
aws_endpoint = '{}.{}.amazonaws.com'.format(aws_service, aws_region)
if args.creds:
from aws_requests_auth.aws_auth import AWSRequestsAuth
# Use the specified AWS access and secret key
try:
access_key, secret_key = args.creds.split(':')
auth = AWSRequestsAuth(aws_access_key=access_key,
aws_secret_access_key=secret_key,
aws_host=aws_endpoint,
aws_region=aws_region,
aws_service=aws_service)
except ValueError as ex:
perror(
'Credentials must be proviced in the format "<AWS_Access_Key_Id>:<AWS_Secret_Access_key>'
)
return -1
else:
# This line will fail if you do not have both aws-requests-auth and botocore installed
from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
# Use Boto to automatically gather AWS credentials from environment variables, AWS config files, or IAM Role
auth = BotoAWSRequestsAuth(aws_host=aws_endpoint,
aws_region=aws_region,
aws_service=aws_service)
# Configure details of the API call
api = args.api
api_version = args.version
params = {'Action': api, 'Version': api_version}
url = 'https://{}'.format(aws_endpoint)
# Send a GET request
try:
# TODO: Support PUT requests for Mutating API calls
response = requests.get(url=url, params=params, auth=auth)
except requests.exceptions.ConnectionError:
perror('Error connecting to host {!r}'.format(url))
return -1
# Gather response details
response_text = 'Response code: {}'.format(response.status_code)
header_text = 'Headers: {}'.format(response.headers)
# Convert the response content from an encoded byte string to a Unicode string
response_bytes = response.content
response_str = response_bytes.decode()
# If the respose is XML, ensure that it is nicely formatted with good indenting and newlines
if response_str.startswith('<?xml'):
try:
import lxml.etree as etree
response_bytes = etree.tostring(etree.fromstring(response.content),
pretty_print=True)
except ImportError:
pass
# Pretty-print the content of the response with syntax highlighting for readability
highlighted_text = highlight(response_bytes, guess_lexer(response_str),
TerminalFormatter())
output_text = '{}\n{}\n{}'.format(response_text, header_text,
highlighted_text)
if args.paginate:
ppaged(output_text, wrap=args.wrap)
else:
print(output_text)
return 0
# RoleArn="arn:aws:iam::029272617770:role/presigner_caller_role",
# RoleSessionName="ProdSessionFromBK")
# credentials = assumed_role_object["Credentials"]
# # Construct the HTTP auth to call the API gateway
# auth = AWSRequestsAuth(
# aws_host="vop4ss7n22.execute-api.us-west-2.amazonaws.com",
# aws_region="us-west-2",
# aws_service="execute-api",
# aws_access_key=credentials["AccessKeyId"],
# aws_secret_access_key=credentials["SecretAccessKey"],
# aws_token=credentials["SessionToken"],
# )
auth = BotoAWSRequestsAuth(
aws_host="vop4ss7n22.execute-api.us-west-2.amazonaws.com",
aws_region="us-west-2",
aws_service="execute-api",
)
resp = requests.get(
"https://vop4ss7n22.execute-api.us-west-2.amazonaws.com/endpoint/",
auth=auth,
params={"job_id": os.environ["BUILDKITE_JOB_ID"]})
print("Getting Presigned URL", resp.status_code)
sha = os.environ["BUILDKITE_COMMIT"]
if is_dir:
paths = [os.path.join(args.path, f) for f in os.listdir(args.path)]
else:
paths = [args.path]
print("Planning to upload", paths)
from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
from elasticsearch import Elasticsearch, RequestsHttpConnection
import time
def delete_index(index_name):
es_client.indices.delete(index=index_name, ignore=[400, 404])
es_host = 'ES_ENDPOINT'
INDEX_NAME = ''
auth = BotoAWSRequestsAuth(aws_host=es_host,
aws_region='eu-west-1',
aws_service='es')
# use the requests connection_class and pass in our custom auth class
es_client = Elasticsearch(host=es_host,
port=80,
connection_class=RequestsHttpConnection,
http_auth=auth)
def lambda_handler(event, context):
res = es_client.search(
index=INDEX_NAME,
body={
"query": {
"constant_score": {
"filter": {
"bool": {
def lambda_handler(event, context):
es_host = 'ES_ENDPOINT'
auth = BotoAWSRequestsAuth(aws_host=es_host,
aws_region='eu-west-1',
aws_service='es')
# use the requests connection_class and pass in our custom auth class
es_client = Elasticsearch(host=es_host,
port=80,
connection_class=RequestsHttpConnection,
http_auth=auth)
request_body = {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
}
}
INDEX_NAME = "awsinventory-" + datetime.date.today().strftime("%Y.%d.%m")
if (es_client.indices.exists(index=INDEX_NAME) == True):
es_client.indices.delete(index=INDEX_NAME, ignore=[400, 404])
es_client.indices.create(index=INDEX_NAME, body=request_body)
print "Index Restored"
else:
res = es_client.indices.create(index=INDEX_NAME, body=request_body)
print "Index Created"
sts_client = boto3.client('sts')
assumedRoleObject = sts_client.assume_role(
RoleArn='arn:aws:iam::828006401370:role/Security-AdministratorAccess',
RoleSessionName='Security-AdministratorAccess')
credentials = assumedRoleObject['Credentials']
iam_client = boto3.client(
'iam',
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
)
paginator = iam_client.get_paginator('list_role_policies')
response_iterator = paginator.paginate(RoleName=default_role_name)
for page in response_iterator:
for policy_name in page['PolicyNames']:
role_policy = iam_client.get_role_policy(
RoleName=default_role_name, PolicyName=policy_name)
listed_roles.extend(
[role_policy['PolicyDocument']['Statement']['Resource']])
for role_arn in listed_roles:
role_name = role_arn.split("/")[-1]
account_name = role_name.rsplit('-', 1)[0]
print "Account: %s" % account_name
assumedRoleObject = sts_client.assume_role(RoleArn=role_arn,
RoleSessionName=role_name)
credentials = assumedRoleObject['Credentials']
inventory['AWS.account'] = account_name
inventory['Service'] = {}
inventory['Service']['EC2'] = []
inventory['Service']['ELB'] = []
ec2_client = boto3.client(
'ec2',
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
)
response = ec2_client.describe_instances()
for instances in response['Reservations']:
for instance in instances['Instances']:
inventory['Service']['EC2'].extend([instance['InstanceId']])
elb_client = boto3.client(
'elb',
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
)
response = elb_client.describe_load_balancers()
for elb in response['LoadBalancerDescriptions']:
inventory['Service']['ELB'].extend([elb['LoadBalancerName']])
res = es_client.index(index=INDEX_NAME,
doc_type="Defect",
body=inventory)
print(res['result'])
inventory.clear
return
parser.add_argument('--verify-certs',
action='store_true',
default=False,
help='verify SSL if using https')
parser.add_argument('--start-position',
type=int,
default=0,
help='file position to where left off')
args = parser.parse_args()
service = 'es'
region = args.region # us-west-1
aws_auth = BotoAWSRequestsAuth(aws_host=args.es_url,
aws_region=region,
aws_service='es')
grq_es = Elasticsearch(hosts=[args.es_url],
port=args.port,
http_auth=aws_auth,
use_ssl=True,
verify_certs=args.verify_certs,
connection_class=RequestsHttpConnection,
timeout=60,
max_retries=10,
retry_on_timeout=True)
print("script start time: %s" % datetime.now().isoformat())
create_geonames_mapping(grq_es) # create geonames index with mapping