from elasticsearch import Elasticsearch, RequestsHttpConnection
import boto3
from requests_aws4auth import AWS4Auth
import json
region = 'eu-west-1'
session = boto3.session.Session(region_name=region)
credentials = session.get_credentials()
host = 'search-idc-7dyp7fwnklhj22hwzpuoa5hgsy.eu-west-1.es.amazonaws.com'
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, 'es')
es = Elasticsearch(
hosts=[{'host': host, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
query = {
"query": {
"range": {
"time": {
"gte": "now-5s/s"
}
}
}
}
result = es.search(index='meir_miran_data', body=query)
print(json.dumps(result))
from requests_aws4auth import AWS4Auth
from elasticsearch import Elasticsearch, RequestsHttpConnection
#AWS ElasticSearch Configuration
HOST = 'aws-es-instance-url.es.amazonaws.com'
AWS_ACCESS_KEY = 'AWS Access Key'
AWS_SECRET_KEY = 'AWS Secret Key'
REGION = 'us-east-2'
SERVICE = 'es'
# Initializing AWS Authentication used to sign our ElasticSearch requests
awsAuth = AWS4Auth(AWS_ACCESS_KEY, AWS_SECRET_KEY, REGION, SERVICE)
# Configuring AWS ElasticSearch connection
es = Elasticsearch(hosts=[{
'host': HOST,
'port': 443
}],
http_auth=awsAuth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
# Index information
theIndex = "movies"
theDocTypeMapping = "movie"
sys.exit(0)
## SVL: get credentials on datastore
awsProfile = "download"
credFilePath = '/home/sbp/.aws/credentials'
with open(credFilePath) as fp:
for line in fp:
if re.match("\[" + awsProfile + "\]", line) is not None:
accessKeyLine = fp.next().rstrip()
secretKeyLine = fp.next().rstrip()
accessKey = accessKeyLine.split(' = ')[1]
secretKey = secretKeyLine.split(' = ')[1]
fp.close()
r = requests.head('https://s3.object02.schubergphilis.com/' + sys.argv[1],
auth=AWS4Auth(accessKey, secretKey, 'eu-nl-prod01', 's3'),
timeout=30)
if 'x-amz-restore' in r.headers:
parsed = parse_amz_restore(r.headers['x-amz-restore'])
if 'expiry-date' in parsed and parsed['expiry-date'] > datetime.now(
timezone('Europe/Amsterdam')) + timedelta(days=7):
print 'Object already restored till ' + str(
parsed['expiry-date']) + ' so not requesting new restore'
sys.exit(0)
r = requests.post('https://s3.object02.schubergphilis.com/' + sys.argv[1] +
'?restore',
auth=AWS4Auth(accessKey, secretKey, 'eu-nl-prod01', 's3'),
timeout=30,
dynamodb = boto3.resource('dynamodb')
yelp_restaurants = dynamodb.Table('yelp-restaurants')
response = yelp_restaurants.scan()
data = response['Items']
while response.get('LastEvaluatedKey'):
response = yelp_restaurants.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
data.extend(response['Items'])
print(len(data))
session = boto3.session.Session()
credentials = session.get_credentials()
service = 'es'
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, session.region_name, service)
endpoint = "https://search-restaurants-ccwjaawlwnwrn73n7v5j3rf2sm.us-east-1.es.amazonaws.com"
es = Elasticsearch(
endpoint,
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
print(es.info())
count = 0
for restaurant in data:
restaurantID = restaurant['id']
cuisine = restaurant['cuisine']
from __future__ import print_function
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
from certificate.my_AWS_key import *
import json
# ES Client
REGION = "us-east-1"
host = 'search-tweet-kkj4pr2lzt2sfl3jzxq2a45a7e.us-east-1.es.amazonaws.com'
INDEX = 1
awsauth = AWS4Auth(ACCESS_KEY, SECRET_KEY, REGION, 'es')
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
def indextoES(event, context):
msg = event['Records'][0]['Sns']['Message']
print("From SNS: " + msg)
message = {
"text": event['Records'][0]['Sns']['Message']['text'],
"lat": event['Records'][0]['Sns']['Message']['lat'],
"lng": event['Records'][0]['Sns']['Message']['lng'],
"sentiment": event['Records'][0]['Sns']['Message']['sentiment']
}
import json
from datetime import datetime
import os
from requests_aws4auth import AWS4Auth
import boto3
import random as rand
#expexting AWS host
host = '<insert host>.us-east-1.es.amazonaws.com'
region = 'us-east-1'
service = 'es'
aws_access_key_id = os.environ['aws_access_key_id'],
aws_secret_access_key = os.environ['aws_secret_access_key']
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(aws_access_key_id, aws_secret_access_key, region, service)
# elasticsearch credentials
port = 443
es = Elasticsearch(hosts=[{
"host": host,
"port": port
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
# returns the name of the next unprocessed file.
def getname(file_uuid):
def __init__(self, configuration):
super(AmazonElasticsearchService, self).__init__(configuration)
self.auth = AWS4Auth(configuration['access_key'],
configuration['secret_key'],
configuration['region'], 'es')
def setUp(self):
self.nons3auth = AWS4Auth('id', 'secret', 'us-east-1', 'es')
self.s3auth = AWS4Auth('id', 'secret', 'us-east-1', 's3')
def setUp(self):
self.auth = AWS4Auth('id', 'secret', 'us-east-1', 'es')
def test_raise_invalid_date_default(self):
auth = AWS4Auth('access_id', 'secret_key', 'region', 'service')
self.assertFalse(auth.raise_invalid_date)
def test_default_include_hdrs(self):
auth = AWS4Auth('access_id', 'secret_key', 'region', 'service')
check_set = {'host', 'content-type', 'date', 'x-amz-*'}
self.assertSetEqual(set(auth.include_hdrs), check_set)
import json
import csv
import boto3
from elasticsearch import Elasticsearch, RequestsHttpConnection
from elasticsearch.helpers import bulk, streaming_bulk
from requests_aws4auth import AWS4Auth
ES_URL = "vpc-movies-3br6jppkeulox64eyaohgzzl7u.ap-south-1.es.amazonaws.com"
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(
credentials.access_key,
credentials.secret_key,
'ap-south-1',
'es',
session_token=credentials.token
)
es_client = Elasticsearch(
hosts=[{'host': ES_URL, 'port': 443}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
def write_into_es(filename):
bulk(es_client, generate_es_doc(filename))
def __init__(self):
self.env_vars = env_variables()
self.stack_resources = stack_resources(self.env_vars)
self.auth = AWS4Auth(self.env_vars['ACCESS_KEY'], self.env_vars['SECRET_KEY'],
self.env_vars['REGION'], 'execute-api')
def lambda_handler(event, context):
sqs = boto3.client('sqs')
queue_url = 'https://sqs.us-west-2.amazonaws.com/844243140541/food-bot-queue'
# Receive message from SQS queue
response = sqs.receive_message(
QueueUrl=queue_url,
AttributeNames=[
'SentTimestamp'
],
MaxNumberOfMessages=1,
MessageAttributeNames=[
'All'
],
VisibilityTimeout=0,
WaitTimeSeconds=0
)
if (response and 'Messages' in response):
# restaurant original host
# host = 'search-es-yelp-555toxbazyu56cgn5sgplxwgqy.us-west-2.es.amazonaws.com'
# extra credit new host
host = 'search-predictions-lmr33sl6iq3l4on4tmmsmzbf4a.us-west-2.es.amazonaws.com'
region = 'us-west-2'
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service)
es = Elasticsearch(
hosts = [{'host': host, 'port': 443}],
# http_auth = awsauth,
use_ssl = True,
verify_certs = True,
connection_class = RequestsHttpConnection
)
dynamodb = boto3.resource('dynamodb', region_name='us-west-2')
table = dynamodb.Table('yelp-restaurants')
for each_message in response['Messages']:
message = each_message
receipt_handle = message['ReceiptHandle']
req_attributes = message['MessageAttributes']
res_category = req_attributes['Categories']['StringValue']
# old query
# searchData = es.search(index="restaurants", body={
# "query": {
# "match": {
# "categories.title": res_category
# }}})
# new query
searchData = es.search(index="predictions", body={
"sort" :[{'score': 'desc'}],
"query": {
"match": {
"cuisine": "American"#res_category
}}})
# print (res_category)
# print (searchData)
print("Got %d Hits:" % searchData['hits']['total'])
businessIds = []
for hit in searchData['hits']['hits']:
businessIds.append(hit['_source']['id'])
# Call the dynemoDB
resultData = getDynemoDbData(table, req_attributes, businessIds)
print (resultData)
print ('req_attributes----', req_attributes)
# send the email
sendMailToUser(req_attributes, resultData)
# Delete received message from queue
sqs.delete_message(
QueueUrl=queue_url,
ReceiptHandle=receipt_handle
)
return {
'statusCode': 200,
'body': json.dumps('Hello from Lambda!')
}
# -*- coding: utf-8 -*-
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
import os
AWS_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY')
AWS_SECRET_KEY = os.environ.get('AWS_SECRET_KEY')
region = 'us-east-1'
service = 'es'
awsauth = AWS4Auth(AWS_ACCESS_KEY, AWS_SECRET_KEY, region, service)
host = 'search-django-bdjdxhfakpxgveshtpficlgk7e.us-east-1.es.amazonaws.com'
if __name__=="__main__":
es = Elasticsearch(
hosts = [{'host': host, 'port': 443}],
http_auth = awsauth,
use_ssl = True,
verify_certs = True,
connection_class = RequestsHttpConnection
)
#es = Elasticsearch([{'host': 'localhost', 'port': 9200}])
#mappings = {"mappings":{"tweet_data":{"properties":{"coordinates":{"type":"geo_point"},"created_at":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"text":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timestamp_ms":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"user_name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"user_screen_name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}},"settings":{"index":{"number_of_shards":"5","number_of_replicas":"1"}}}
mappings = {"mappings":{"tweet_data":{"properties":{"coordinates":{"type":"geo_point"},"created_at":{"type":"text"},"text":{"type":"text"},"timestamp_ms":{"type":"text"},"user_name":{"type":"text"},"user_screen_name":{"type":"text"}}}}}
try:
es.indices.create(index='tweet', body=mappings)
except:
pass
error = 0
with open("tweets_coord.txt", "r") as f:
class HkdoctorsPipeline(object):
es_host = os.environ['ES']
db_table = os.environ['HKTABLE']
region = os.environ['REGION']
boto3Session = boto3.Session()
credentials = boto3Session.get_credentials()
awsauth = AWS4Auth(
credentials.access_key,
credentials.secret_key,
'us-east-1',
'es',
session_token=credentials.token)
es = connections.create_connection(
hosts= es_host,
port = 443,
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection
)
count = 0
def open_spider(self, spider):
self.dynamodb = boto3.client('dynamodb', region_name='us-east-1')
def process_item(self, item, spider):
self.spider2db(item, self.db_table)
self.spider2es(item)
self.count+=1
return item
def spider2db(self,item, table):
Itm={}
for key in item:
Itm[key]= {}
Itm[key]['S'] = item[key]
# Table="test"
self.dynamodb.put_item(TableName=table, Item=Itm)
def spider2es(self,item):
Itm={}
for key in item:
Itm[key]= item[key]
class Hkdoctor(Document):
name = Text(fields={'raw': Keyword()})
medical_procedures_and_operations = Text(fields={'raw': Keyword()})
url = Text(fields={'raw': Keyword()})
gender = Text(fields={'raw': Keyword()})
qualifications = Text(fields={'raw': Keyword()})
specialty= Text(fields={'raw': Keyword()})
registered_address= Text(fields={'raw': Keyword()})
district= Text(fields={'raw': Keyword()})
type_of_practice= Text(fields={'raw': Keyword()})
consultation_hours= Text(fields={'raw': Keyword()})
emergency_service_available= Text(fields={'raw': Keyword()})
consultation_fee = Text(fields={'raw': Keyword()})
languages_or_dialects_spoken = Text(fields={'raw': Keyword()})
medical_services_available = Text(fields={'raw': Keyword()})
medical_services_provided_other_than_in_the_office = Text(fields={'raw': Keyword()})
affiliated_hospitals = Text(fields={'raw': Keyword()})
telephone = Text(fields={'raw': Keyword()})
fax = Text(fields={'raw': Keyword()})
pager = Text(fields={'raw': Keyword()})
mobile_phone = Text(fields={'raw': Keyword()})
email = Text(fields={'raw': Keyword()})
create_time = Date()
class Index:
name = 'hkdoctor-index'
Hkdoctor.init()
hkdoctor = Hkdoctor(meta={'id': self.count}, Body = Itm)
hkdoctor.save()
def get_client(**kwargs):
"""
NOTE: AWS IAM parameters `aws_key`, `aws_secret_key`, and `aws_region` are
provided for future compatibility, should AWS ES support the
``/_cluster/state/metadata`` endpoint. So long as this endpoint does not
function in AWS ES, the client will not be able to use
:class:`curator.indexlist.IndexList`, which is the backbone of Curator 4
Return an :class:`elasticsearch.Elasticsearch` client object using the
provided parameters. Any of the keyword arguments the
:class:`elasticsearch.Elasticsearch` client object can receive are valid,
such as:
:arg hosts: A list of one or more Elasticsearch client hostnames or IP
addresses to connect to. Can send a single host.
:type hosts: list
:arg port: The Elasticsearch client port to connect to.
:type port: int
:arg url_prefix: `Optional` url prefix, if needed to reach the Elasticsearch
API (i.e., it's not at the root level)
:type url_prefix: str
:arg use_ssl: Whether to connect to the client via SSL/TLS
:type use_ssl: bool
:arg certificate: Path to SSL/TLS certificate
:arg client_cert: Path to SSL/TLS client certificate (public key)
:arg client_key: Path to SSL/TLS private key
:arg aws_key: AWS IAM Access Key (Only used if the :mod:`requests-aws4auth`
python module is installed)
:arg aws_secret_key: AWS IAM Secret Access Key (Only used if the
:mod:`requests-aws4auth` python module is installed)
:arg aws_region: AWS Region (Only used if the :mod:`requests-aws4auth`
python module is installed)
:arg ssl_no_validate: If `True`, do not validate the certificate
chain. This is an insecure option and you will see warnings in the
log output.
:type ssl_no_validate: bool
:arg http_auth: Authentication credentials in `user:pass` format.
:type http_auth: str
:arg timeout: Number of seconds before the client will timeout.
:type timeout: int
:arg master_only: If `True`, the client will `only` connect if the
endpoint is the elected master node of the cluster. **This option does
not work if `hosts` has more than one value.** It will raise an
Exception in that case.
:type master_only: bool
:rtype: :class:`elasticsearch.Elasticsearch`
"""
if 'url_prefix' in kwargs:
if (type(kwargs['url_prefix']) == type(None)
or kwargs['url_prefix'] == "None"):
kwargs['url_prefix'] = ''
if 'host' in kwargs and 'hosts' in kwargs:
raise ConfigurationError(
'Both "host" and "hosts" are defined. Pick only one.')
elif 'host' in kwargs and not 'hosts' in kwargs:
kwargs['hosts'] = kwargs['host']
del kwargs['host']
kwargs['hosts'] = '127.0.0.1' if not 'hosts' in kwargs else kwargs['hosts']
kwargs['master_only'] = False if not 'master_only' in kwargs \
else kwargs['master_only']
kwargs['use_ssl'] = False if not 'use_ssl' in kwargs else kwargs['use_ssl']
kwargs['ssl_no_validate'] = False if not 'ssl_no_validate' in kwargs \
else kwargs['ssl_no_validate']
kwargs['certificate'] = False if not 'certificate' in kwargs \
else kwargs['certificate']
kwargs['client_cert'] = False if not 'client_cert' in kwargs \
else kwargs['client_cert']
kwargs['client_key'] = False if not 'client_key' in kwargs \
else kwargs['client_key']
kwargs['hosts'] = ensure_list(kwargs['hosts'])
logger.debug("kwargs = {0}".format(kwargs))
master_only = kwargs.pop('master_only')
if kwargs['use_ssl']:
if kwargs['ssl_no_validate']:
kwargs[
'verify_certs'] = False # Not needed, but explicitly defined
else:
logger.debug('Attempting to verify SSL certificate.')
# If user provides a certificate:
if kwargs['certificate']:
kwargs['verify_certs'] = True
kwargs['ca_certs'] = kwargs['certificate']
else: # Try to use bundled certifi certificates
if getattr(sys, 'frozen', False):
# The application is frozen (compiled)
datadir = os.path.dirname(sys.executable)
kwargs['verify_certs'] = True
kwargs['ca_certs'] = os.path.join(datadir, 'cacert.pem')
else:
# Use certifi certificates via certifi.where():
import certifi
kwargs['verify_certs'] = True
kwargs['ca_certs'] = certifi.where()
try:
from requests_aws4auth import AWS4Auth
kwargs['aws_key'] = False if not 'aws_key' in kwargs \
else kwargs['aws_key']
kwargs['aws_secret_key'] = False if not 'aws_secret_key' in kwargs \
else kwargs['aws_secret_key']
kwargs['aws_region'] = False if not 'aws_region' in kwargs \
else kwargs['aws_region']
if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs[
'aws_region']:
if not kwargs['aws_key'] and kwargs['aws_secret_key'] \
and kwargs['aws_region']:
raise MissingArgument(
'Missing one or more of "aws_key", "aws_secret_key", '
'or "aws_region".')
# Override these kwargs
kwargs['use_ssl'] = True
kwargs['verify_certs'] = True
kwargs['connection_class'] = elasticsearch.RequestsHttpConnection
kwargs['http_auth'] = (AWS4Auth(kwargs['aws_key'],
kwargs['aws_secret_key'],
kwargs['aws_region'], 'es'))
else:
logger.debug('"requests_aws4auth" module present, but not used.')
except ImportError:
logger.debug('Not using "requests_aws4auth" python module to connect.')
if master_only:
if len(kwargs['hosts']) > 1:
raise ConfigurationError(
'"master_only" cannot be True if more than one host is '
'specified. Hosts = {0}'.format(kwargs['hosts']))
try:
client = elasticsearch.Elasticsearch(**kwargs)
# Verify the version is acceptable.
check_version(client)
# Verify "master_only" status, if applicable
check_master(client, master_only=master_only)
return client
except Exception as e:
raise elasticsearch.ElasticsearchException(
'Unable to create client connection to Elasticsearch. '
'Error: {0}'.format(e))
import pandas as pd
import requests
from requests_aws4auth import AWS4Auth
import boto3
import matplotlib.pyplot as plot
import seaborn
url = 'https://my.elasticsearch.domain/my_awesome_index/_search'
service = 'es'
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, AWS_REGION, service, session_token=credentials.token)
def search(query):
# executes the query over the search url
headers = {"Content-Type": "application/json"}
r = requests.get(url, auth=awsauth, headers=headers, data=json.dumps(query))
return json.loads(r.text)
def parseData(esResponse):
# unpack values from elastic search response object if any
dataArr = []
rows = esRes['hits']['hits']
for row in rows:
data = row['_source']
data['id'] = row['id']
dataArr.append(row['_source'])
return dataAtt
import os
import sys
import uuid
import time
import requests
from requests_aws4auth import AWS4Auth
from requests.auth import HTTPBasicAuth
REGION = 'us-east-1'
service = "es"
credentials = boto3.Session().get_credentials()
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
awsauth = AWS4Auth(access_key,
secret_key,
REGION,
service,
session_token=credentials.token)
def lambda_handler(event, context):
# recieve from API Gateway
print("EVENT --- {}".format(json.dumps(event)))
headers = {"Content-Type": "application/json"}
lex = boto3.client('lex-runtime')
query = event["queryStringParameters"]["q"]
print("query --- {}".format(query))
lex_response = lex.post_text(botName='photolex',
import googlemaps
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
from config import GMAP_KEY, AWS_ES_HOST, AWS_ES_PORT, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION
awsauth = AWS4Auth(AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, 'es')
es = Elasticsearch(hosts=[{
'host': AWS_ES_HOST,
'port': AWS_ES_PORT
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
print(es.info())
# es = Elasticsearch()
gmaps = googlemaps.Client(key=GMAP_KEY)
print("Endpoint: {0}".format(Endpoint))
url = 'http://{0}:9200'.format(Endpoint)
print 'url: {0}'.format(url)
'''
res = requests.get(url)
print(res.content)
'''
aws_access_key_id = ""
aws_secret_access_key = ""
REGION = response['DomainStatus']['ARN'].split(':')[3]
print("REGION: {}".format(REGION))
host = Endpoint
awsauth = AWS4Auth(aws_access_key_id, aws_secret_access_key, REGION, 'es')
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
print("es.info: {0}".format(es.info()))
#es.indices.refresh(index="test-index")
res = es.search(index="test-index", body={"query": {"match_all": {}}})
pprint.pprint("res = {0}".format(res))
print("Got %d Hits:" % res['hits']['total'])
def lambda_handler(event, context):
# TODO implement
#print(event)
name = event['Records'][0]['s3']['object']['key']
#name = "bus.jpg"
#================ rekognition - acquire labels ====================
client = boto3.client('rekognition')
response = client.detect_labels(
Image={'S3Object': {
'Bucket': 'hw3-photo-bucket',
'Name': name
}},
MaxLabels=5)
labels = []
for value in response['Labels']:
labels.append(value['Name'].lower())
print(labels)
#================ lex - add values in the slot (photoType) ====================
client = boto3.client('lex-models')
# get checksum (params used later)
response = client.get_slot_type(
name='photoType',
version='$LATEST',
)
checksum = response["checksum"]
# read current slot type values, then update
enumerationValues = response[
'enumerationValues'] # [{'value': 'mammal'}, {'value': 'pillow'}, {'value': 'canine'}, {'value': 'dog'}, {'value': 'pet'}]
existed_values = set(d['value'] for d in enumerationValues)
new_values = set(labels)
values = existed_values | new_values
enumerationValues = [{'value': v} for v in values]
print(enumerationValues)
# update(replace) slot type values
response = client.put_slot_type(
name='photoType',
enumerationValues=enumerationValues,
checksum=checksum,
)
#================ elasticSearch - insert photo labels ====================
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
host = 'vpc-photos-3pl2e2gkp23t52sjzv3rove4ru.us-east-1.es.amazonaws.com' # For example, my-test-domain.us-east-1.es.amazonaws.com
region = 'us-east-1' # e.g. us-west-1
service = 'es'
#credentials = boto3.Session().get_credentials()
#print(credentials)
awsauth = AWS4Auth('AKIA2OMBG3RF7CNI3CQS',
'omAp0iAinwmBPyiYh8yByqE0FW/0hIw3hil92D4n',
region,
service,
session_token=None)
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
document = {}
document['objectKey'] = name
document['bucket'] = 'hw3-photo-bucket'
from datetime import datetime
now = datetime.now()
timestamp = datetime.timestamp(now)
document['timestamp'] = timestamp
document['labels'] = labels
es.index(index="photos", doc_type="Photo", body=document)
res = es.search(index="photos",
doc_type="Photo",
body={"query": {
"match": {
"bucket": 'hw3-photo-bucket'
}
}})
print('---')
print(res)
return {'statusCode': 200, 'body': json.dumps('Hello from Lambda!')}
# -*- coding: utf-8 -*-
import sys
from os import environ
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
# hack to deal with encoding issue
reload(sys)
sys.setdefaultencoding('UTF8')
host = environ['es_host']
awsauth = AWS4Auth(environ['aws_key'], environ['aws_secret'],
environ['aws_region'], 'es')
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
if __name__ == '__main__':
# define structure
request_body = {
'mappings': {
'tweet': {
'properties': {
'user_name': {
#!/usr/bin/python
from __future__ import print_function
from requests_aws4auth import AWS4Auth
import requests
import datetime
import time
import sys
import boto3
from pprint import pprint
credentials = boto3.Session().get_credentials()
awsauth = AWS4Auth('xxxxxxxxxxxxxxxxx', 'yyyyyyyyyyyyyyyyyyyy', 'us-east-1',
'es')
########## MAIN CODE ##########
print("-------------------")
print("Starting program...\n")
def get_indices():
##### Variable Declarations
esEndpoint = "http://" + sys.argv[1] + "/"
indicesList = []
creationTimes = []
removeElements = []
temp_list = []
# get the list of all available indices.
indices = requests.get(esEndpoint + "_cat/indices", auth=awsauth)
import boto3
from elasticsearch import Elasticsearch, RequestsHttpConnection
from requests_aws4auth import AWS4Auth
cred = boto3.session.Session().get_credentials()
host = ''
awsauth = AWS4Auth(cred.access_key,
cred.secret_key,
'us-east-2',
'es',
session_token=cred.token)
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
print(es.info())
es.indices.delete(index='tweet-index', ignore=[400, 404])
def lambda_handler(event, context):
url = 'https://%s/_bulk' % os.environ['ES_ENDPOINT']
region = os.environ['REGION']
access_key = boto3.client('kms').decrypt(
CiphertextBlob=b64decode(os.environ['ACCESS_KEY']))['Plaintext']
secret_key = boto3.client('kms').decrypt(
CiphertextBlob=b64decode(os.environ['SECRET_KEY']))['Plaintext']
decoded = b64decode(event['awslogs']['data'])
log = json.loads(
zlib.decompress(b64decode(event['awslogs']['data']),
16 + zlib.MAX_WBITS))
instance_id = log['logStream']
aws_account = log['owner']
auth = AWS4Auth(
access_key,
secret_key,
region,
'es',
)
try:
ec2_resource = boto3.resource('ec2')
tags = ec2_resource.Instance(instance_id).tags
except Exception as e:
logger.warning('Unable to get tags for instance: %s' % e)
tags = []
logger.info('%s log entries in this event' % len(log['logEvents']))
session = requests.Session()
logchunk = list(
log['logEvents'][i:i + logs_per_request]
for i in xrange(0, len(log['logEvents']), logs_per_request))
for chunk in logchunk:
bulk_txt = ''
for item in chunk:
timestamp = datetime.datetime.fromtimestamp(
item['timestamp'] / 1000).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
event_date = datetime.datetime.fromtimestamp(
item['timestamp'] /
1000).strftime('%Y-%m-%dT%H:%M:%SZ').split('T')[0].replace(
'-', '.')
index_dict = {
'index': {
'_index': 'logstash-%s' % event_date,
'_type': 'iislog'
}
}
index_str = '%s\n' % json.dumps(index_dict)
fields = item['message'].split()
if not fields:
logger.warning('%s did not match Iis EventLog Pattern: %s' %
item['message'])
logger.warning('Skipping this message')
continue
if fields[0].startswith('#'):
logger.info('Skipping comment line')
continue
try:
msg_format = {
's-ip': fields[0],
'cs-method': fields[1],
'cs-uri-stem': fields[2],
'cs-uri-query': fields[3],
's-port': int(fields[4]),
'cs-username': fields[5],
'c-ip': fields[6],
'cs(User-Agent)': fields[7],
'cs(Referer)': fields[8],
'sc-status': int(fields[9]),
'sc-substatus': fields[10],
'sc-win32-status': fields[11],
'time-taken': float(fields[12]) / 1000,
'@timestamp': timestamp,
'instance-id': instance_id,
'aws-account': aws_account,
'type': 'iislog'
}
for tag in tags:
msg_format['tag.%s' % tag['Key']] = tag['Value']
except (IndexError, ValueError):
logger.warning('Invalid Message Format: %s' % item['message'])
logger.warning('Skipping this message')
else:
bulk_txt += index_str
bulk_txt += json.dumps(msg_format)
bulk_txt += '\n'
amz_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
headers = {
'Content-Type': 'application/x-ndjson',
'X-Amz-Date': amz_date
}
response = session.post(url, data=bulk_txt, headers=headers, auth=auth)
try:
response.raise_for_status()
except Exception as e:
logger.warning('Error Reponse Code Received: %s',
response.status_code)
logger.warning('Text: %s', response.content)
logger.warning('This message was not inserted into elasticsearch')
else:
logger.info('Log Event sent to Elastic Search')
session.close()
DATE_TYPE_FIELDS = [
e for e in os.getenv('DATE_TYPE_FIELDS', '').split(',') if e
]
DATE_FORMAT = os.getenv('DATE_FORMAT', '%Y-%m-%d %H:%M:%S')
AWS_REGION = os.getenv('REGION_NAME', 'us-east-1')
session = boto3.Session(region_name=AWS_REGION)
credentials = session.get_credentials()
credentials = credentials.get_frozen_credentials()
access_key, secret_key, token = (credentials.access_key,
credentials.secret_key, credentials.token)
aws_auth = AWS4Auth(access_key,
secret_key,
AWS_REGION,
'es',
session_token=token)
es_client = Elasticsearch(hosts=[{
'host': ES_HOST,
'port': 443
}],
http_auth=aws_auth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
print('[INFO] ElasticSearch Service',
json.dumps(es_client.info(), indent=2),
file=sys.stderr)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static/')
ES_VARS = {
'INDEX': 'organizations',
'INDEX_TEMPLATE': os.path.join(BASE_DIR, 'rorapi', 'index_template.json'),
'BATCH_SIZE': 20,
'MAX_PAGE': 500, # = <ES LIMIT 10000> / BATCH_SIZE
'BULK_SIZE': 500
}
# use AWS4Auth for AWS Elasticsearch unless running locally via docker
if os.environ.get('ELASTIC_HOST', 'elasticsearch') != 'elasticsearch':
http_auth = AWS4Auth(os.environ.get('AWS_ACCESS_KEY_ID'),
os.environ.get('AWS_SECRET_ACCESS_KEY'),
os.environ.get('AWS_REGION'), 'es')
else:
http_auth = ('elastic', os.environ.get('ELASTIC_PASSWORD', 'changeme'))
ES = Elasticsearch([{
'host': os.environ.get('ELASTIC_HOST', 'elasticsearch'),
'port': int(os.environ.get('ELASTIC_PORT', '9200'))
}],
http_auth=http_auth,
use_ssl=False,
timeout=60,
connection_class=RequestsHttpConnection)
# GRID = {
# 'VERSION': '2018-11-14',
from requests_aws4auth import AWS4Auth
import json
import boto3
import os
credentials = boto3.Session(
aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get(
'AWS_SECRET_ACCESS_KEY')).get_credentials()
region = 'us-east-1'
service = 'es'
awsauth = AWS4Auth(credentials.access_key,
credentials.secret_key,
region,
service,
session_token=credentials.token)
host = 'search-yelp-restaurants-pi5uu6x2lzpdmtdrtbchj4wkju.us-east-1.es.amazonaws.com'
print(awsauth)
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
import boto3
import requests
from requests_aws4auth import AWS4Auth
session = requests.Session()
credentials = boto3.session.Session().get_credentials()
print(credentials.access_key)
print(credentials.secret_key)
print(credentials.token)
print(boto3.session.Session().region_name)
session.auth = AWS4Auth(
credentials.access_key,
credentials.secret_key,
boto3.session.Session().region_name,
'appsync',
session_token=credentials.token
)
API_URL = os.environ.get('API_URL')
query = """
query MyQuery {
searchBlogs(input: {word: "Cognito", fuzziness: 0}) {
id
}
}
"""
def lambda_handler(event, context):
