from elasticsearch import Elasticsearch, RequestsHttpConnection import boto3 from requests_aws4auth import AWS4Auth import json region = 'eu-west-1' session = boto3.session.Session(region_name=region) credentials = session.get_credentials() host = 'search-idc-7dyp7fwnklhj22hwzpuoa5hgsy.eu-west-1.es.amazonaws.com' awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, 'es') es = Elasticsearch( hosts=[{'host': host, 'port': 443}], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection ) query = { "query": { "range": { "time": { "gte": "now-5s/s" } } } } result = es.search(index='meir_miran_data', body=query) print(json.dumps(result))
from requests_aws4auth import AWS4Auth from elasticsearch import Elasticsearch, RequestsHttpConnection #AWS ElasticSearch Configuration HOST = 'aws-es-instance-url.es.amazonaws.com' AWS_ACCESS_KEY = 'AWS Access Key' AWS_SECRET_KEY = 'AWS Secret Key' REGION = 'us-east-2' SERVICE = 'es' # Initializing AWS Authentication used to sign our ElasticSearch requests awsAuth = AWS4Auth(AWS_ACCESS_KEY, AWS_SECRET_KEY, REGION, SERVICE) # Configuring AWS ElasticSearch connection es = Elasticsearch(hosts=[{ 'host': HOST, 'port': 443 }], http_auth=awsAuth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) # Index information theIndex = "movies" theDocTypeMapping = "movie"
sys.exit(0) ## SVL: get credentials on datastore awsProfile = "download" credFilePath = '/home/sbp/.aws/credentials' with open(credFilePath) as fp: for line in fp: if re.match("\[" + awsProfile + "\]", line) is not None: accessKeyLine = fp.next().rstrip() secretKeyLine = fp.next().rstrip() accessKey = accessKeyLine.split(' = ')[1] secretKey = secretKeyLine.split(' = ')[1] fp.close() r = requests.head('https://s3.object02.schubergphilis.com/' + sys.argv[1], auth=AWS4Auth(accessKey, secretKey, 'eu-nl-prod01', 's3'), timeout=30) if 'x-amz-restore' in r.headers: parsed = parse_amz_restore(r.headers['x-amz-restore']) if 'expiry-date' in parsed and parsed['expiry-date'] > datetime.now( timezone('Europe/Amsterdam')) + timedelta(days=7): print 'Object already restored till ' + str( parsed['expiry-date']) + ' so not requesting new restore' sys.exit(0) r = requests.post('https://s3.object02.schubergphilis.com/' + sys.argv[1] + '?restore', auth=AWS4Auth(accessKey, secretKey, 'eu-nl-prod01', 's3'), timeout=30,
dynamodb = boto3.resource('dynamodb') yelp_restaurants = dynamodb.Table('yelp-restaurants') response = yelp_restaurants.scan() data = response['Items'] while response.get('LastEvaluatedKey'): response = yelp_restaurants.scan(ExclusiveStartKey=response['LastEvaluatedKey']) data.extend(response['Items']) print(len(data)) session = boto3.session.Session() credentials = session.get_credentials() service = 'es' awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, session.region_name, service) endpoint = "https://search-restaurants-ccwjaawlwnwrn73n7v5j3rf2sm.us-east-1.es.amazonaws.com" es = Elasticsearch( endpoint, http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection ) print(es.info()) count = 0 for restaurant in data: restaurantID = restaurant['id'] cuisine = restaurant['cuisine']
from __future__ import print_function from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth from certificate.my_AWS_key import * import json # ES Client REGION = "us-east-1" host = 'search-tweet-kkj4pr2lzt2sfl3jzxq2a45a7e.us-east-1.es.amazonaws.com' INDEX = 1 awsauth = AWS4Auth(ACCESS_KEY, SECRET_KEY, REGION, 'es') es = Elasticsearch(hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) def indextoES(event, context): msg = event['Records'][0]['Sns']['Message'] print("From SNS: " + msg) message = { "text": event['Records'][0]['Sns']['Message']['text'], "lat": event['Records'][0]['Sns']['Message']['lat'], "lng": event['Records'][0]['Sns']['Message']['lng'], "sentiment": event['Records'][0]['Sns']['Message']['sentiment'] }
import json from datetime import datetime import os from requests_aws4auth import AWS4Auth import boto3 import random as rand #expexting AWS host host = '<insert host>.us-east-1.es.amazonaws.com' region = 'us-east-1' service = 'es' aws_access_key_id = os.environ['aws_access_key_id'], aws_secret_access_key = os.environ['aws_secret_access_key'] credentials = boto3.Session().get_credentials() awsauth = AWS4Auth(aws_access_key_id, aws_secret_access_key, region, service) # elasticsearch credentials port = 443 es = Elasticsearch(hosts=[{ "host": host, "port": port }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) # returns the name of the next unprocessed file. def getname(file_uuid):
def __init__(self, configuration): super(AmazonElasticsearchService, self).__init__(configuration) self.auth = AWS4Auth(configuration['access_key'], configuration['secret_key'], configuration['region'], 'es')
def setUp(self): self.nons3auth = AWS4Auth('id', 'secret', 'us-east-1', 'es') self.s3auth = AWS4Auth('id', 'secret', 'us-east-1', 's3')
def setUp(self): self.auth = AWS4Auth('id', 'secret', 'us-east-1', 'es')
def test_raise_invalid_date_default(self): auth = AWS4Auth('access_id', 'secret_key', 'region', 'service') self.assertFalse(auth.raise_invalid_date)
def test_default_include_hdrs(self): auth = AWS4Auth('access_id', 'secret_key', 'region', 'service') check_set = {'host', 'content-type', 'date', 'x-amz-*'} self.assertSetEqual(set(auth.include_hdrs), check_set)
import json import csv import boto3 from elasticsearch import Elasticsearch, RequestsHttpConnection from elasticsearch.helpers import bulk, streaming_bulk from requests_aws4auth import AWS4Auth ES_URL = "vpc-movies-3br6jppkeulox64eyaohgzzl7u.ap-south-1.es.amazonaws.com" credentials = boto3.Session().get_credentials() awsauth = AWS4Auth( credentials.access_key, credentials.secret_key, 'ap-south-1', 'es', session_token=credentials.token ) es_client = Elasticsearch( hosts=[{'host': ES_URL, 'port': 443}], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection ) def write_into_es(filename): bulk(es_client, generate_es_doc(filename))
def __init__(self): self.env_vars = env_variables() self.stack_resources = stack_resources(self.env_vars) self.auth = AWS4Auth(self.env_vars['ACCESS_KEY'], self.env_vars['SECRET_KEY'], self.env_vars['REGION'], 'execute-api')
def lambda_handler(event, context): sqs = boto3.client('sqs') queue_url = 'https://sqs.us-west-2.amazonaws.com/844243140541/food-bot-queue' # Receive message from SQS queue response = sqs.receive_message( QueueUrl=queue_url, AttributeNames=[ 'SentTimestamp' ], MaxNumberOfMessages=1, MessageAttributeNames=[ 'All' ], VisibilityTimeout=0, WaitTimeSeconds=0 ) if (response and 'Messages' in response): # restaurant original host # host = 'search-es-yelp-555toxbazyu56cgn5sgplxwgqy.us-west-2.es.amazonaws.com' # extra credit new host host = 'search-predictions-lmr33sl6iq3l4on4tmmsmzbf4a.us-west-2.es.amazonaws.com' region = 'us-west-2' service = 'es' credentials = boto3.Session().get_credentials() awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service) es = Elasticsearch( hosts = [{'host': host, 'port': 443}], # http_auth = awsauth, use_ssl = True, verify_certs = True, connection_class = RequestsHttpConnection ) dynamodb = boto3.resource('dynamodb', region_name='us-west-2') table = dynamodb.Table('yelp-restaurants') for each_message in response['Messages']: message = each_message receipt_handle = message['ReceiptHandle'] req_attributes = message['MessageAttributes'] res_category = req_attributes['Categories']['StringValue'] # old query # searchData = es.search(index="restaurants", body={ # "query": { # "match": { # "categories.title": res_category # }}}) # new query searchData = es.search(index="predictions", body={ "sort" :[{'score': 'desc'}], "query": { "match": { "cuisine": "American"#res_category }}}) # print (res_category) # print (searchData) print("Got %d Hits:" % searchData['hits']['total']) businessIds = [] for hit in searchData['hits']['hits']: businessIds.append(hit['_source']['id']) # Call the dynemoDB resultData = getDynemoDbData(table, req_attributes, businessIds) print (resultData) print ('req_attributes----', req_attributes) # send the email sendMailToUser(req_attributes, resultData) # Delete received message from queue sqs.delete_message( QueueUrl=queue_url, ReceiptHandle=receipt_handle ) return { 'statusCode': 200, 'body': json.dumps('Hello from Lambda!') }
# -*- coding: utf-8 -*- from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth import os AWS_ACCESS_KEY = os.environ.get('AWS_ACCESS_KEY') AWS_SECRET_KEY = os.environ.get('AWS_SECRET_KEY') region = 'us-east-1' service = 'es' awsauth = AWS4Auth(AWS_ACCESS_KEY, AWS_SECRET_KEY, region, service) host = 'search-django-bdjdxhfakpxgveshtpficlgk7e.us-east-1.es.amazonaws.com' if __name__=="__main__": es = Elasticsearch( hosts = [{'host': host, 'port': 443}], http_auth = awsauth, use_ssl = True, verify_certs = True, connection_class = RequestsHttpConnection ) #es = Elasticsearch([{'host': 'localhost', 'port': 9200}]) #mappings = {"mappings":{"tweet_data":{"properties":{"coordinates":{"type":"geo_point"},"created_at":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"text":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"timestamp_ms":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"user_name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},"user_screen_name":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}}},"settings":{"index":{"number_of_shards":"5","number_of_replicas":"1"}}} mappings = {"mappings":{"tweet_data":{"properties":{"coordinates":{"type":"geo_point"},"created_at":{"type":"text"},"text":{"type":"text"},"timestamp_ms":{"type":"text"},"user_name":{"type":"text"},"user_screen_name":{"type":"text"}}}}} try: es.indices.create(index='tweet', body=mappings) except: pass error = 0 with open("tweets_coord.txt", "r") as f:
class HkdoctorsPipeline(object): es_host = os.environ['ES'] db_table = os.environ['HKTABLE'] region = os.environ['REGION'] boto3Session = boto3.Session() credentials = boto3Session.get_credentials() awsauth = AWS4Auth( credentials.access_key, credentials.secret_key, 'us-east-1', 'es', session_token=credentials.token) es = connections.create_connection( hosts= es_host, port = 443, http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection ) count = 0 def open_spider(self, spider): self.dynamodb = boto3.client('dynamodb', region_name='us-east-1') def process_item(self, item, spider): self.spider2db(item, self.db_table) self.spider2es(item) self.count+=1 return item def spider2db(self,item, table): Itm={} for key in item: Itm[key]= {} Itm[key]['S'] = item[key] # Table="test" self.dynamodb.put_item(TableName=table, Item=Itm) def spider2es(self,item): Itm={} for key in item: Itm[key]= item[key] class Hkdoctor(Document): name = Text(fields={'raw': Keyword()}) medical_procedures_and_operations = Text(fields={'raw': Keyword()}) url = Text(fields={'raw': Keyword()}) gender = Text(fields={'raw': Keyword()}) qualifications = Text(fields={'raw': Keyword()}) specialty= Text(fields={'raw': Keyword()}) registered_address= Text(fields={'raw': Keyword()}) district= Text(fields={'raw': Keyword()}) type_of_practice= Text(fields={'raw': Keyword()}) consultation_hours= Text(fields={'raw': Keyword()}) emergency_service_available= Text(fields={'raw': Keyword()}) consultation_fee = Text(fields={'raw': Keyword()}) languages_or_dialects_spoken = Text(fields={'raw': Keyword()}) medical_services_available = Text(fields={'raw': Keyword()}) medical_services_provided_other_than_in_the_office = Text(fields={'raw': Keyword()}) affiliated_hospitals = Text(fields={'raw': Keyword()}) telephone = Text(fields={'raw': Keyword()}) fax = Text(fields={'raw': Keyword()}) pager = Text(fields={'raw': Keyword()}) mobile_phone = Text(fields={'raw': Keyword()}) email = Text(fields={'raw': Keyword()}) create_time = Date() class Index: name = 'hkdoctor-index' Hkdoctor.init() hkdoctor = Hkdoctor(meta={'id': self.count}, Body = Itm) hkdoctor.save()
def get_client(**kwargs): """ NOTE: AWS IAM parameters `aws_key`, `aws_secret_key`, and `aws_region` are provided for future compatibility, should AWS ES support the ``/_cluster/state/metadata`` endpoint. So long as this endpoint does not function in AWS ES, the client will not be able to use :class:`curator.indexlist.IndexList`, which is the backbone of Curator 4 Return an :class:`elasticsearch.Elasticsearch` client object using the provided parameters. Any of the keyword arguments the :class:`elasticsearch.Elasticsearch` client object can receive are valid, such as: :arg hosts: A list of one or more Elasticsearch client hostnames or IP addresses to connect to. Can send a single host. :type hosts: list :arg port: The Elasticsearch client port to connect to. :type port: int :arg url_prefix: `Optional` url prefix, if needed to reach the Elasticsearch API (i.e., it's not at the root level) :type url_prefix: str :arg use_ssl: Whether to connect to the client via SSL/TLS :type use_ssl: bool :arg certificate: Path to SSL/TLS certificate :arg client_cert: Path to SSL/TLS client certificate (public key) :arg client_key: Path to SSL/TLS private key :arg aws_key: AWS IAM Access Key (Only used if the :mod:`requests-aws4auth` python module is installed) :arg aws_secret_key: AWS IAM Secret Access Key (Only used if the :mod:`requests-aws4auth` python module is installed) :arg aws_region: AWS Region (Only used if the :mod:`requests-aws4auth` python module is installed) :arg ssl_no_validate: If `True`, do not validate the certificate chain. This is an insecure option and you will see warnings in the log output. :type ssl_no_validate: bool :arg http_auth: Authentication credentials in `user:pass` format. :type http_auth: str :arg timeout: Number of seconds before the client will timeout. :type timeout: int :arg master_only: If `True`, the client will `only` connect if the endpoint is the elected master node of the cluster. **This option does not work if `hosts` has more than one value.** It will raise an Exception in that case. :type master_only: bool :rtype: :class:`elasticsearch.Elasticsearch` """ if 'url_prefix' in kwargs: if (type(kwargs['url_prefix']) == type(None) or kwargs['url_prefix'] == "None"): kwargs['url_prefix'] = '' if 'host' in kwargs and 'hosts' in kwargs: raise ConfigurationError( 'Both "host" and "hosts" are defined. Pick only one.') elif 'host' in kwargs and not 'hosts' in kwargs: kwargs['hosts'] = kwargs['host'] del kwargs['host'] kwargs['hosts'] = '127.0.0.1' if not 'hosts' in kwargs else kwargs['hosts'] kwargs['master_only'] = False if not 'master_only' in kwargs \ else kwargs['master_only'] kwargs['use_ssl'] = False if not 'use_ssl' in kwargs else kwargs['use_ssl'] kwargs['ssl_no_validate'] = False if not 'ssl_no_validate' in kwargs \ else kwargs['ssl_no_validate'] kwargs['certificate'] = False if not 'certificate' in kwargs \ else kwargs['certificate'] kwargs['client_cert'] = False if not 'client_cert' in kwargs \ else kwargs['client_cert'] kwargs['client_key'] = False if not 'client_key' in kwargs \ else kwargs['client_key'] kwargs['hosts'] = ensure_list(kwargs['hosts']) logger.debug("kwargs = {0}".format(kwargs)) master_only = kwargs.pop('master_only') if kwargs['use_ssl']: if kwargs['ssl_no_validate']: kwargs[ 'verify_certs'] = False # Not needed, but explicitly defined else: logger.debug('Attempting to verify SSL certificate.') # If user provides a certificate: if kwargs['certificate']: kwargs['verify_certs'] = True kwargs['ca_certs'] = kwargs['certificate'] else: # Try to use bundled certifi certificates if getattr(sys, 'frozen', False): # The application is frozen (compiled) datadir = os.path.dirname(sys.executable) kwargs['verify_certs'] = True kwargs['ca_certs'] = os.path.join(datadir, 'cacert.pem') else: # Use certifi certificates via certifi.where(): import certifi kwargs['verify_certs'] = True kwargs['ca_certs'] = certifi.where() try: from requests_aws4auth import AWS4Auth kwargs['aws_key'] = False if not 'aws_key' in kwargs \ else kwargs['aws_key'] kwargs['aws_secret_key'] = False if not 'aws_secret_key' in kwargs \ else kwargs['aws_secret_key'] kwargs['aws_region'] = False if not 'aws_region' in kwargs \ else kwargs['aws_region'] if kwargs['aws_key'] or kwargs['aws_secret_key'] or kwargs[ 'aws_region']: if not kwargs['aws_key'] and kwargs['aws_secret_key'] \ and kwargs['aws_region']: raise MissingArgument( 'Missing one or more of "aws_key", "aws_secret_key", ' 'or "aws_region".') # Override these kwargs kwargs['use_ssl'] = True kwargs['verify_certs'] = True kwargs['connection_class'] = elasticsearch.RequestsHttpConnection kwargs['http_auth'] = (AWS4Auth(kwargs['aws_key'], kwargs['aws_secret_key'], kwargs['aws_region'], 'es')) else: logger.debug('"requests_aws4auth" module present, but not used.') except ImportError: logger.debug('Not using "requests_aws4auth" python module to connect.') if master_only: if len(kwargs['hosts']) > 1: raise ConfigurationError( '"master_only" cannot be True if more than one host is ' 'specified. Hosts = {0}'.format(kwargs['hosts'])) try: client = elasticsearch.Elasticsearch(**kwargs) # Verify the version is acceptable. check_version(client) # Verify "master_only" status, if applicable check_master(client, master_only=master_only) return client except Exception as e: raise elasticsearch.ElasticsearchException( 'Unable to create client connection to Elasticsearch. ' 'Error: {0}'.format(e))
import pandas as pd import requests from requests_aws4auth import AWS4Auth import boto3 import matplotlib.pyplot as plot import seaborn url = 'https://my.elasticsearch.domain/my_awesome_index/_search' service = 'es' credentials = boto3.Session().get_credentials() awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, AWS_REGION, service, session_token=credentials.token) def search(query): # executes the query over the search url headers = {"Content-Type": "application/json"} r = requests.get(url, auth=awsauth, headers=headers, data=json.dumps(query)) return json.loads(r.text) def parseData(esResponse): # unpack values from elastic search response object if any dataArr = [] rows = esRes['hits']['hits'] for row in rows: data = row['_source'] data['id'] = row['id'] dataArr.append(row['_source']) return dataAtt
import os import sys import uuid import time import requests from requests_aws4auth import AWS4Auth from requests.auth import HTTPBasicAuth REGION = 'us-east-1' service = "es" credentials = boto3.Session().get_credentials() access_key = os.environ.get('AWS_ACCESS_KEY_ID') secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') awsauth = AWS4Auth(access_key, secret_key, REGION, service, session_token=credentials.token) def lambda_handler(event, context): # recieve from API Gateway print("EVENT --- {}".format(json.dumps(event))) headers = {"Content-Type": "application/json"} lex = boto3.client('lex-runtime') query = event["queryStringParameters"]["q"] print("query --- {}".format(query)) lex_response = lex.post_text(botName='photolex',
import googlemaps from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth from config import GMAP_KEY, AWS_ES_HOST, AWS_ES_PORT, AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION awsauth = AWS4Auth(AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, 'es') es = Elasticsearch(hosts=[{ 'host': AWS_ES_HOST, 'port': AWS_ES_PORT }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) print(es.info()) # es = Elasticsearch() gmaps = googlemaps.Client(key=GMAP_KEY)
print("Endpoint: {0}".format(Endpoint)) url = 'http://{0}:9200'.format(Endpoint) print 'url: {0}'.format(url) ''' res = requests.get(url) print(res.content) ''' aws_access_key_id = "" aws_secret_access_key = "" REGION = response['DomainStatus']['ARN'].split(':')[3] print("REGION: {}".format(REGION)) host = Endpoint awsauth = AWS4Auth(aws_access_key_id, aws_secret_access_key, REGION, 'es') es = Elasticsearch(hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) print("es.info: {0}".format(es.info())) #es.indices.refresh(index="test-index") res = es.search(index="test-index", body={"query": {"match_all": {}}}) pprint.pprint("res = {0}".format(res)) print("Got %d Hits:" % res['hits']['total'])
def lambda_handler(event, context): # TODO implement #print(event) name = event['Records'][0]['s3']['object']['key'] #name = "bus.jpg" #================ rekognition - acquire labels ==================== client = boto3.client('rekognition') response = client.detect_labels( Image={'S3Object': { 'Bucket': 'hw3-photo-bucket', 'Name': name }}, MaxLabels=5) labels = [] for value in response['Labels']: labels.append(value['Name'].lower()) print(labels) #================ lex - add values in the slot (photoType) ==================== client = boto3.client('lex-models') # get checksum (params used later) response = client.get_slot_type( name='photoType', version='$LATEST', ) checksum = response["checksum"] # read current slot type values, then update enumerationValues = response[ 'enumerationValues'] # [{'value': 'mammal'}, {'value': 'pillow'}, {'value': 'canine'}, {'value': 'dog'}, {'value': 'pet'}] existed_values = set(d['value'] for d in enumerationValues) new_values = set(labels) values = existed_values | new_values enumerationValues = [{'value': v} for v in values] print(enumerationValues) # update(replace) slot type values response = client.put_slot_type( name='photoType', enumerationValues=enumerationValues, checksum=checksum, ) #================ elasticSearch - insert photo labels ==================== from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth host = 'vpc-photos-3pl2e2gkp23t52sjzv3rove4ru.us-east-1.es.amazonaws.com' # For example, my-test-domain.us-east-1.es.amazonaws.com region = 'us-east-1' # e.g. us-west-1 service = 'es' #credentials = boto3.Session().get_credentials() #print(credentials) awsauth = AWS4Auth('AKIA2OMBG3RF7CNI3CQS', 'omAp0iAinwmBPyiYh8yByqE0FW/0hIw3hil92D4n', region, service, session_token=None) es = Elasticsearch(hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) document = {} document['objectKey'] = name document['bucket'] = 'hw3-photo-bucket' from datetime import datetime now = datetime.now() timestamp = datetime.timestamp(now) document['timestamp'] = timestamp document['labels'] = labels es.index(index="photos", doc_type="Photo", body=document) res = es.search(index="photos", doc_type="Photo", body={"query": { "match": { "bucket": 'hw3-photo-bucket' } }}) print('---') print(res) return {'statusCode': 200, 'body': json.dumps('Hello from Lambda!')}
# -*- coding: utf-8 -*- import sys from os import environ from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth # hack to deal with encoding issue reload(sys) sys.setdefaultencoding('UTF8') host = environ['es_host'] awsauth = AWS4Auth(environ['aws_key'], environ['aws_secret'], environ['aws_region'], 'es') es = Elasticsearch(hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) if __name__ == '__main__': # define structure request_body = { 'mappings': { 'tweet': { 'properties': { 'user_name': {
#!/usr/bin/python from __future__ import print_function from requests_aws4auth import AWS4Auth import requests import datetime import time import sys import boto3 from pprint import pprint credentials = boto3.Session().get_credentials() awsauth = AWS4Auth('xxxxxxxxxxxxxxxxx', 'yyyyyyyyyyyyyyyyyyyy', 'us-east-1', 'es') ########## MAIN CODE ########## print("-------------------") print("Starting program...\n") def get_indices(): ##### Variable Declarations esEndpoint = "http://" + sys.argv[1] + "/" indicesList = [] creationTimes = [] removeElements = [] temp_list = [] # get the list of all available indices. indices = requests.get(esEndpoint + "_cat/indices", auth=awsauth)
import boto3 from elasticsearch import Elasticsearch, RequestsHttpConnection from requests_aws4auth import AWS4Auth cred = boto3.session.Session().get_credentials() host = '' awsauth = AWS4Auth(cred.access_key, cred.secret_key, 'us-east-2', 'es', session_token=cred.token) es = Elasticsearch(hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) print(es.info()) es.indices.delete(index='tweet-index', ignore=[400, 404])
def lambda_handler(event, context): url = 'https://%s/_bulk' % os.environ['ES_ENDPOINT'] region = os.environ['REGION'] access_key = boto3.client('kms').decrypt( CiphertextBlob=b64decode(os.environ['ACCESS_KEY']))['Plaintext'] secret_key = boto3.client('kms').decrypt( CiphertextBlob=b64decode(os.environ['SECRET_KEY']))['Plaintext'] decoded = b64decode(event['awslogs']['data']) log = json.loads( zlib.decompress(b64decode(event['awslogs']['data']), 16 + zlib.MAX_WBITS)) instance_id = log['logStream'] aws_account = log['owner'] auth = AWS4Auth( access_key, secret_key, region, 'es', ) try: ec2_resource = boto3.resource('ec2') tags = ec2_resource.Instance(instance_id).tags except Exception as e: logger.warning('Unable to get tags for instance: %s' % e) tags = [] logger.info('%s log entries in this event' % len(log['logEvents'])) session = requests.Session() logchunk = list( log['logEvents'][i:i + logs_per_request] for i in xrange(0, len(log['logEvents']), logs_per_request)) for chunk in logchunk: bulk_txt = '' for item in chunk: timestamp = datetime.datetime.fromtimestamp( item['timestamp'] / 1000).strftime('%Y-%m-%dT%H:%M:%S.%fZ') event_date = datetime.datetime.fromtimestamp( item['timestamp'] / 1000).strftime('%Y-%m-%dT%H:%M:%SZ').split('T')[0].replace( '-', '.') index_dict = { 'index': { '_index': 'logstash-%s' % event_date, '_type': 'iislog' } } index_str = '%s\n' % json.dumps(index_dict) fields = item['message'].split() if not fields: logger.warning('%s did not match Iis EventLog Pattern: %s' % item['message']) logger.warning('Skipping this message') continue if fields[0].startswith('#'): logger.info('Skipping comment line') continue try: msg_format = { 's-ip': fields[0], 'cs-method': fields[1], 'cs-uri-stem': fields[2], 'cs-uri-query': fields[3], 's-port': int(fields[4]), 'cs-username': fields[5], 'c-ip': fields[6], 'cs(User-Agent)': fields[7], 'cs(Referer)': fields[8], 'sc-status': int(fields[9]), 'sc-substatus': fields[10], 'sc-win32-status': fields[11], 'time-taken': float(fields[12]) / 1000, '@timestamp': timestamp, 'instance-id': instance_id, 'aws-account': aws_account, 'type': 'iislog' } for tag in tags: msg_format['tag.%s' % tag['Key']] = tag['Value'] except (IndexError, ValueError): logger.warning('Invalid Message Format: %s' % item['message']) logger.warning('Skipping this message') else: bulk_txt += index_str bulk_txt += json.dumps(msg_format) bulk_txt += '\n' amz_date = datetime.datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') headers = { 'Content-Type': 'application/x-ndjson', 'X-Amz-Date': amz_date } response = session.post(url, data=bulk_txt, headers=headers, auth=auth) try: response.raise_for_status() except Exception as e: logger.warning('Error Reponse Code Received: %s', response.status_code) logger.warning('Text: %s', response.content) logger.warning('This message was not inserted into elasticsearch') else: logger.info('Log Event sent to Elastic Search') session.close()
DATE_TYPE_FIELDS = [ e for e in os.getenv('DATE_TYPE_FIELDS', '').split(',') if e ] DATE_FORMAT = os.getenv('DATE_FORMAT', '%Y-%m-%d %H:%M:%S') AWS_REGION = os.getenv('REGION_NAME', 'us-east-1') session = boto3.Session(region_name=AWS_REGION) credentials = session.get_credentials() credentials = credentials.get_frozen_credentials() access_key, secret_key, token = (credentials.access_key, credentials.secret_key, credentials.token) aws_auth = AWS4Auth(access_key, secret_key, AWS_REGION, 'es', session_token=token) es_client = Elasticsearch(hosts=[{ 'host': ES_HOST, 'port': 443 }], http_auth=aws_auth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection) print('[INFO] ElasticSearch Service', json.dumps(es_client.info(), indent=2), file=sys.stderr)
# https://docs.djangoproject.com/en/2.2/howto/static-files/ STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'static/') ES_VARS = { 'INDEX': 'organizations', 'INDEX_TEMPLATE': os.path.join(BASE_DIR, 'rorapi', 'index_template.json'), 'BATCH_SIZE': 20, 'MAX_PAGE': 500, # = <ES LIMIT 10000> / BATCH_SIZE 'BULK_SIZE': 500 } # use AWS4Auth for AWS Elasticsearch unless running locally via docker if os.environ.get('ELASTIC_HOST', 'elasticsearch') != 'elasticsearch': http_auth = AWS4Auth(os.environ.get('AWS_ACCESS_KEY_ID'), os.environ.get('AWS_SECRET_ACCESS_KEY'), os.environ.get('AWS_REGION'), 'es') else: http_auth = ('elastic', os.environ.get('ELASTIC_PASSWORD', 'changeme')) ES = Elasticsearch([{ 'host': os.environ.get('ELASTIC_HOST', 'elasticsearch'), 'port': int(os.environ.get('ELASTIC_PORT', '9200')) }], http_auth=http_auth, use_ssl=False, timeout=60, connection_class=RequestsHttpConnection) # GRID = { # 'VERSION': '2018-11-14',
from requests_aws4auth import AWS4Auth import json import boto3 import os credentials = boto3.Session( aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get( 'AWS_SECRET_ACCESS_KEY')).get_credentials() region = 'us-east-1' service = 'es' awsauth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service, session_token=credentials.token) host = 'search-yelp-restaurants-pi5uu6x2lzpdmtdrtbchj4wkju.us-east-1.es.amazonaws.com' print(awsauth) es = Elasticsearch(hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection)
import boto3 import requests from requests_aws4auth import AWS4Auth session = requests.Session() credentials = boto3.session.Session().get_credentials() print(credentials.access_key) print(credentials.secret_key) print(credentials.token) print(boto3.session.Session().region_name) session.auth = AWS4Auth( credentials.access_key, credentials.secret_key, boto3.session.Session().region_name, 'appsync', session_token=credentials.token ) API_URL = os.environ.get('API_URL') query = """ query MyQuery { searchBlogs(input: {word: "Cognito", fuzziness: 0}) { id } } """ def lambda_handler(event, context):