def main():
# Read config file
config = utils.read_config(file_path='./credentials.yml')
# Connect to AWS
session = utils.aws_authorizer(**config)
# Creat firehose client
firehose = session.client('firehose')
# List deliver streams
print(firehose.list_delivery_streams())
# Putting record to stream
# TODO: Change `stream_name` below to your firehose stream name.
stream_name = 'worldcup-tweets'
# TODO: (Optional) Change `data` to any string.
data = 'testing'
response = firehose.put_record(DeliveryStreamName=stream_name,
Record={'Data': data})
print(response)
def lambda_handler(event, context):
print('Lambda trigger begin...')
# Read config file
config = utils.read_config('./credentials.yml')
# Connect to es
# TODO: Change `es_host` to your elasticsearch domain Endpoint
es_host = config['es_host']
es = utils.es_connector(host=es_host, **config)
print(es.info())
# Check es index
if not es.indices.exists(index=es_index):
print('Creating mapping...')
es.indices.create(index=es_index, body={'mappings': mapping})
# Connect to S3 and comprehend
session = utils.aws_authorizer(**config)
s3 = session.client(service_name='s3')
comprehend = session.client(service_name='comprehend')
# Define extractor
extractor = functools.partial(utils.tweet_extractor,
sentiment=utils.get_sentiment,
comprehend=comprehend)
print('Importing tweets to ES...')
for record in event['Records']:
bucket = record['s3']['bucket']['name']
key = record['s3']['object']['key']
print(bucket, key)
total = utils.s3_to_es_uploader(s3=s3,
es=es,
bucket=bucket,
key=key,
extractor=extractor,
upload=True)
print('Total tweets uploaded: {}'.format(total))
print('Lambda trigger end...')
return event
import utils
if __name__ == '__main__':
# Read config file
config = utils.read_config(file_path='./credentials.yml')
# Connect to AWS
session = utils.aws_authorizer(**config)
# Creat firehose client
firehose = session.client('firehose')
# List deliver streams
print(firehose.list_delivery_streams())
# TODO: Change `stream_name` below to your firehose stream name.
stream_name = 'demo-tweet-to-s3'
# TODO: Change `data` to any string or JSON object.
data = 'testing_2'
# Putting record to stream
# response = firehose.put_record(
# DeliveryStreamName=stream_name,
# Record={'Data': data}
# )
# print(response)