def report(tweet_id):
if hook.is_alive():
thread_status = hook.status()
json_response = flask.jsonify(thread_status)
status = CONSTANTS.get('HTTP_STATUS').get('200_OK')
if tweet_id is not None:
count = thread_status.get('count')
elapsed_time = thread_status.get('elapsed_time')
separator = ', '
topics_list = separator.join(topics)
twitter_client.request(
'statuses/update',
{
'status': f'{count} tweets processed '
f'during {elapsed_time} '
f'keywords: {topics_list}',
'in_reply_to_status_id': tweet_id,
},
)
else:
json_response = flask.jsonify({'error': 'Service not started'})
status = CONSTANTS.get('HTTP_STATUS').get('404_NOT_FOUND')
return flask.make_response(
json_response,
status,
)
def stop():
if hook.is_alive():
hook.join()
json_response = flask.jsonify({'info': 'Service has stopped'})
return flask.make_response(
json_response,
CONSTANTS.get('HTTP_STATUS').get('200_OK'),
)
def index():
if hook.is_alive():
latest_tweet = twitter_hook.TwitterDataStream.latest_tweet
json_response = flask.jsonify({
'name': latest_tweet.get('user').get('screen_name'),
'time': dateutil.parser.parse(
latest_tweet.get('created_at')
).strftime("%B %d, %Y – %H:%M %z"),
'tweet': latest_tweet,
})
status = CONSTANTS.get('HTTP_STATUS').get('200_OK')
else:
json_response = flask.jsonify({'error': 'Service not started'})
status = CONSTANTS.get('HTTP_STATUS').get('404_NOT_FOUND')
return flask.make_response(
json_response,
status,
)
def start():
global hook
if not hook.is_alive():
hook = twitter_hook.TwitterDataStream(
kinesis,
twitter_client,
logger=app.logger,
topics=topics
)
hook.start()
json_response = flask.jsonify({'info': 'Service has started'})
return flask.make_response(
json_response,
CONSTANTS.get('HTTP_STATUS').get('200_OK'),
)
def page_not_found(error):
json_response = flask.jsonify({'error': 'Page not found'})
return flask.make_response(
json_response,
CONSTANTS.get('HTTP_STATUS').get('404_NOT_FOUND'),
)
}},
'handlers': {'wsgi': {
'class': 'logging.StreamHandler',
'stream': 'ext://flask.logging.wsgi_errors_stream',
'formatter': 'default'
}},
'root': {
'level': 'INFO',
'handlers': ['wsgi']
}
})
app = flask.Flask(__name__)
kinesis = boto3.client(
'kinesis',
aws_access_key_id=CONSTANTS.get('AWS').get('aws_access_key_id'),
aws_secret_access_key=CONSTANTS.get('AWS').get('aws_secret_access_key'),
region_name=CONSTANTS.get('AWS').get('region_name'),
)
twitter_client = twitter.TwitterAPI(
CONSTANTS.get('TWITTER').get('consumer_key'),
CONSTANTS.get('TWITTER').get('consumer_secret'),
CONSTANTS.get('TWITTER').get('access_token_key'),
CONSTANTS.get('TWITTER').get('access_token_secret'),
)
topics = ['trans', 'transgender']
hook = twitter_hook.TwitterDataStream(
kinesis,
twitter_client,
logger=app.logger,
topics=topics
while finish > datetime.datetime.now():
response = kinesis.get_records(ShardIterator=next_iterator,
Limit=25)
records = response['Records']
if records:
self.process_records(records)
next_iterator = response['NextShardIterator']
time.sleep(self.sleep_interval)
if __name__ == '__main__':
kinesis = boto3.client(
'kinesis',
aws_access_key_id=CONSTANTS.get('AWS').get('aws_access_key_id'),
aws_secret_access_key=CONSTANTS.get('AWS').get(
'aws_secret_access_key'),
region_name=CONSTANTS.get('AWS').get('region_name'),
)
dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table('tweets')
shards = kinesis.list_shards(
StreamName=CONSTANTS.get('KINESIS').get('STREAM')).get('Shards')
shard_id = shards[0].get('ShardId')
kinesis_consumer = KinesisConsumer(
CONSTANTS.get('KINESIS').get('STREAM'), shard_id, 'LATEST', kinesis,
table)
kinesis_consumer.start()
import argparse
import boto3
import logging
import os
from botocore.exceptions import ClientError
from constants import CONSTANTS
ACCESS_KEY = CONSTANTS.get('aws').get('aws_access_key_id')
SECRET_KEY = CONSTANTS.get('aws').get('aws_secret_access_key')
def put_object(s3, dest_bucket_name, dest_object_name, src_data):
if isinstance(src_data, str):
try:
object_data = open(src_data, 'rb')
# possible FileNotFoundError/IOError exception
except Exception as e:
logging.error(e)
return False
else:
logging.error(f'Type of {str(type(src_data))} for the argument '
'\'src_data\' is not supported.')
return False
try:
s3.put_object(Bucket=dest_bucket_name,
Key=dest_object_name,
Body=object_data)
except ClientError as e:
logging.error(e)
return False
finally:
