def report(tweet_id): if hook.is_alive(): thread_status = hook.status() json_response = flask.jsonify(thread_status) status = CONSTANTS.get('HTTP_STATUS').get('200_OK') if tweet_id is not None: count = thread_status.get('count') elapsed_time = thread_status.get('elapsed_time') separator = ', ' topics_list = separator.join(topics) twitter_client.request( 'statuses/update', { 'status': f'{count} tweets processed ' f'during {elapsed_time} ' f'keywords: {topics_list}', 'in_reply_to_status_id': tweet_id, }, ) else: json_response = flask.jsonify({'error': 'Service not started'}) status = CONSTANTS.get('HTTP_STATUS').get('404_NOT_FOUND') return flask.make_response( json_response, status, )
def stop(): if hook.is_alive(): hook.join() json_response = flask.jsonify({'info': 'Service has stopped'}) return flask.make_response( json_response, CONSTANTS.get('HTTP_STATUS').get('200_OK'), )
def index(): if hook.is_alive(): latest_tweet = twitter_hook.TwitterDataStream.latest_tweet json_response = flask.jsonify({ 'name': latest_tweet.get('user').get('screen_name'), 'time': dateutil.parser.parse( latest_tweet.get('created_at') ).strftime("%B %d, %Y – %H:%M %z"), 'tweet': latest_tweet, }) status = CONSTANTS.get('HTTP_STATUS').get('200_OK') else: json_response = flask.jsonify({'error': 'Service not started'}) status = CONSTANTS.get('HTTP_STATUS').get('404_NOT_FOUND') return flask.make_response( json_response, status, )
def start(): global hook if not hook.is_alive(): hook = twitter_hook.TwitterDataStream( kinesis, twitter_client, logger=app.logger, topics=topics ) hook.start() json_response = flask.jsonify({'info': 'Service has started'}) return flask.make_response( json_response, CONSTANTS.get('HTTP_STATUS').get('200_OK'), )
def page_not_found(error): json_response = flask.jsonify({'error': 'Page not found'}) return flask.make_response( json_response, CONSTANTS.get('HTTP_STATUS').get('404_NOT_FOUND'), )
}}, 'handlers': {'wsgi': { 'class': 'logging.StreamHandler', 'stream': 'ext://flask.logging.wsgi_errors_stream', 'formatter': 'default' }}, 'root': { 'level': 'INFO', 'handlers': ['wsgi'] } }) app = flask.Flask(__name__) kinesis = boto3.client( 'kinesis', aws_access_key_id=CONSTANTS.get('AWS').get('aws_access_key_id'), aws_secret_access_key=CONSTANTS.get('AWS').get('aws_secret_access_key'), region_name=CONSTANTS.get('AWS').get('region_name'), ) twitter_client = twitter.TwitterAPI( CONSTANTS.get('TWITTER').get('consumer_key'), CONSTANTS.get('TWITTER').get('consumer_secret'), CONSTANTS.get('TWITTER').get('access_token_key'), CONSTANTS.get('TWITTER').get('access_token_secret'), ) topics = ['trans', 'transgender'] hook = twitter_hook.TwitterDataStream( kinesis, twitter_client, logger=app.logger, topics=topics
while finish > datetime.datetime.now(): response = kinesis.get_records(ShardIterator=next_iterator, Limit=25) records = response['Records'] if records: self.process_records(records) next_iterator = response['NextShardIterator'] time.sleep(self.sleep_interval) if __name__ == '__main__': kinesis = boto3.client( 'kinesis', aws_access_key_id=CONSTANTS.get('AWS').get('aws_access_key_id'), aws_secret_access_key=CONSTANTS.get('AWS').get( 'aws_secret_access_key'), region_name=CONSTANTS.get('AWS').get('region_name'), ) dynamodb = boto3.resource('dynamodb') table = dynamodb.Table('tweets') shards = kinesis.list_shards( StreamName=CONSTANTS.get('KINESIS').get('STREAM')).get('Shards') shard_id = shards[0].get('ShardId') kinesis_consumer = KinesisConsumer( CONSTANTS.get('KINESIS').get('STREAM'), shard_id, 'LATEST', kinesis, table) kinesis_consumer.start()
import argparse import boto3 import logging import os from botocore.exceptions import ClientError from constants import CONSTANTS ACCESS_KEY = CONSTANTS.get('aws').get('aws_access_key_id') SECRET_KEY = CONSTANTS.get('aws').get('aws_secret_access_key') def put_object(s3, dest_bucket_name, dest_object_name, src_data): if isinstance(src_data, str): try: object_data = open(src_data, 'rb') # possible FileNotFoundError/IOError exception except Exception as e: logging.error(e) return False else: logging.error(f'Type of {str(type(src_data))} for the argument ' '\'src_data\' is not supported.') return False try: s3.put_object(Bucket=dest_bucket_name, Key=dest_object_name, Body=object_data) except ClientError as e: logging.error(e) return False finally: