def streaming(): global kinesis CONSUMER_KEY = '' CONSUMER_SECRET = '' ACCESS_TOKEN_KEY = '' ACCESS_TOKEN_SECRET = '' consumer = oauth.Consumer(key=CONSUMER_KEY, secret=CONSUMER_SECRET) token = oauth.Token(key=ACCESS_TOKEN_KEY, secret=ACCESS_TOKEN_SECRET) url = 'https://stream.twitter.com/1.1/statuses/sample.json' params = {} request = oauth.Request.from_consumer_and_token(consumer, token, http_url=url, parameters=params) request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), consumer, token) res = urllib2.urlopen(request.to_url()) if res == urllib2.URLError: raise Exception else: for r in res: try: data = json.loads(r) time = str(data['created_at']).split(' ') send_data = {'year':time[5],'month':time[1],'date':time[2],'time':time[3]} send_json = json.dumps(send_data) kinesis.put_record('TwitterStreamTest',send_json,'one') except: continue if __name__ == '__main__': global kinesis kinesis = boto.connect_kinesis(aws_access_key_id='',aws_secret_access_key='') streaming()
def create_kinesis_stream(stream): """ Creates our Kinesis stream """ kinesis = boto.connect_kinesis() response = kinesis.create_stream(stream, 1) pause_until_kinesis_active(stream) print("Kinesis successfully created")
def dumpKinesis(self): ''' Method to dump Kinesis streams info. ''' try: if self.botoprfl[0] != "default": conn = boto.connect_kinesis(profile_name = self.botoprfl) else: conn = boto.connect_kinesis() if conn: print("\n<Start of Kinesis streams>\n") for c in conn.list_streams(): print(" %s" %c.title()) print("\n<End of Kinesis streams>\n") except Exception, e: serr = ('%s :: dumpKinesis(...) : connect_kinesis,list_streams(...), ' '%s' %(self.sclsnme, str(e)))
def dumpKinesis(self): ''' Method to dump Kinesis streams info. ''' try: if self.botoprfl[0] != "default": conn = boto.connect_kinesis(profile_name=self.botoprfl) else: conn = boto.connect_kinesis() if conn: print("\n<Start of Kinesis streams>\n") for c in conn.list_streams(): print(" %s" % c.title()) print("\n<End of Kinesis streams>\n") except Exception, e: serr = ( '%s :: dumpKinesis(...) : connect_kinesis,list_streams(...), ' '%s' % (self.sclsnme, str(e)))
def scan(): '''Scans for public tweets using Tweeter's public stream API with a basic filter. See https://dev.twitter.com/streaming/reference/post/statuses/filter ''' kinesis = boto.connect_kinesis() auth = OAuth(consumer_key=config.CONSUMER_KEY, consumer_secret=config.CONSUMER_SECRET, token=config.ACCESS_TOKEN, token_secret=config.ACCESS_TOKEN_SECRET) stream = TwitterStream(auth=auth) tweets = stream.statuses.filter(track=config.TWITTER_FILTER) for tweet in tweets: payload = prepare(tweet) kinesis.put_record(config.KINESIS_STREAM, payload, config.KINESIS_PARTITION) logger.debug('Sent to Amazon Kinesis.')
def send_tweet_to_stream(self, tweet): conn = boto.connect_kinesis(aws_access_key, aws_access_secret) r = conn.describe_stream(stream_name) description = r.get('StreamDescription') status= description.get('StreamStatus') try: if status == 'DELETING': print 'The stream: {s} is being deleted, please rerun the script.'.format(s=stream_name) #sys.exit(1) elif status == 'ACTIVE': time.sleep(10) except: # We'll assume the stream didn't exist so we will try to create it with just one shard conn.create_stream(stream_name, 1) time.sleep(10) conn.put_record(stream_name, json.dumps(tweet), str(int(time.time())))
def get_kinesis_data_iterator(stream_name, minutes_running): # Connect to Kinesis kinesis = boto.connect_kinesis() # Get data about Kinesis stream for Tag Monitor kinesis_stream = kinesis.describe_stream(stream_name) # Get the shards in that stream shards = kinesis_stream['StreamDescription']['Shards'] # Collect together the shard IDs shard_ids = [shard['ShardId'] for shard in shards] # Get shard iterator iter_response = kinesis.get_shard_iterator(stream_name, shard_ids[0], "TRIM_HORIZON") shard_iterator = iter_response['ShardIterator'] # Calculate end time end_time = datetime.now() + timedelta(minutes=minutes_running) while True: try: # Get data record_response = kinesis.get_records(shard_iterator) # Only run for a certain amount of time. # Stop looping if no data returned. This means it's done now = datetime.now() print 'Time: {0}'.format(now.strftime('%Y/%m/%d %H:%M:%S')) if end_time < now or not record_response: break # yield data to outside calling iterator for record in record_response['Records']: last_sequence = record['SequenceNumber'] yield json.loads(record['Data']) # Get next iterator for shard from previous request shard_iterator = record_response['NextShardIterator'] # Catch exception meaning hitting API too much except boto.kinesis.exceptions.ProvisionedThroughputExceededException: print 'ProvisionedThroughputExceededException found. Sleeping for 0.5 seconds...' time.sleep(0.5) # Catch exception meaning iterator has expired except boto.kinesis.exceptions.ExpiredIteratorException: iter_response = kinesis.get_shard_iterator(stream_name, shard_ids[0], "AFTER_SEQUENCE_NUMBER", last_sequence) shard_iterator = iter_response['ShardIterator'] kinesis.close()
import datetime from argparse import RawTextHelpFormatter from random import choice from string import lowercase from boto.kinesis.exceptions import ResourceNotFoundException # To preclude inclusion of aws keys into this code, you may temporarily add # your AWS credentials to the file: # ~/.boto # as follows: # [Credentials] # aws_access_key_id = <your access key> # aws_secret_access_key = <your secret key> kinesis = boto.connect_kinesis() make_string = lambda x: "".join(choice(lowercase) for i in range(x)) def get_or_create_stream(stream_name, shard_count): stream = None try: stream = kinesis.describe_stream(stream_name) print( json.dumps(stream, sort_keys=True, indent=2, separators=(',', ': '))) except ResourceNotFoundException as rnfe: while (stream is None) or (
import json import os from time import sleep from zipfile import ZipFile import boto from boto.kinesis.exceptions import ResourceInUseException import config # To enable logging: # boto.set_stream_logger('boto') # Initialize the AWS clients. sns = boto.connect_sns() kinesis = boto.connect_kinesis() aws_lambda = boto.connect_awslambda() ml = boto.connect_machinelearning() lambda_execution_policy = open('lambdaExecutionPolicyTemplate.json').read().format(**config.AWS) aws_account_id = config.AWS["awsAccountId"] region = config.AWS["region"] kinesis_stream = config.AWS["kinesisStream"] sns_topic = config.AWS["snsTopic"] lambda_function_name = config.AWS["lambdaFunctionName"] lambda_execution_role = config.AWS["lambdaExecutionRole"] lambda_trust_policy = '{"Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}'
cognito_id = cognito.get_id(ACCOUNT_ID, IDENTITY_POOL_ID) oidc = cognito.get_open_id_token(cognito_id['IdentityId']) # Further setup your STS using the code below sts = boto.connect_sts() assumedRoleObject = sts.assume_role_with_web_identity(ROLE_ARN, "XX", oidc['Token']) # Connect to dynamoDB and kinesis client_dynamo = boto.dynamodb2.connect_to_region( 'us-east-1', aws_access_key_id=assumedRoleObject.credentials.access_key, aws_secret_access_key=assumedRoleObject.credentials.secret_key, security_token=assumedRoleObject.credentials.session_token) client_kinesis = boto.connect_kinesis( aws_access_key_id=assumedRoleObject.credentials.access_key, aws_secret_access_key=assumedRoleObject.credentials.secret_key, security_token=assumedRoleObject.credentials.session_token) from boto.dynamodb2.table import Table from boto.dynamodb2.fields import HashKey ###################### # Setup DynamoDB Table ###################### table_dynamo = # YOUR CODE HERE # ################################################# # Setup switch and temperature sensor # #################################################
# Use cognito to get an identity. cognito = boto.connect_cognito_identity() cognito_id = cognito.get_id(ACCOUNT_ID, IDENTITY_POOL_ID) oidc = cognito.get_open_id_token(cognito_id['IdentityId']) # Further setup your STS using the code below sts = boto.connect_sts() assumedRoleObject = sts.assume_role_with_web_identity(ROLE_ARN, "XX", oidc['Token']) DYNAMODB_TABLE_NAME = 'pingdata' # Prepare DynamoDB client client_dynamo = boto.dynamodb2.connect_to_region('us-east-1',aws_access_key_id=assumedRoleObject.credentials.access_key,aws_secret_access_key=assumedRoleObject.credentials.secret_key,security_token=assumedRoleObject.credentials.session_token) KINESIS_STREAM_NAME = 'edisonDemoKinesis' # Prepare Kinesis client client_kinesis = boto.connect_kinesis(aws_access_key_id=assumedRoleObject.credentials.access_key,aws_secret_access_key=assumedRoleObject.credentials.secret_key,security_token=assumedRoleObject.credentials.session_token) from boto import kinesis #kinesis = kinesis.connect_to_region("us-east-1") #stream = kinesis.create_stream("edisonDemoKinesis", 1) #kinesis.describe_stream("edisonDemoKinesis") #kinesis.list_streams() #global count_dyn count_dyn=0 from boto.dynamodb2.table import Table table_dynamo = Table(DYNAMODB_TABLE_NAME, connection=client_dynamo) try: pingdata = Table.create('pingdata', schema=[HashKey('timestamp')],connection = client_dynamo) count_dyn=0 pingdata = Table('pingdata', schema = [HashKey('timestamp')], connection = client_dynamo) except:
def setUp(self): self.kinesis = boto.connect_kinesis()
def __init__(self): boto.connect_kinesis() self.kinesis = boto.kinesis.connect_to_region('us-west-2') s = self.kinesis.describe_stream('latest_temp_data') s = self.kinesis.describe_stream('temp_history_dev') print(s)
def kinesis_stream(stream): """ Returns Kinesis stream arn """ kinesis = boto.connect_kinesis() return kinesis.describe_stream(stream)['StreamDescription']['StreamARN']
def pause_until_kinesis_active(stream): kinesis = boto.connect_kinesis() # Wait for Kinesis stream to be active while kinesis.describe_stream(stream)['StreamDescription']['StreamStatus'] != 'ACTIVE': print('Kinesis stream [' + stream + '] not active yet') time.sleep(5)
try: data = json.loads(r) reply_text = data['text'].encode('utf_8').replace('\n', '') if reply_text.find("earthquake")!=-1 or\ reply_text.find("Erdbeben")!=-1 or\ reply_text.find("地震")!=-1: location = str(data['geo']['coordinates']) location = location.replace('[', '') location = location.replace(']', '') location = location.replace(' ', '') location_str = location.split(',') time = str(data['created_at']).split(' ') send_data = { 'year': time[5], 'month': time[1], 'date': time[2], 'time': time[3], 'longitude': location_str[1], 'latitude': location_str[0] } send_json = json.dumps(send_data) kinesis.put_record('TwitterStreamTest', send_json, 'one') except: continue if __name__ == '__main__': global kinesis kinesis = boto.connect_kinesis(aws_access_key_id='', aws_secret_access_key='') streaming()
params = {} request = oauth.Request.from_consumer_and_token(consumer, token, http_url=url, parameters=params) request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), consumer, token) res = urllib2.urlopen(request.to_url()) if res == urllib2.URLError: raise Exception else: for r in res: try: data = json.loads(r) reply_text = data['text'].encode('utf_8').replace('\n', '') if reply_text.find("earthquake")!=-1 or\ reply_text.find("Erdbeben")!=-1 or\ reply_text.find("地震")!=-1: location = str(data['geo']['coordinates']) location = location.replace('[','') location = location.replace(']','') location = location.replace(' ','') location_str = location.split(',') time = str(data['created_at']).split(' ') send_data = {'year':time[5],'month':time[1],'date':time[2],'time':time[3],'longitude':location_str[1],'latitude':location_str[0]} send_json = json.dumps(send_data) kinesis.put_record('TwitterStreamTest',send_json,'one') except: continue if __name__ == '__main__': global kinesis kinesis = boto.connect_kinesis(aws_access_key_id='',aws_secret_access_key='') streaming()
consumer_key = 'xx' consumer_secret = 'xx' aws_access_key = 'xx' aws_access_secret = 'xx/X/lNrYLMILRk0z1tEUfHg' #Kinesis stream stream_name = 'tweet_test_dps' #List of keywords list_of_keywords = ['MMM', 'AXP'] class TweetListener(StreamListener): def on_status(self, tweet): _tweet = tweet.__dict__['_json'] print "Tweet: ", _tweet self.send_tweet_to_stream(_tweet) def send_tweet_to_stream(self, tweet): #this block assumes the stream exists and is in a state to accept data #conn = boto.connect_kinesis(aws_access_key, aws_access_secret) conn.put_record(stream_name, json.dumps(tweet), str(int(time.time()))) if __name__ == '__main__': auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) conn = boto.connect_kinesis(aws_access_key, aws_access_secret) l = TweetListener() streamer = Stream(auth, listener=l) streamer.filter(track = list_of_keywords)