def streaming():
global kinesis
CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_TOKEN_KEY = ''
ACCESS_TOKEN_SECRET = ''
consumer = oauth.Consumer(key=CONSUMER_KEY, secret=CONSUMER_SECRET)
token = oauth.Token(key=ACCESS_TOKEN_KEY, secret=ACCESS_TOKEN_SECRET)
url = 'https://stream.twitter.com/1.1/statuses/sample.json'
params = {}
request = oauth.Request.from_consumer_and_token(consumer, token, http_url=url, parameters=params)
request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), consumer, token)
res = urllib2.urlopen(request.to_url())
if res == urllib2.URLError:
raise Exception
else:
for r in res:
try:
data = json.loads(r)
time = str(data['created_at']).split(' ')
send_data = {'year':time[5],'month':time[1],'date':time[2],'time':time[3]}
send_json = json.dumps(send_data)
kinesis.put_record('TwitterStreamTest',send_json,'one')
except:
continue
if __name__ == '__main__':
global kinesis
kinesis = boto.connect_kinesis(aws_access_key_id='',aws_secret_access_key='')
streaming()
def create_kinesis_stream(stream):
"""
Creates our Kinesis stream
"""
kinesis = boto.connect_kinesis()
response = kinesis.create_stream(stream, 1)
pause_until_kinesis_active(stream)
print("Kinesis successfully created")
def dumpKinesis(self):
'''
Method to dump Kinesis streams info.
'''
try:
if self.botoprfl[0] != "default":
conn = boto.connect_kinesis(profile_name = self.botoprfl)
else:
conn = boto.connect_kinesis()
if conn:
print("\n<Start of Kinesis streams>\n")
for c in conn.list_streams():
print(" %s" %c.title())
print("\n<End of Kinesis streams>\n")
except Exception, e:
serr = ('%s :: dumpKinesis(...) : connect_kinesis,list_streams(...), '
'%s' %(self.sclsnme, str(e)))
def dumpKinesis(self):
'''
Method to dump Kinesis streams info.
'''
try:
if self.botoprfl[0] != "default":
conn = boto.connect_kinesis(profile_name=self.botoprfl)
else:
conn = boto.connect_kinesis()
if conn:
print("\n<Start of Kinesis streams>\n")
for c in conn.list_streams():
print(" %s" % c.title())
print("\n<End of Kinesis streams>\n")
except Exception, e:
serr = (
'%s :: dumpKinesis(...) : connect_kinesis,list_streams(...), '
'%s' % (self.sclsnme, str(e)))
def scan():
'''Scans for public tweets using Tweeter's public stream API with a
basic filter. See
https://dev.twitter.com/streaming/reference/post/statuses/filter
'''
kinesis = boto.connect_kinesis()
auth = OAuth(consumer_key=config.CONSUMER_KEY, consumer_secret=config.CONSUMER_SECRET,
token=config.ACCESS_TOKEN, token_secret=config.ACCESS_TOKEN_SECRET)
stream = TwitterStream(auth=auth)
tweets = stream.statuses.filter(track=config.TWITTER_FILTER)
for tweet in tweets:
payload = prepare(tweet)
kinesis.put_record(config.KINESIS_STREAM, payload, config.KINESIS_PARTITION)
logger.debug('Sent to Amazon Kinesis.')
def scan():
'''Scans for public tweets using Tweeter's public stream API with a
basic filter. See
https://dev.twitter.com/streaming/reference/post/statuses/filter
'''
kinesis = boto.connect_kinesis()
auth = OAuth(consumer_key=config.CONSUMER_KEY,
consumer_secret=config.CONSUMER_SECRET,
token=config.ACCESS_TOKEN,
token_secret=config.ACCESS_TOKEN_SECRET)
stream = TwitterStream(auth=auth)
tweets = stream.statuses.filter(track=config.TWITTER_FILTER)
for tweet in tweets:
payload = prepare(tweet)
kinesis.put_record(config.KINESIS_STREAM, payload,
config.KINESIS_PARTITION)
logger.debug('Sent to Amazon Kinesis.')
def send_tweet_to_stream(self, tweet):
conn = boto.connect_kinesis(aws_access_key, aws_access_secret)
r = conn.describe_stream(stream_name)
description = r.get('StreamDescription')
status= description.get('StreamStatus')
try:
if status == 'DELETING':
print 'The stream: {s} is being deleted, please rerun the script.'.format(s=stream_name)
#sys.exit(1)
elif status == 'ACTIVE':
time.sleep(10)
except:
# We'll assume the stream didn't exist so we will try to create it with just one shard
conn.create_stream(stream_name, 1)
time.sleep(10)
conn.put_record(stream_name, json.dumps(tweet), str(int(time.time())))
def get_kinesis_data_iterator(stream_name, minutes_running):
# Connect to Kinesis
kinesis = boto.connect_kinesis()
# Get data about Kinesis stream for Tag Monitor
kinesis_stream = kinesis.describe_stream(stream_name)
# Get the shards in that stream
shards = kinesis_stream['StreamDescription']['Shards']
# Collect together the shard IDs
shard_ids = [shard['ShardId'] for shard in shards]
# Get shard iterator
iter_response = kinesis.get_shard_iterator(stream_name, shard_ids[0], "TRIM_HORIZON")
shard_iterator = iter_response['ShardIterator']
# Calculate end time
end_time = datetime.now() + timedelta(minutes=minutes_running)
while True:
try:
# Get data
record_response = kinesis.get_records(shard_iterator)
# Only run for a certain amount of time.
# Stop looping if no data returned. This means it's done
now = datetime.now()
print 'Time: {0}'.format(now.strftime('%Y/%m/%d %H:%M:%S'))
if end_time < now or not record_response:
break
# yield data to outside calling iterator
for record in record_response['Records']:
last_sequence = record['SequenceNumber']
yield json.loads(record['Data'])
# Get next iterator for shard from previous request
shard_iterator = record_response['NextShardIterator']
# Catch exception meaning hitting API too much
except boto.kinesis.exceptions.ProvisionedThroughputExceededException:
print 'ProvisionedThroughputExceededException found. Sleeping for 0.5 seconds...'
time.sleep(0.5)
# Catch exception meaning iterator has expired
except boto.kinesis.exceptions.ExpiredIteratorException:
iter_response = kinesis.get_shard_iterator(stream_name, shard_ids[0], "AFTER_SEQUENCE_NUMBER", last_sequence)
shard_iterator = iter_response['ShardIterator']
kinesis.close()
import datetime
from argparse import RawTextHelpFormatter
from random import choice
from string import lowercase
from boto.kinesis.exceptions import ResourceNotFoundException
# To preclude inclusion of aws keys into this code, you may temporarily add
# your AWS credentials to the file:
# ~/.boto
# as follows:
# [Credentials]
# aws_access_key_id = <your access key>
# aws_secret_access_key = <your secret key>
kinesis = boto.connect_kinesis()
make_string = lambda x: "".join(choice(lowercase) for i in range(x))
def get_or_create_stream(stream_name, shard_count):
stream = None
try:
stream = kinesis.describe_stream(stream_name)
print(
json.dumps(stream,
sort_keys=True,
indent=2,
separators=(',', ': ')))
except ResourceNotFoundException as rnfe:
while (stream is None) or (
import json
import os
from time import sleep
from zipfile import ZipFile
import boto
from boto.kinesis.exceptions import ResourceInUseException
import config
# To enable logging:
# boto.set_stream_logger('boto')
# Initialize the AWS clients.
sns = boto.connect_sns()
kinesis = boto.connect_kinesis()
aws_lambda = boto.connect_awslambda()
ml = boto.connect_machinelearning()
lambda_execution_policy = open('lambdaExecutionPolicyTemplate.json').read().format(**config.AWS)
aws_account_id = config.AWS["awsAccountId"]
region = config.AWS["region"]
kinesis_stream = config.AWS["kinesisStream"]
sns_topic = config.AWS["snsTopic"]
lambda_function_name = config.AWS["lambdaFunctionName"]
lambda_execution_role = config.AWS["lambdaExecutionRole"]
lambda_trust_policy = '{"Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}'
cognito_id = cognito.get_id(ACCOUNT_ID, IDENTITY_POOL_ID)
oidc = cognito.get_open_id_token(cognito_id['IdentityId'])
# Further setup your STS using the code below
sts = boto.connect_sts()
assumedRoleObject = sts.assume_role_with_web_identity(ROLE_ARN, "XX", oidc['Token'])
# Connect to dynamoDB and kinesis
client_dynamo = boto.dynamodb2.connect_to_region(
'us-east-1',
aws_access_key_id=assumedRoleObject.credentials.access_key,
aws_secret_access_key=assumedRoleObject.credentials.secret_key,
security_token=assumedRoleObject.credentials.session_token)
client_kinesis = boto.connect_kinesis(
aws_access_key_id=assumedRoleObject.credentials.access_key,
aws_secret_access_key=assumedRoleObject.credentials.secret_key,
security_token=assumedRoleObject.credentials.session_token)
from boto.dynamodb2.table import Table
from boto.dynamodb2.fields import HashKey
######################
# Setup DynamoDB Table
######################
table_dynamo = # YOUR CODE HERE #
#################################################
# Setup switch and temperature sensor #
#################################################
# Use cognito to get an identity.
cognito = boto.connect_cognito_identity()
cognito_id = cognito.get_id(ACCOUNT_ID, IDENTITY_POOL_ID)
oidc = cognito.get_open_id_token(cognito_id['IdentityId'])
# Further setup your STS using the code below
sts = boto.connect_sts()
assumedRoleObject = sts.assume_role_with_web_identity(ROLE_ARN, "XX", oidc['Token'])
DYNAMODB_TABLE_NAME = 'pingdata'
# Prepare DynamoDB client
client_dynamo = boto.dynamodb2.connect_to_region('us-east-1',aws_access_key_id=assumedRoleObject.credentials.access_key,aws_secret_access_key=assumedRoleObject.credentials.secret_key,security_token=assumedRoleObject.credentials.session_token)
KINESIS_STREAM_NAME = 'edisonDemoKinesis'
# Prepare Kinesis client
client_kinesis = boto.connect_kinesis(aws_access_key_id=assumedRoleObject.credentials.access_key,aws_secret_access_key=assumedRoleObject.credentials.secret_key,security_token=assumedRoleObject.credentials.session_token)
from boto import kinesis
#kinesis = kinesis.connect_to_region("us-east-1")
#stream = kinesis.create_stream("edisonDemoKinesis", 1)
#kinesis.describe_stream("edisonDemoKinesis")
#kinesis.list_streams()
#global count_dyn
count_dyn=0
from boto.dynamodb2.table import Table
table_dynamo = Table(DYNAMODB_TABLE_NAME, connection=client_dynamo)
try:
pingdata = Table.create('pingdata', schema=[HashKey('timestamp')],connection = client_dynamo)
count_dyn=0
pingdata = Table('pingdata', schema = [HashKey('timestamp')], connection = client_dynamo)
except:
def setUp(self):
self.kinesis = boto.connect_kinesis()
def setUp(self):
self.kinesis = boto.connect_kinesis()
def __init__(self):
boto.connect_kinesis()
self.kinesis = boto.kinesis.connect_to_region('us-west-2')
s = self.kinesis.describe_stream('latest_temp_data')
s = self.kinesis.describe_stream('temp_history_dev')
print(s)
def kinesis_stream(stream):
"""
Returns Kinesis stream arn
"""
kinesis = boto.connect_kinesis()
return kinesis.describe_stream(stream)['StreamDescription']['StreamARN']
def pause_until_kinesis_active(stream):
kinesis = boto.connect_kinesis()
# Wait for Kinesis stream to be active
while kinesis.describe_stream(stream)['StreamDescription']['StreamStatus'] != 'ACTIVE':
print('Kinesis stream [' + stream + '] not active yet')
time.sleep(5)
try:
data = json.loads(r)
reply_text = data['text'].encode('utf_8').replace('\n', '')
if reply_text.find("earthquake")!=-1 or\
reply_text.find("Erdbeben")!=-1 or\
reply_text.find("地震")!=-1:
location = str(data['geo']['coordinates'])
location = location.replace('[', '')
location = location.replace(']', '')
location = location.replace(' ', '')
location_str = location.split(',')
time = str(data['created_at']).split(' ')
send_data = {
'year': time[5],
'month': time[1],
'date': time[2],
'time': time[3],
'longitude': location_str[1],
'latitude': location_str[0]
}
send_json = json.dumps(send_data)
kinesis.put_record('TwitterStreamTest', send_json, 'one')
except:
continue
if __name__ == '__main__':
global kinesis
kinesis = boto.connect_kinesis(aws_access_key_id='',
aws_secret_access_key='')
streaming()
params = {}
request = oauth.Request.from_consumer_and_token(consumer, token, http_url=url, parameters=params)
request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), consumer, token)
res = urllib2.urlopen(request.to_url())
if res == urllib2.URLError:
raise Exception
else:
for r in res:
try:
data = json.loads(r)
reply_text = data['text'].encode('utf_8').replace('\n', '')
if reply_text.find("earthquake")!=-1 or\
reply_text.find("Erdbeben")!=-1 or\
reply_text.find("地震")!=-1:
location = str(data['geo']['coordinates'])
location = location.replace('[','')
location = location.replace(']','')
location = location.replace(' ','')
location_str = location.split(',')
time = str(data['created_at']).split(' ')
send_data = {'year':time[5],'month':time[1],'date':time[2],'time':time[3],'longitude':location_str[1],'latitude':location_str[0]}
send_json = json.dumps(send_data)
kinesis.put_record('TwitterStreamTest',send_json,'one')
except:
continue
if __name__ == '__main__':
global kinesis
kinesis = boto.connect_kinesis(aws_access_key_id='',aws_secret_access_key='')
streaming()
consumer_key = 'xx'
consumer_secret = 'xx'
aws_access_key = 'xx'
aws_access_secret = 'xx/X/lNrYLMILRk0z1tEUfHg'
#Kinesis stream
stream_name = 'tweet_test_dps'
#List of keywords
list_of_keywords = ['MMM', 'AXP']
class TweetListener(StreamListener):
def on_status(self, tweet):
_tweet = tweet.__dict__['_json']
print "Tweet: ", _tweet
self.send_tweet_to_stream(_tweet)
def send_tweet_to_stream(self, tweet):
#this block assumes the stream exists and is in a state to accept data
#conn = boto.connect_kinesis(aws_access_key, aws_access_secret)
conn.put_record(stream_name, json.dumps(tweet), str(int(time.time())))
if __name__ == '__main__':
auth = OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_secret)
conn = boto.connect_kinesis(aws_access_key, aws_access_secret)
l = TweetListener()
streamer = Stream(auth, listener=l)
streamer.filter(track = list_of_keywords)