def init_broker_stuffs(self, logger):
def log_error_and_raise(err_msg, logger):
print(err_msg)
logger.error(err_msg)
raise Exception('Got error inside "init_broker_stuffs" function.')
try:
client = kinesis.connect_to_region(
self.region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key
) if self.aws_access_key_id != '' \
and self.aws_secret_access_key != '' else kinesis.connect_to_region(self.region)
stream_descriptor = client.describe_stream(
stream_name=self.stream_name)
self.stream_descriptor = stream_descriptor
self.client = client
except ResourceNotFoundException:
if self.stream_descriptor is None:
err_msg = 'Could not find {stream_name} stream.' \
.format(stream_name=self.stream_name)
log_error_and_raise(err_msg, logger)
return
status = self.stream_descriptor['StreamDescription'][
'StreamStatus']
if 'ACTIVE' not in status:
err_msg = 'Stream status: {status}. Should be ACTIVE.' \
.format(status=status)
log_error_and_raise(err_msg, logger)
except Exception as e:
raise e
def __init__(self,
aws_region=None,
access_key=None,
secret_access_key=None):
'''
Construtor
:param aws_region: Nome da região global com a qual será feita a conexão
:param access_key: Chave pública de acesso do usuário
:param secret_access_key: Chave secreta de acesso do usuário
'''
configWrapper = Configuration()
self.config = configWrapper.getConfigurationFromFile()
if access_key is not None and secret_access_key is not None and aws_region is not None:
self.conn = kinesis.connect_to_region(
aws_region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_access_key)
else:
self.conn = kinesis.connect_to_region(
self.config['aws']['awsAuth']['region'],
aws_access_key_id=self.config['aws']['awsAuth']['access_key'],
aws_secret_access_key=self.config['aws']['awsAuth']
['secret_access_key'])
def put_record(stream_name, partition_key, message, cred_exp) :
if (cred_exp == True) :
auth = {"aws_access_key_id":aws_api_key, "aws_secret_access_key":aws_api_secret}
kin = kinesis.connect_to_region("us-east-1", **auth)
else :
kin = kinesis.connect_to_region("us-east-1")
print '@%s: %s' % (partition_key, message)
kin.put_record(stream_name, message, partition_key)
def run():
from boto import kinesis
kinesis = kinesis.connect_to_region("ca-central-1")
#stream = kinesis.create_stream("ErikSparkPOC", 1)
start = time.time()
reddit = praw.Reddit(client_id='gluFwvMrQLqLuA',
client_secret='nowLOmNuC8tS76mrc-LQUlarngw',
user_agent='testscript by /u/plasmatrendybot',
password='******',
username='******')
# Subreddit name
subreddit = reddit.subreddit('askreddit')
comments = subreddit.stream.comments()
escape_limit = 100000
x = 0
for comment in comments:
if comment.ups > 0:
print(comment.body, comment.ups)
reddit_comment = RedditComment(
comment.body.encode('ascii', 'ignore'))
try:
kinesis.put_record("ErikSparkPOC", json.dumps(reddit_comment),
"partitionkey")
except:
print("Failed to insert to Kinesis")
x = x + 1
if x > escape_limit:
break
def write_partition(partition):
# Access the Kinesis client object
kinesisClient = kinesis.connect_to_region("us-east-1")
# Iterate over rows
for row in partition:
# Send the row as a JSON string into the Kinesis stream
kinesisClient.put_record(kinesisStreamName, json.dumps(row),"partitionKey")
def generate_events(profile, region, stream):
"""
load demo data with python generator script for SimpleEvents
"""
conn = kinesis.connect_to_region(region, profile_name=profile)
while True:
event_json = write_event(conn, stream)
print "Event sent to Kinesis: {}".format(event_json)
def generate_events(profile, region, stream):
"""
load demo data with python generator script for SimpleEvents
"""
conn = kinesis.connect_to_region(region, profile_name=profile)
while True:
event_json = write_event(conn, stream)
print "Event sent to Kinesis: {}".format(event_json)
def connect_kinesis():
auth = {"aws_access_key_id":aws_access_key_id, "aws_secret_access_key":aws_secret_access_key}
try:
conn = kinesis.connect_to_region(region,**auth)
return conn
except Exception as e:
print('KINESIS connection error: '+str(e))
sys.exit(1)
def put_data_into_stream(stream, region, sleep_duration, records_to_create):
LOG.error("put_data_into_stream(%s, %s)", stream, region)
connection = kinesis.connect_to_region(region)
for i in range(records_to_create):
data = uuid.uuid4().hex
LOG.debug("Putting %s onto stream", data)
res = connection.put_record(stream, data, data)
LOG.debug(res)
time.sleep(sleep_duration)
def write_partition(partition):
# Access the Kinesis client object
kinesisClient = kinesis.connect_to_region("us-east-1")
# Iterate over rows
for row in partition:
# Send the row as a JSON string into the Kinesis stream
kinesisClient.put_record(kinesisStreamName, json.dumps(row),
"partitionKey")
def process_stream(stream, region):
LOG.error("process_stream(%s, %s)", stream, region)
connection = kinesis.connect_to_region(region)
description = connection.describe_stream(stream)
shard_ids = [shard['ShardId'] for shard in description['StreamDescription']['Shards']]
for shard_id in shard_ids:
t = Thread(target=log_stream_iterator, args=(stream, connection, shard_id,))
t.start()
def __init__(self, stream_name, region='us-east-2'):
self.stream_name = stream_name
self.region = region
self.kinesis_conn = kinesis.connect_to_region(
region) # Connect to a Kinesis stream in a specific region
self.description = self.kinesis_conn.describe_stream(stream)
self.shards = self.get_shards()
self.first_shard_id = self.shards[0]["ShardId"]
self.partition_key = 0
def __init__(self, **kwargs):
"""
Event tracker backend that uses a Kinesis Stream
`streamName` is the stream to send the events to
"""
self.streamName = kwargs.get('streamName', None)
self.regionName = kwargs.get('regionName', None)
self.kinesis = kinesis.connect_to_region(region_name = self.regionName)
def __init__(self, num_partitions, topic, *args, **kwargs):
self.brokertype = "kinesis"
self.con = kinesis.connect_to_region(REGION)
self.num_shards = num_partitions
try:
self.bulksize = kwargs["bulksize"]
except:
self.bulksize = 1
self.msg_bulk = []
self.topic = topic
self._create_stream()
def __init__(self, num_partitions, topic, *args, **kwargs):
self.brokertype = "kinesis"
self.con = kinesis.connect_to_region(REGION)
self.num_shards = num_partitions
try:
self.bulksize = kwargs["bulksize"]
except:
self.bulksize = 1
self.msg_bulk = []
self.topic = topic
self._create_stream()
def main(args):
if args.key is None or args.keysecret is None:
kin = kinesis.connect_to_region(args.datacenter)
else:
kin = kinesis.connect_to_region(args.datacenter,
aws_access_key_id=args.key,
aws_secret_access_key=args.keysecret)
shard_id = args.shardid
shard_it = kin.get_shard_iterator(args.name, shard_id,
args.iterator)['ShardIterator']
while True:
out = kin.get_records(shard_it, limit=args.limit)
for o in out['Records']:
print(o['Data'])
jdat = json.loads(o['Data'])
sys.stdout.write('%s\n' % json.dumps(jdat))
sys.stdout.flush()
shard_it = out['NextShardIterator']
time.sleep(args.interval)
def update_kinesis(project_model, message_type):
"""
notifies media atom of a project update or create by pushing a message onto its kinesis stream.
the kinesis stream is indicated in settings.
:param project_model: ProjectModel instance that has been created/updated
:param message_type: either `media_atom.MSG_PROJECT_CREATED` or `media_atom.MSG_PROJECT_UPDATED`
:return:
"""
from portal.plugins.gnm_vidispine_utils.vs_helpers import site_id
from boto import sts, kinesis
from django.conf import settings
import json, logging
SESSION_NAME = 'pluto-media-atom-integration'
project_id = site_id + "-" + str(project_model.collection_id)
logger.info("{0}: Project updated, notifying {1} via role {2}".format(
project_id, settings.MEDIA_ATOM_STREAM_NAME,
settings.MEDIA_ATOM_ROLE_ARN))
sts_connection = sts.STSConnection(
aws_access_key_id=settings.MEDIA_ATOM_AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.MEDIA_ATOM_AWS_SECRET_ACCESS_KEY)
assume_role_result = sts_connection.assume_role(
role_arn=settings.MEDIA_ATOM_ROLE_ARN, role_session_name=SESSION_NAME)
credentials = assume_role_result.credentials
logger.debug("{0}: Got kinesis credentials".format(project_id))
kinesis_connection = kinesis.connect_to_region(
region_name='eu-west-1',
aws_access_key_id=credentials.access_key,
aws_secret_access_key=credentials.secret_key,
security_token=credentials.session_token)
message_content = {
'type': message_type,
'id': project_id,
'title': project_model.gnm_project_headline,
'status': project_model.gnm_project_status,
'commissionId':
site_id + "-" + str(project_model.commission.collection_id),
'commissionTitle': project_model.commission.gnm_commission_title,
'productionOffice': project_model.gnm_project_production_office,
'created': project_model.created.isoformat()
}
logger.debug("{0}: Message is {1}".format(project_id, message_content))
kinesis_connection.put_record(stream_name=settings.MEDIA_ATOM_STREAM_NAME,
data=json.dumps(message_content),
partition_key=project_id)
logger.info("{0}: Project update sent".format(project_id))
def refresh_access_credentials(self):
sts_conn = sts.connect_to_region(
'eu-west-1',
aws_access_key_id=self._aws_access_key_id,
aws_secret_access_key=self._aws_secret_access_key)
credentials = sts_conn.assume_role(self.role_name, self.session_name)
self._conn = kinesis.connect_to_region(
'eu-west-1',
aws_access_key_id=credentials.credentials.access_key,
aws_secret_access_key=credentials.credentials.secret_key,
security_token=credentials.credentials.session_token)
def put_words_in_stream(w):
conn = kinesis.connect_to_region(region_name='ap-northeast-2')
stream_name = "datastream_2"
conn.describe_stream(stream_name)
w = json.dumps(w)
try:
conn.put_record(stream_name, w, "partitionkey")
print("Put word: " + w + " into stream: " + stream_name)
except Exception as e:
sys.stderr.write(
"Encountered an exception while trying to put a word: " + w +
" into stream: " + stream_name + " exception was: " + str(e))
def put_aws(self, data):
"""
数据及日志上传
:param data: 数据
:param stream_name: 流名称
:return:
"""
config = self.spider.config
message = json.dumps(data).encode('utf-8')
kinesis_client = kinesis.connect_to_region('cn-north-1')
kinesis_client.put_record(config.get("KINESIS", "IMAGE_STREAM_NAME"),
data=message,
partition_key='partitionKey1')
def run():
from boto import kinesis
kinesis = kinesis.connect_to_region("ca-central-1")
#stream = kinesis.create_stream("ErikDemo", 1)
i = 0
for i in xrange(10):
user = User("Demo")
#print(json.dumps(user))
print("Did I make it here?")
kinesis.put_record("ErikDemo", json.dumps(user), "partitionkey")
def main():
region = 'us-west-2'
aws_access_key_id = 'aws_access_key_id'
aws_secret_access_key = 'aws_secret_access_key'
client = kinesis.connect_to_region(
region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
stream_name = 'stream_name'
stream_descriptor = client.describe_stream(stream_name=stream_name)
shards = stream_descriptor['StreamDescription']['Shards']
for shard in shards:
executor = Executor(client, stream_name, shard['ShardId'])
executor.start()
def startStreaming(self):
print("Starting stream.")
self.kinesisConn = kinesis.connect_to_region(
self.connectionDetails['kinesis']['region'])
thread.start_new_thread(self.streamSbs1StreamAsync, (
self,
config['sbs1']['portADSB'],
))
thread.start_new_thread(self.streamSbs1StreamAsync, (
self,
config['sbs1']['portMLAT'],
))
while 1:
pass
def test_simple_emit(self):
# need to create a stream to test against
conn = kinesis.connect_to_region("us-east-1")
conn.create_stream(StreamName='my_stream', ShardCount=1)
self.backend = KinesisBackend(streamName='my_stream')
event = {
'name': 'bob uncle',
'context': {
'user_id': 'bob'
},
'data': {
'foo': 'test'
}
}
self.backend.send(event)
def main(order_or_vendor):
order_df = to_df(producer_var[order_or_vendor]["order_df_file_path"])
kinesis_put = kinesis.connect_to_region(producer_var[order_or_vendor]["kinesis_region"])
order_df_json = order_df.to_dict(orient="records")
for i in range(len(order_df_json)):
if i > 0:
pprint.pprint(i)
kinesis_put.put_record(producer_var[order_or_vendor]["order_stream_name"], json.dumps(order_df_json[i]).encode("utf-8"), "partitionkey")
sleep_time = (pd.to_datetime(order_df_json[i]['datetime'], format="%Y-%m-%d %H:%M:%S") - pd.to_datetime(
order_df_json[i - 1]['datetime'], format="%Y-%m-%d %H:%M:%S")).total_seconds()
time.sleep(sleep_time)
else:
kinesis_put.put_record(producer_var[order_or_vendor]["order_stream_name"], json.dumps(order_df_json[i]).encode("utf-8"), "partitionkey")
def main():
from boto import kinesis
kinesis = kinesis.connect_to_region("us-east-2")
shard_id = 'shardId-000000000000' #we only have one shard!
shard_it = kinesis.get_shard_iterator("end-stream", shard_id,
"LATEST")["ShardIterator"]
while 1 == 1:
out = kinesis.get_records(shard_it, limit=1)
shard_it = out["NextShardIterator"]
if len(out["Records"]) > 0:
print(out["Records"][0]["Data"])
data_dict = json.loads(out["Records"][0]["Data"])
timestamp, userName, device_name, detail_result = extract_data_from_kinesis_stream(
data_dict)
# print(shard_it)
write_to_db(timestamp, userName, device_name, detail_result)
time.sleep(0.1)
def handle(self, *args, **options):
pprint(options)
if 'aws_access_key_id' in options and 'aws_secret_access_key' in options:
sts_conn = sts.connect_to_region(
'eu-west-1',
aws_access_key_id=options['aws_access_key_id'],
aws_secret_access_key=options['aws_secret_access_key'])
else:
sts_conn = sts.connect_to_region('eu-west-1')
credentials = sts_conn.assume_role(self.role_name, self.session_name)
pprint(credentials.credentials.__dict__)
conn = kinesis.connect_to_region(
'eu-west-1',
aws_access_key_id=credentials.credentials.access_key,
aws_secret_access_key=credentials.credentials.secret_key,
security_token=credentials.credentials.session_token)
streaminfo = conn.describe_stream(self.stream_name)
pprint(streaminfo)
threadlist = map(
lambda shardinfo: self.startup_thread(credentials.credentials,
shardinfo),
streaminfo['StreamDescription']['Shards'])
print "Stream {0} has {1} shards".format(self.stream_name,
len(threadlist))
for t in threadlist:
t.daemon = True
t.start()
print "Started up and processing. Hit CTRL-C to stop."
#simplest way to allow ctrl-C when dealing with threads
try:
while True:
sleep(3600)
except KeyboardInterrupt:
print "CTRL-C caught, cleaning up"
def put_data(self):
region = self.get_region()
stream_name = self.get_kinesis_stream_name()
conn = kinesis.connect_to_region(region_name=region)
try:
# Check stream status
status = self.get_stream_status(conn, stream_name)
if 'DELETING' == status:
print('The stream: {s} is being deleted, please rerun the script.'.format(s=stream_name))
sys.exit(1)
elif 'ACTIVE' != status:
self.wait_for_stream(conn, stream_name)
# put data into stream
data = self.get_mock_data(israndom= False)
self.put_record_in_stream(conn, stream_name, data)
data = self.get_mock_data(israndom=True)
self.put_record_in_stream(conn, stream_name, data)
except Exception as error:
print('{}'.format(error))
sys.exit(1)
def main(args):
kin = kinesis.connect_to_region(args.datacenter,
aws_access_key_id=args.key,
aws_secret_access_key=args.keysecret)
shard_id = args.shardid
shard_it = kin.get_shard_iterator(args.name, shard_id,
args.iterator)['ShardIterator']
while True:
result = []
out = kin.get_records(shard_it, limit=args.limit)
for o in out['Records']:
result.append(o['Data'].decode("utf-8"))
sys.stdout.write('%s\n' % o['Data'])
jdat = json.loads(o['Data'].decode("utf-8"))
sys.stdout.write('%s\n' % json.dumps(jdat))
sys.stdout.flush()
sys.stdout.write('%s\n' % result)
sys.stdout.flush()
shard_it = out['NextShardIterator']
time.sleep(args.interval)
def upload_kinesis():
connection = kinesis.connect_to_region('us-east-1')
tries = 0
while tries < 10:
tries += 1
time.sleep(1)
try:
response = connection.describe_stream('Temperature')
if response['StreamDescription']['StreamStatus'] == 'ACTIVE':
break
except:
logger.error('error while trying to describe kinesis stream : %s')
else:
raise TimeoutError('Stream is still not active, aborting...')
data = tempData()
print(data)
try:
connection.put_record("Temperature", json.dumps(data), "partitionkey")
except IOError:
print ErrorString
def main(order_or_vendor):
engine = create_engine(
"mysql+mysqlconnector://root:toor@localhost:3306/grab")
taxi_zone_lookup = pd.read_csv("data/taxi_zone_lookup.csv",
index_col="Unnamed: 0")
def timeseries2loc(msg):
ts_df = pd.DataFrame([json.loads(i["Data"]) for i in a["Records"]])
print(ts_df)
order_timeseries_with_gps = ts_df.join(taxi_zone_lookup,
on='locationID',
how="left",
lsuffix="_1")
order_timeseries_with_gps["datetime"] = pd.to_datetime(
order_timeseries_with_gps["datetime"])
print(order_timeseries_with_gps)
order_timeseries_with_gps.to_sql(
con=engine,
name=consumer_var[order_or_vendor]["sql_table_name"],
if_exists="append",
chunksize=1000)
return order_timeseries_with_gps
kinesis_get = kinesis.connect_to_region(kinesis_region)
shard_id = 'shardId-000000000000'
shard_it = kinesis_get.get_shard_iterator(
consumer_var[order_or_vendor]["order_stream_name"], shard_id,
"LATEST")["ShardIterator"]
while True:
a = kinesis_get.get_records(shard_it)
# print (a)
shard_it = a["NextShardIterator"]
print("get %d records" % len(a["Records"]))
if len(a["Records"]) > 0:
pprint.pprint(a["Records"][0])
timeseries2loc(a)
# print (">>>>>next >>>> \t"+shard_it)
time.sleep(2)
def __init__(self,
consumer_name,
stream,
shard=None,
starting_point=None,
region='ap-southeast-1'):
self.connection = kinesis.connect_to_region(region, **kinesis_auth)
self.stream = stream
self.shard = shard
self.shard_starting_point = SpoolManager("kinesis_sync")
_, self.starting_point = self.shard_starting_point.get_file_point_file(
)
self.starting_point = None
shards = self._list_shards(stream)
if self.shard:
for _shard in shards:
if _shard["ShardId"] == shard:
break
else:
self.shard = shards[0]["ShardId"]
self.shard_iterator = self._get_stream_iterator(
self.stream, self.shard)
def main():
from boto import kinesis
kinesis = kinesis.connect_to_region("us-east-2")
shard_id = 'shardId-000000000000' #we only have one shard!
shard_it = kinesis.get_shard_iterator("end-stream", shard_id,
"LATEST")["ShardIterator"]
timestamps = [i for i in range(20)]
bodyorientationpitches = [i for i in range(20)]
steps = [i for i in range(20)]
while 1 == 1:
out = kinesis.get_records(shard_it, limit=1)
shard_it = out["NextShardIterator"]
if len(out["Records"]) > 0:
print(out["Records"][0]["Data"])
data_dict = json.loads(out["Records"][0]["Data"])
timestamp, userName, device_name, detail_result = extract_data_from_kinesis_stream(
data_dict)
timestamps.insert(0, timestamp)
timestamps = timestamps[:-1]
bodyorientationpitches.insert(
0, detail_result["IMU"]["BodyOrientationPitch"])
bodyorientationpitches = bodyorientationpitches[:-1]
steps.insert(0, detail_result["IMU"]["StepCount"])
steps = steps[:-1]
x1 = np.array(timestamps)
y1 = np.array(bodyorientationpitches)
x2 = np.array(timestamps)
y2 = np.array(steps)
plot_vals(x1, y1, x2, y2)
def create_write(region, stream_name, key_func):
conn = kinesis.connect_to_region(region_name = region)
try:
status = _get_stream_status(conn, stream_name)
if 'DELETING' == status:
raise KinesisException('The stream: {s} is being deleted.'.format(s=stream_name))
elif 'ACTIVE' != status:
_wait_for_stream(conn, stream_name)
except:
raise KinesisException('The stream: {s} is being deleted.'.format(s=stream_name))
def write(event):
key = key_func(event)
json_event = json.dumps(event)
try:
conn.put_record(stream_name, json_event, key)
except Exception as e:
raise KinesisException("Encountered an exception while trying to put: "
+ event + " into stream: " + stream_name
+ " exception was: " + str(e))
return write
from boto import kinesis
import mysql.connector
import time
import json
mydb = mysql.connector.connect(
host=HOST,
user=USER,
passwd=PASSWORD,
db=DB
)
cursor = mydb.cursor()
kinesis = kinesis.connect_to_region('us-east-1')
shard_id = 'shardId-0000000000000'
shard_it = kinesis.get_shard_iterator(MYSTREAM, shard_id, 'LATEST')['ShardIterator']
while 1==1:
print("getting record")
out = kinesis.get_records(shard_it,limit=1)
for i in out['Records']:
recordId = json.loads(i['Data'])['SaleId']['N']
productId = json.loads(i['Data'])['ProductId']['N']
quantity = json.loads(i['Data'])['Quantity']['N']
saleDate = json.loads(i['Data'])['SaleDate']['S']
help="The region you'd like to make this stream in. Default is 'us-east-1'", metavar="REGION_NAME",)
parser.add_argument("-w", "--word", dest="words", default=[], action="append",
help="A word to add to the stream. Can be specified multiple times to add multiple words.", metavar="WORD",)
parser.add_argument("-p", "--period", dest="period", type=int,
help="If you'd like to repeatedly put words into the stream, this option provides the period for putting "
+ "words into the stream in SECONDS. If no period is given then the words are put once.",
metavar="SECONDS",)
args = parser.parse_args()
stream_name = args.stream_name
'''
Getting a connection to Amazon Kinesis will require that you have your credentials available to
one of the standard credentials providers.
'''
print("Connecting to stream: {s} in {r}".format(s=stream_name, r=args.region))
conn = kinesis.connect_to_region(region_name = args.region)
try:
status = get_stream_status(conn, stream_name)
if 'DELETING' == status:
print('The stream: {s} is being deleted, please rerun the script.'.format(s=stream_name))
sys.exit(1)
elif 'ACTIVE' != status:
wait_for_stream(conn, stream_name)
except:
# We'll assume the stream didn't exist so we will try to create it with just one shard
conn.create_stream(stream_name, 1)
wait_for_stream(conn, stream_name)
# Now the stream should exist
if len(args.words) == 0:
print('No -w options provided. Waiting on input from STDIN')
words = [l.strip() for l in sys.stdin.readlines() if l.strip() != '']
from boto import kinesis
aws_profile = "learner"
kinesis = kinesis.connect_to_region("us-east-1")
description = kinesis.describe_stream("pharmacy-records")
print(description)
from boto import kinesis
import testdata,json
# Creating fake data
class Users(testdata.DictFactory):
firstname = testdata.FakeDataFactory("firstName")
lastname = testdata.FakeDataFactory("lastName")
age = testdata.RandomInteger(10,30)
gender = testdata.RandomSelection(['female','male'])
# Using boto connect to the region in which your kinesis stream is created
kinesis = kinesis.connect_to_region("eu-west-1")
for user in Users().generate(50):
print user
kinesis.put_record("EdisonDemo", json.dumps(user), "partitionkey")
from boto import kinesis
import sys
# Go to http://apps.twitter.com and create an app.
# The consumer key and secret will be generated for you after
consumer_key=""
consumer_secret=""
# After the step above, you will be redirected to your app's page.
# Create an access token under the the "Your access token" section
access_token=""
access_token_secret=""
#Kinesis stream connection parameters
#Shard names would be dynamically created for applications looking to scale out the stream
conn = kinesis.connect_to_region(region_name = "us-east-1")
stream_name = ""
shard_name = "shard1"
def put_tweet_in_stream(tweet):
"""
Put each tweet into the kinesis stream
:type tweet: str
"""
try:
tweet_text = json.loads(tweet)['text']
conn.put_record(stream_name,tweet,shard_name)
except Exception as e:
sys.stderr.write("Encountered an exception while trying to put a tweet: "
+ tweet + " into stream: " + stream_name + " exception was: " + str(e))
},
"verb": "read",
"direct_object": {
"filesystem_metrics": {
"size": size,
"free": free,
"available": avail
}
},
"at": get_event_time(),
"on": {
"server": {
"hostname": get_hostname()
}
}
})
def write_event(conn, stream_name):
event_id, event_payload = create_event()
event_json = json.dumps(event_payload)
conn.put_record(stream_name, event_json, event_id)
return event_id
if __name__ == '__main__': # a
conn = kinesis.connect_to_region(region_name="us-east-1",
profile_name="ulp")
while True: # b
event_id = write_event(conn, "events")
print "Wrote event: {}".format(event_id)
time.sleep(10) # c
def init_kinesis():
k = kinesis.connect_to_region(KINESIS_AWS_REGION)
if KINESIS_STREAM_NAME not in k.list_streams()['StreamNames']:
k.create_stream(KINESIS_STREAM_NAME, 1)
return k
lines = text.split("\n")
header = lines[0]
m = re.match(r"<doc id=\"([0-9]+)\" url=\"(.+)\" title=\"(.+)\"", header)
if m:
article_id = int(m.group(1))
article_text = "\n".join(lines[1:-1])
doc = {"article_id": str(uuid.uuid4()), "url": m.group(2), "title": m.group(3), "body": article_text}
if INSERT_IN_BATCHES:
doc_buffer.append(json.dumps(doc))
if len(doc_buffer) == BATCH_SIZE:
kinesis.put_records(kinesis_stream, doc_buffer, str(uuid.uuid4()))
doc_buffer = []
else:
kinesis.put_record(kinesis_stream, json.dumps(doc), str(uuid.uuid4()))
print "Sent article '{}' ({})".format(doc["title"], doc["article_id"])
else:
print "Header not found in {0}".format(article_file)
if __name__ == "__main__":
processed_count = 0
doc_buffer = []
start_time = time.time()
data_dir = "data"
kinesis_stream = "lambdastorm_input"
kinesis = kinesis.connect_to_region(AWS_REGION)
process(data_dir, NUM_PROCESSES)
import cv2 import numpy as np import sys import pickle as cPickle #import struct #from multiprocessing import Pool import pytz ## for timezone calculations import datetime aws_region = "us-west-2" stream_name = "TestStream" cap = cv2.VideoCapture(0) #pool = Pool(processes=2) kinesis = kinesis.connect_to_region(aws_region) camera_index = 0 # 0 is usually the built-in webcam capture_rate = 2 # Frame capture rate.. every X frames. Positive integer. rekog_max_labels = 123 rekog_min_conf = 50.0 frame_count = 0 ## Old code #************************************************************************ #for line in x.iter_lines(): # kinesis = kinesis.connect_to_region(aws_region) # kinesis.put_record(stream_name, line, "partitionkey") # if line: # print (line) #************************************************************************
#!/usr/bin/env python
import time
import os
import RPi.GPIO as GPIO
from boto import kinesis
GPIO.setmode(GPIO.BCM)
DEBUG = 1
# set values
STREAM_NAME = 'rPi_v1'
# connect to oregon
conn = kinesis.connect_to_region('us-west-2')
response = conn.describe_stream(STREAM_NAME)
shard_ids = []
stream_name = None
if response and 'StreamDescription' in response:
stream_name = response['StreamDescription']['StreamName']
for shard_id in response['StreamDescription']['Shards']:
shard_id = shard_id['ShardId']
shard_iterator = conn.get_shard_iterator(stream_name, shard_id, shard_iterator_type)
shard_ids.append({'shard_id' : shard_id ,'shard_iterator' : shard_iterator['ShardIterator'] })
# read SPI data from MCP3008 chip, 8 possible adc's (0 thru 7)
def readadc(adcnum, clockpin, mosipin, misopin, cspin):
if ((adcnum > 7) or (adcnum < 0)):
return -1
GPIO.output(cspin, True)
import json
import time
from boto import kinesis
auth = {
"aws_access_key_id":"AKIAJ7YDUGOXLHX2WWMQ",
"aws_secret_access_key":"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"
}
connection = kinesis.connect_to_region('us-west-2', **auth)
print connection.list_streams()
#stream_name = u'ArchimedesTaskStream'
stream_name = u'TestStream'
"""
tries = 0
while tries < 10:
tries += 1
time.sleep(1)
try:
response = connection.describe_stream(stream_name)
if response['StreamDescription']['StreamStatus'] == 'ACTIVE':
break
except :
logger.error('error while trying to describe kinesis stream : %s')
else:
raise TimeoutError('Stream is still not active, aborting...')
shard_ids = []
stream_name = None
if response and 'StreamDescription' in response:
#! /usr/bin/python
#import boto
import boto.kinesis as boto_kinesis
import datetime
import time
#kinesis = boto.connect_kinesis()
#streams = kinesis.list_streams()
#stream = kinesis.describe_stream('js-test')
kinesis = boto_kinesis.connect_to_region('us-west-2')
for i in xrange(1,100001):
time.sleep(0.3)
ct = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
#print 'The current time is \t'+ct
print kinesis.put_record('js-test','The current time is\t'+ct, ct)
from boto import kinesis
import datetime
import json
import csv
import configparser
config = configparser.ConfigParser()
ini = config.read('conf2.ini')
AWS_ACCESS_KEY_ID = config.get('AWS Credentials', 'key')
AWS_SECRET_ACCESS_KEY = config.get('AWS Credentials', 'secret')
kinesis = kinesis.connect_to_region("eu-west-1", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
print kinesis.list_streams()
print kinesis.describe_stream("rawdata")
initial_uid = 32767376
bruce = "awesome"
while bruce == "awesome":
with open("Kinesis_Test_Data.csv", 'rb') as source_file:
contents = csv.reader(source_file, delimiter=',', quotechar='|')
for event in contents:
data = dict()
initial_uid = initial_uid + 1
data['uid'] = initial_uid
# Markus Schmidberger, [email protected] # July 14, 2015 # Python Connector to Kinesis in same region # #################### # # Copyright 2014, Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Amazon Software License (the "License"). # You may not use this file except in compliance with the License. # A copy of the License is located at # # http://aws.amazon.com/asl/ # # or in the "license" file accompanying this file. This file is distributed # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. # ##################### from boto import kinesis import urllib # we assume that Kinesis is running in same region REGION = urllib.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read()[:-1] STREAMNAME = 'IoTSensorDemo' # Kinesis connection kinesisConn = kinesis.connect_to_region(REGION)
"direct_object": {
"filesystem_metrics": {
"size": size,
"free": free,
"available": avail
}
},
"at": get_event_time(),
"on": {
"server": {
"hostname": get_hostname()
}
}
})
def write_event(conn, stream_name):
event_id, event_payload = create_event()
event_json = json.dumps(event_payload)
conn.put_record(stream_name, event_json, event_id)
return event_id
if __name__ == '__main__': # a
conn = kinesis.connect_to_region(region_name="us-east-1",
profile_name="ulp")
while True: # b
event_id = write_event(conn, "events")
print "Wrote event: {}".format(event_id)
time.sleep(10) # c
import random
import time
from datetime import datetime
from boto import kinesis
def random_generator(size=6, chars=string.ascii_lowercase + string.digits):
return ''.join(random.choice(chars) for x in range(size))
#connecting to Kinesis stream
region = 'eu-west-1'
kinesisStreamName = 'BotoDemo'
kinesis = kinesis.connect_to_region(region)
# generating data and feeding kinesis.
while True:
y = random_generator(10,"techsummit2015")
urls = ['foo.com','amazon.com','testing.com','google.com','sydney.com']
x = random.randint(0,4)
userid = random.randint(25,35)+1200
now = datetime.now()
timeformatted = str(now.month) + "/" + str(now.day) + "/" + str(now.year) + " " + str(now.hour) + ":" +str(now.minute) + ":" + str(now.second)
def sim(header,linesACC,linesORI,SIMIP,RUNS,REGION,STREAMNAME,RANDOMSTART):
# Kinesis connection
kinesisConn = kinesis.connect_to_region(REGION)
# for every line create json and send to Kinesis
records = []
start = time.time()
for k in range(RUNS):
count = 0
if RANDOMSTART == True:
startline = random.randint(0,len(linesACC))
else:
startline = 0
for lineACC in linesACC[startline:]:
jsonstr = ''
liACC = lineACC.split(',')
i = 0
for l in liACC[1:]:
l=l.replace("\n","").replace("\"","")
if( headerACC[i] == "device" or headerACC[i] == "sensorname" or headerACC[i] == "cognitoId"):
dat = '"' + headerACC[i] + '":"' + l +'"'
else:
dat = '"' + headerACC[i] + '":' + l
jsonstr = jsonstr + "," + dat
i = i +1
jsonstr = "{"+ jsonstr[1:] + "}"
jsonobj = json.loads(jsonstr)
ts = time.time()
jsonobj["recordTime"] = ts
jsonobj["cognitoId"] = "sim:" + str(abs(hash(SIMIP)))
record = {'Data': json.dumps(jsonobj),'PartitionKey':str('screenAccG')}
records.append(record)
lineORI = linesORI[startline+count]
jsonstr = ''
liORI = lineORI.split(',')
i = 0
for l in liORI[1:]:
l=l.replace("\n","").replace("\"","")
if( headerORI[i] == "device" or headerORI[i] == "sensorname" or headerORI[i] == "cognitoId"):
dat = '"' + headerORI[i] + '":"' + l +'"'
else:
dat = '"' + headerORI[i] + '":' + l
jsonstr = jsonstr + "," + dat
i = i +1
jsonstr = "{"+ jsonstr[1:] + "}"
jsonobj = json.loads(jsonstr)
ts = time.time()
jsonobj["recordTime"] = ts
jsonobj["cognitoId"] = "sim:" + str(abs(hash(SIMIP)))
record = {'Data': json.dumps(jsonobj),'PartitionKey':str('screenAdjustedEvent')}
records.append(record)
count = count + 1
if( count % (SENDTIME/0.3) == 0):
kinesisConn.put_records(records, STREAMNAME)
records=[]
end = time.time()
if ( (end - start) < 1.0/3):
print("we are fast - make a break")
time.sleep( (1.0/3) - (end-start) )
start = time.time()
from boto import kinesis
import time
import json
import pickle
# AWS Connection Credentials
aws_access_key = '*****'
aws_access_secret = '*****'
# Selected Kinesis Stream from where to pull data
stream = 'test-dev'
# Aws Authentication
auth = {"aws_access_key_id": aws_access_key, "aws_secret_access_key": aws_access_secret}
conn = kinesis.connect_to_region('eu-west-1',**auth)
# Targeted file to be pushed to S3 bucket
fileName = "KinesisDataTest1.txt"
f = open("C:\\Users\\test\\Desktop\\data\\Kinesis_S3Data.json", "w+")
# Describe stream and get shards
tries = 0
while tries < 10:
tries += 1
time.sleep(1)
response = conn.describe_stream(stream)
if response['StreamDescription']['StreamStatus'] == 'ACTIVE':
break
else:
raise TimeoutError('Stream is still not active, aborting...')
