def init_broker_stuffs(self, logger): def log_error_and_raise(err_msg, logger): print(err_msg) logger.error(err_msg) raise Exception('Got error inside "init_broker_stuffs" function.') try: client = kinesis.connect_to_region( self.region, aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key ) if self.aws_access_key_id != '' \ and self.aws_secret_access_key != '' else kinesis.connect_to_region(self.region) stream_descriptor = client.describe_stream( stream_name=self.stream_name) self.stream_descriptor = stream_descriptor self.client = client except ResourceNotFoundException: if self.stream_descriptor is None: err_msg = 'Could not find {stream_name} stream.' \ .format(stream_name=self.stream_name) log_error_and_raise(err_msg, logger) return status = self.stream_descriptor['StreamDescription'][ 'StreamStatus'] if 'ACTIVE' not in status: err_msg = 'Stream status: {status}. Should be ACTIVE.' \ .format(status=status) log_error_and_raise(err_msg, logger) except Exception as e: raise e
def __init__(self, aws_region=None, access_key=None, secret_access_key=None): ''' Construtor :param aws_region: Nome da região global com a qual será feita a conexão :param access_key: Chave pública de acesso do usuário :param secret_access_key: Chave secreta de acesso do usuário ''' configWrapper = Configuration() self.config = configWrapper.getConfigurationFromFile() if access_key is not None and secret_access_key is not None and aws_region is not None: self.conn = kinesis.connect_to_region( aws_region, aws_access_key_id=access_key, aws_secret_access_key=secret_access_key) else: self.conn = kinesis.connect_to_region( self.config['aws']['awsAuth']['region'], aws_access_key_id=self.config['aws']['awsAuth']['access_key'], aws_secret_access_key=self.config['aws']['awsAuth'] ['secret_access_key'])
def put_record(stream_name, partition_key, message, cred_exp) : if (cred_exp == True) : auth = {"aws_access_key_id":aws_api_key, "aws_secret_access_key":aws_api_secret} kin = kinesis.connect_to_region("us-east-1", **auth) else : kin = kinesis.connect_to_region("us-east-1") print '@%s: %s' % (partition_key, message) kin.put_record(stream_name, message, partition_key)
def run(): from boto import kinesis kinesis = kinesis.connect_to_region("ca-central-1") #stream = kinesis.create_stream("ErikSparkPOC", 1) start = time.time() reddit = praw.Reddit(client_id='gluFwvMrQLqLuA', client_secret='nowLOmNuC8tS76mrc-LQUlarngw', user_agent='testscript by /u/plasmatrendybot', password='******', username='******') # Subreddit name subreddit = reddit.subreddit('askreddit') comments = subreddit.stream.comments() escape_limit = 100000 x = 0 for comment in comments: if comment.ups > 0: print(comment.body, comment.ups) reddit_comment = RedditComment( comment.body.encode('ascii', 'ignore')) try: kinesis.put_record("ErikSparkPOC", json.dumps(reddit_comment), "partitionkey") except: print("Failed to insert to Kinesis") x = x + 1 if x > escape_limit: break
def write_partition(partition): # Access the Kinesis client object kinesisClient = kinesis.connect_to_region("us-east-1") # Iterate over rows for row in partition: # Send the row as a JSON string into the Kinesis stream kinesisClient.put_record(kinesisStreamName, json.dumps(row),"partitionKey")
def generate_events(profile, region, stream): """ load demo data with python generator script for SimpleEvents """ conn = kinesis.connect_to_region(region, profile_name=profile) while True: event_json = write_event(conn, stream) print "Event sent to Kinesis: {}".format(event_json)
def connect_kinesis(): auth = {"aws_access_key_id":aws_access_key_id, "aws_secret_access_key":aws_secret_access_key} try: conn = kinesis.connect_to_region(region,**auth) return conn except Exception as e: print('KINESIS connection error: '+str(e)) sys.exit(1)
def put_data_into_stream(stream, region, sleep_duration, records_to_create): LOG.error("put_data_into_stream(%s, %s)", stream, region) connection = kinesis.connect_to_region(region) for i in range(records_to_create): data = uuid.uuid4().hex LOG.debug("Putting %s onto stream", data) res = connection.put_record(stream, data, data) LOG.debug(res) time.sleep(sleep_duration)
def write_partition(partition): # Access the Kinesis client object kinesisClient = kinesis.connect_to_region("us-east-1") # Iterate over rows for row in partition: # Send the row as a JSON string into the Kinesis stream kinesisClient.put_record(kinesisStreamName, json.dumps(row), "partitionKey")
def process_stream(stream, region): LOG.error("process_stream(%s, %s)", stream, region) connection = kinesis.connect_to_region(region) description = connection.describe_stream(stream) shard_ids = [shard['ShardId'] for shard in description['StreamDescription']['Shards']] for shard_id in shard_ids: t = Thread(target=log_stream_iterator, args=(stream, connection, shard_id,)) t.start()
def __init__(self, stream_name, region='us-east-2'): self.stream_name = stream_name self.region = region self.kinesis_conn = kinesis.connect_to_region( region) # Connect to a Kinesis stream in a specific region self.description = self.kinesis_conn.describe_stream(stream) self.shards = self.get_shards() self.first_shard_id = self.shards[0]["ShardId"] self.partition_key = 0
def __init__(self, **kwargs): """ Event tracker backend that uses a Kinesis Stream `streamName` is the stream to send the events to """ self.streamName = kwargs.get('streamName', None) self.regionName = kwargs.get('regionName', None) self.kinesis = kinesis.connect_to_region(region_name = self.regionName)
def __init__(self, num_partitions, topic, *args, **kwargs): self.brokertype = "kinesis" self.con = kinesis.connect_to_region(REGION) self.num_shards = num_partitions try: self.bulksize = kwargs["bulksize"] except: self.bulksize = 1 self.msg_bulk = [] self.topic = topic self._create_stream()
def main(args): if args.key is None or args.keysecret is None: kin = kinesis.connect_to_region(args.datacenter) else: kin = kinesis.connect_to_region(args.datacenter, aws_access_key_id=args.key, aws_secret_access_key=args.keysecret) shard_id = args.shardid shard_it = kin.get_shard_iterator(args.name, shard_id, args.iterator)['ShardIterator'] while True: out = kin.get_records(shard_it, limit=args.limit) for o in out['Records']: print(o['Data']) jdat = json.loads(o['Data']) sys.stdout.write('%s\n' % json.dumps(jdat)) sys.stdout.flush() shard_it = out['NextShardIterator'] time.sleep(args.interval)
def update_kinesis(project_model, message_type): """ notifies media atom of a project update or create by pushing a message onto its kinesis stream. the kinesis stream is indicated in settings. :param project_model: ProjectModel instance that has been created/updated :param message_type: either `media_atom.MSG_PROJECT_CREATED` or `media_atom.MSG_PROJECT_UPDATED` :return: """ from portal.plugins.gnm_vidispine_utils.vs_helpers import site_id from boto import sts, kinesis from django.conf import settings import json, logging SESSION_NAME = 'pluto-media-atom-integration' project_id = site_id + "-" + str(project_model.collection_id) logger.info("{0}: Project updated, notifying {1} via role {2}".format( project_id, settings.MEDIA_ATOM_STREAM_NAME, settings.MEDIA_ATOM_ROLE_ARN)) sts_connection = sts.STSConnection( aws_access_key_id=settings.MEDIA_ATOM_AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.MEDIA_ATOM_AWS_SECRET_ACCESS_KEY) assume_role_result = sts_connection.assume_role( role_arn=settings.MEDIA_ATOM_ROLE_ARN, role_session_name=SESSION_NAME) credentials = assume_role_result.credentials logger.debug("{0}: Got kinesis credentials".format(project_id)) kinesis_connection = kinesis.connect_to_region( region_name='eu-west-1', aws_access_key_id=credentials.access_key, aws_secret_access_key=credentials.secret_key, security_token=credentials.session_token) message_content = { 'type': message_type, 'id': project_id, 'title': project_model.gnm_project_headline, 'status': project_model.gnm_project_status, 'commissionId': site_id + "-" + str(project_model.commission.collection_id), 'commissionTitle': project_model.commission.gnm_commission_title, 'productionOffice': project_model.gnm_project_production_office, 'created': project_model.created.isoformat() } logger.debug("{0}: Message is {1}".format(project_id, message_content)) kinesis_connection.put_record(stream_name=settings.MEDIA_ATOM_STREAM_NAME, data=json.dumps(message_content), partition_key=project_id) logger.info("{0}: Project update sent".format(project_id))
def refresh_access_credentials(self): sts_conn = sts.connect_to_region( 'eu-west-1', aws_access_key_id=self._aws_access_key_id, aws_secret_access_key=self._aws_secret_access_key) credentials = sts_conn.assume_role(self.role_name, self.session_name) self._conn = kinesis.connect_to_region( 'eu-west-1', aws_access_key_id=credentials.credentials.access_key, aws_secret_access_key=credentials.credentials.secret_key, security_token=credentials.credentials.session_token)
def put_words_in_stream(w): conn = kinesis.connect_to_region(region_name='ap-northeast-2') stream_name = "datastream_2" conn.describe_stream(stream_name) w = json.dumps(w) try: conn.put_record(stream_name, w, "partitionkey") print("Put word: " + w + " into stream: " + stream_name) except Exception as e: sys.stderr.write( "Encountered an exception while trying to put a word: " + w + " into stream: " + stream_name + " exception was: " + str(e))
def put_aws(self, data): """ 数据及日志上传 :param data: 数据 :param stream_name: 流名称 :return: """ config = self.spider.config message = json.dumps(data).encode('utf-8') kinesis_client = kinesis.connect_to_region('cn-north-1') kinesis_client.put_record(config.get("KINESIS", "IMAGE_STREAM_NAME"), data=message, partition_key='partitionKey1')
def run(): from boto import kinesis kinesis = kinesis.connect_to_region("ca-central-1") #stream = kinesis.create_stream("ErikDemo", 1) i = 0 for i in xrange(10): user = User("Demo") #print(json.dumps(user)) print("Did I make it here?") kinesis.put_record("ErikDemo", json.dumps(user), "partitionkey")
def main(): region = 'us-west-2' aws_access_key_id = 'aws_access_key_id' aws_secret_access_key = 'aws_secret_access_key' client = kinesis.connect_to_region( region, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) stream_name = 'stream_name' stream_descriptor = client.describe_stream(stream_name=stream_name) shards = stream_descriptor['StreamDescription']['Shards'] for shard in shards: executor = Executor(client, stream_name, shard['ShardId']) executor.start()
def startStreaming(self): print("Starting stream.") self.kinesisConn = kinesis.connect_to_region( self.connectionDetails['kinesis']['region']) thread.start_new_thread(self.streamSbs1StreamAsync, ( self, config['sbs1']['portADSB'], )) thread.start_new_thread(self.streamSbs1StreamAsync, ( self, config['sbs1']['portMLAT'], )) while 1: pass
def test_simple_emit(self): # need to create a stream to test against conn = kinesis.connect_to_region("us-east-1") conn.create_stream(StreamName='my_stream', ShardCount=1) self.backend = KinesisBackend(streamName='my_stream') event = { 'name': 'bob uncle', 'context': { 'user_id': 'bob' }, 'data': { 'foo': 'test' } } self.backend.send(event)
def main(order_or_vendor): order_df = to_df(producer_var[order_or_vendor]["order_df_file_path"]) kinesis_put = kinesis.connect_to_region(producer_var[order_or_vendor]["kinesis_region"]) order_df_json = order_df.to_dict(orient="records") for i in range(len(order_df_json)): if i > 0: pprint.pprint(i) kinesis_put.put_record(producer_var[order_or_vendor]["order_stream_name"], json.dumps(order_df_json[i]).encode("utf-8"), "partitionkey") sleep_time = (pd.to_datetime(order_df_json[i]['datetime'], format="%Y-%m-%d %H:%M:%S") - pd.to_datetime( order_df_json[i - 1]['datetime'], format="%Y-%m-%d %H:%M:%S")).total_seconds() time.sleep(sleep_time) else: kinesis_put.put_record(producer_var[order_or_vendor]["order_stream_name"], json.dumps(order_df_json[i]).encode("utf-8"), "partitionkey")
def main(): from boto import kinesis kinesis = kinesis.connect_to_region("us-east-2") shard_id = 'shardId-000000000000' #we only have one shard! shard_it = kinesis.get_shard_iterator("end-stream", shard_id, "LATEST")["ShardIterator"] while 1 == 1: out = kinesis.get_records(shard_it, limit=1) shard_it = out["NextShardIterator"] if len(out["Records"]) > 0: print(out["Records"][0]["Data"]) data_dict = json.loads(out["Records"][0]["Data"]) timestamp, userName, device_name, detail_result = extract_data_from_kinesis_stream( data_dict) # print(shard_it) write_to_db(timestamp, userName, device_name, detail_result) time.sleep(0.1)
def handle(self, *args, **options): pprint(options) if 'aws_access_key_id' in options and 'aws_secret_access_key' in options: sts_conn = sts.connect_to_region( 'eu-west-1', aws_access_key_id=options['aws_access_key_id'], aws_secret_access_key=options['aws_secret_access_key']) else: sts_conn = sts.connect_to_region('eu-west-1') credentials = sts_conn.assume_role(self.role_name, self.session_name) pprint(credentials.credentials.__dict__) conn = kinesis.connect_to_region( 'eu-west-1', aws_access_key_id=credentials.credentials.access_key, aws_secret_access_key=credentials.credentials.secret_key, security_token=credentials.credentials.session_token) streaminfo = conn.describe_stream(self.stream_name) pprint(streaminfo) threadlist = map( lambda shardinfo: self.startup_thread(credentials.credentials, shardinfo), streaminfo['StreamDescription']['Shards']) print "Stream {0} has {1} shards".format(self.stream_name, len(threadlist)) for t in threadlist: t.daemon = True t.start() print "Started up and processing. Hit CTRL-C to stop." #simplest way to allow ctrl-C when dealing with threads try: while True: sleep(3600) except KeyboardInterrupt: print "CTRL-C caught, cleaning up"
def put_data(self): region = self.get_region() stream_name = self.get_kinesis_stream_name() conn = kinesis.connect_to_region(region_name=region) try: # Check stream status status = self.get_stream_status(conn, stream_name) if 'DELETING' == status: print('The stream: {s} is being deleted, please rerun the script.'.format(s=stream_name)) sys.exit(1) elif 'ACTIVE' != status: self.wait_for_stream(conn, stream_name) # put data into stream data = self.get_mock_data(israndom= False) self.put_record_in_stream(conn, stream_name, data) data = self.get_mock_data(israndom=True) self.put_record_in_stream(conn, stream_name, data) except Exception as error: print('{}'.format(error)) sys.exit(1)
def main(args): kin = kinesis.connect_to_region(args.datacenter, aws_access_key_id=args.key, aws_secret_access_key=args.keysecret) shard_id = args.shardid shard_it = kin.get_shard_iterator(args.name, shard_id, args.iterator)['ShardIterator'] while True: result = [] out = kin.get_records(shard_it, limit=args.limit) for o in out['Records']: result.append(o['Data'].decode("utf-8")) sys.stdout.write('%s\n' % o['Data']) jdat = json.loads(o['Data'].decode("utf-8")) sys.stdout.write('%s\n' % json.dumps(jdat)) sys.stdout.flush() sys.stdout.write('%s\n' % result) sys.stdout.flush() shard_it = out['NextShardIterator'] time.sleep(args.interval)
def upload_kinesis(): connection = kinesis.connect_to_region('us-east-1') tries = 0 while tries < 10: tries += 1 time.sleep(1) try: response = connection.describe_stream('Temperature') if response['StreamDescription']['StreamStatus'] == 'ACTIVE': break except: logger.error('error while trying to describe kinesis stream : %s') else: raise TimeoutError('Stream is still not active, aborting...') data = tempData() print(data) try: connection.put_record("Temperature", json.dumps(data), "partitionkey") except IOError: print ErrorString
def main(order_or_vendor): engine = create_engine( "mysql+mysqlconnector://root:toor@localhost:3306/grab") taxi_zone_lookup = pd.read_csv("data/taxi_zone_lookup.csv", index_col="Unnamed: 0") def timeseries2loc(msg): ts_df = pd.DataFrame([json.loads(i["Data"]) for i in a["Records"]]) print(ts_df) order_timeseries_with_gps = ts_df.join(taxi_zone_lookup, on='locationID', how="left", lsuffix="_1") order_timeseries_with_gps["datetime"] = pd.to_datetime( order_timeseries_with_gps["datetime"]) print(order_timeseries_with_gps) order_timeseries_with_gps.to_sql( con=engine, name=consumer_var[order_or_vendor]["sql_table_name"], if_exists="append", chunksize=1000) return order_timeseries_with_gps kinesis_get = kinesis.connect_to_region(kinesis_region) shard_id = 'shardId-000000000000' shard_it = kinesis_get.get_shard_iterator( consumer_var[order_or_vendor]["order_stream_name"], shard_id, "LATEST")["ShardIterator"] while True: a = kinesis_get.get_records(shard_it) # print (a) shard_it = a["NextShardIterator"] print("get %d records" % len(a["Records"])) if len(a["Records"]) > 0: pprint.pprint(a["Records"][0]) timeseries2loc(a) # print (">>>>>next >>>> \t"+shard_it) time.sleep(2)
def __init__(self, consumer_name, stream, shard=None, starting_point=None, region='ap-southeast-1'): self.connection = kinesis.connect_to_region(region, **kinesis_auth) self.stream = stream self.shard = shard self.shard_starting_point = SpoolManager("kinesis_sync") _, self.starting_point = self.shard_starting_point.get_file_point_file( ) self.starting_point = None shards = self._list_shards(stream) if self.shard: for _shard in shards: if _shard["ShardId"] == shard: break else: self.shard = shards[0]["ShardId"] self.shard_iterator = self._get_stream_iterator( self.stream, self.shard)
def main(): from boto import kinesis kinesis = kinesis.connect_to_region("us-east-2") shard_id = 'shardId-000000000000' #we only have one shard! shard_it = kinesis.get_shard_iterator("end-stream", shard_id, "LATEST")["ShardIterator"] timestamps = [i for i in range(20)] bodyorientationpitches = [i for i in range(20)] steps = [i for i in range(20)] while 1 == 1: out = kinesis.get_records(shard_it, limit=1) shard_it = out["NextShardIterator"] if len(out["Records"]) > 0: print(out["Records"][0]["Data"]) data_dict = json.loads(out["Records"][0]["Data"]) timestamp, userName, device_name, detail_result = extract_data_from_kinesis_stream( data_dict) timestamps.insert(0, timestamp) timestamps = timestamps[:-1] bodyorientationpitches.insert( 0, detail_result["IMU"]["BodyOrientationPitch"]) bodyorientationpitches = bodyorientationpitches[:-1] steps.insert(0, detail_result["IMU"]["StepCount"]) steps = steps[:-1] x1 = np.array(timestamps) y1 = np.array(bodyorientationpitches) x2 = np.array(timestamps) y2 = np.array(steps) plot_vals(x1, y1, x2, y2)
def create_write(region, stream_name, key_func): conn = kinesis.connect_to_region(region_name = region) try: status = _get_stream_status(conn, stream_name) if 'DELETING' == status: raise KinesisException('The stream: {s} is being deleted.'.format(s=stream_name)) elif 'ACTIVE' != status: _wait_for_stream(conn, stream_name) except: raise KinesisException('The stream: {s} is being deleted.'.format(s=stream_name)) def write(event): key = key_func(event) json_event = json.dumps(event) try: conn.put_record(stream_name, json_event, key) except Exception as e: raise KinesisException("Encountered an exception while trying to put: " + event + " into stream: " + stream_name + " exception was: " + str(e)) return write
from boto import kinesis import mysql.connector import time import json mydb = mysql.connector.connect( host=HOST, user=USER, passwd=PASSWORD, db=DB ) cursor = mydb.cursor() kinesis = kinesis.connect_to_region('us-east-1') shard_id = 'shardId-0000000000000' shard_it = kinesis.get_shard_iterator(MYSTREAM, shard_id, 'LATEST')['ShardIterator'] while 1==1: print("getting record") out = kinesis.get_records(shard_it,limit=1) for i in out['Records']: recordId = json.loads(i['Data'])['SaleId']['N'] productId = json.loads(i['Data'])['ProductId']['N'] quantity = json.loads(i['Data'])['Quantity']['N'] saleDate = json.loads(i['Data'])['SaleDate']['S']
help="The region you'd like to make this stream in. Default is 'us-east-1'", metavar="REGION_NAME",) parser.add_argument("-w", "--word", dest="words", default=[], action="append", help="A word to add to the stream. Can be specified multiple times to add multiple words.", metavar="WORD",) parser.add_argument("-p", "--period", dest="period", type=int, help="If you'd like to repeatedly put words into the stream, this option provides the period for putting " + "words into the stream in SECONDS. If no period is given then the words are put once.", metavar="SECONDS",) args = parser.parse_args() stream_name = args.stream_name ''' Getting a connection to Amazon Kinesis will require that you have your credentials available to one of the standard credentials providers. ''' print("Connecting to stream: {s} in {r}".format(s=stream_name, r=args.region)) conn = kinesis.connect_to_region(region_name = args.region) try: status = get_stream_status(conn, stream_name) if 'DELETING' == status: print('The stream: {s} is being deleted, please rerun the script.'.format(s=stream_name)) sys.exit(1) elif 'ACTIVE' != status: wait_for_stream(conn, stream_name) except: # We'll assume the stream didn't exist so we will try to create it with just one shard conn.create_stream(stream_name, 1) wait_for_stream(conn, stream_name) # Now the stream should exist if len(args.words) == 0: print('No -w options provided. Waiting on input from STDIN') words = [l.strip() for l in sys.stdin.readlines() if l.strip() != '']
from boto import kinesis aws_profile = "learner" kinesis = kinesis.connect_to_region("us-east-1") description = kinesis.describe_stream("pharmacy-records") print(description)
from boto import kinesis import testdata,json # Creating fake data class Users(testdata.DictFactory): firstname = testdata.FakeDataFactory("firstName") lastname = testdata.FakeDataFactory("lastName") age = testdata.RandomInteger(10,30) gender = testdata.RandomSelection(['female','male']) # Using boto connect to the region in which your kinesis stream is created kinesis = kinesis.connect_to_region("eu-west-1") for user in Users().generate(50): print user kinesis.put_record("EdisonDemo", json.dumps(user), "partitionkey")
from boto import kinesis import sys # Go to http://apps.twitter.com and create an app. # The consumer key and secret will be generated for you after consumer_key="" consumer_secret="" # After the step above, you will be redirected to your app's page. # Create an access token under the the "Your access token" section access_token="" access_token_secret="" #Kinesis stream connection parameters #Shard names would be dynamically created for applications looking to scale out the stream conn = kinesis.connect_to_region(region_name = "us-east-1") stream_name = "" shard_name = "shard1" def put_tweet_in_stream(tweet): """ Put each tweet into the kinesis stream :type tweet: str """ try: tweet_text = json.loads(tweet)['text'] conn.put_record(stream_name,tweet,shard_name) except Exception as e: sys.stderr.write("Encountered an exception while trying to put a tweet: " + tweet + " into stream: " + stream_name + " exception was: " + str(e))
}, "verb": "read", "direct_object": { "filesystem_metrics": { "size": size, "free": free, "available": avail } }, "at": get_event_time(), "on": { "server": { "hostname": get_hostname() } } }) def write_event(conn, stream_name): event_id, event_payload = create_event() event_json = json.dumps(event_payload) conn.put_record(stream_name, event_json, event_id) return event_id if __name__ == '__main__': # a conn = kinesis.connect_to_region(region_name="us-east-1", profile_name="ulp") while True: # b event_id = write_event(conn, "events") print "Wrote event: {}".format(event_id) time.sleep(10) # c
def init_kinesis(): k = kinesis.connect_to_region(KINESIS_AWS_REGION) if KINESIS_STREAM_NAME not in k.list_streams()['StreamNames']: k.create_stream(KINESIS_STREAM_NAME, 1) return k
lines = text.split("\n") header = lines[0] m = re.match(r"<doc id=\"([0-9]+)\" url=\"(.+)\" title=\"(.+)\"", header) if m: article_id = int(m.group(1)) article_text = "\n".join(lines[1:-1]) doc = {"article_id": str(uuid.uuid4()), "url": m.group(2), "title": m.group(3), "body": article_text} if INSERT_IN_BATCHES: doc_buffer.append(json.dumps(doc)) if len(doc_buffer) == BATCH_SIZE: kinesis.put_records(kinesis_stream, doc_buffer, str(uuid.uuid4())) doc_buffer = [] else: kinesis.put_record(kinesis_stream, json.dumps(doc), str(uuid.uuid4())) print "Sent article '{}' ({})".format(doc["title"], doc["article_id"]) else: print "Header not found in {0}".format(article_file) if __name__ == "__main__": processed_count = 0 doc_buffer = [] start_time = time.time() data_dir = "data" kinesis_stream = "lambdastorm_input" kinesis = kinesis.connect_to_region(AWS_REGION) process(data_dir, NUM_PROCESSES)
import cv2 import numpy as np import sys import pickle as cPickle #import struct #from multiprocessing import Pool import pytz ## for timezone calculations import datetime aws_region = "us-west-2" stream_name = "TestStream" cap = cv2.VideoCapture(0) #pool = Pool(processes=2) kinesis = kinesis.connect_to_region(aws_region) camera_index = 0 # 0 is usually the built-in webcam capture_rate = 2 # Frame capture rate.. every X frames. Positive integer. rekog_max_labels = 123 rekog_min_conf = 50.0 frame_count = 0 ## Old code #************************************************************************ #for line in x.iter_lines(): # kinesis = kinesis.connect_to_region(aws_region) # kinesis.put_record(stream_name, line, "partitionkey") # if line: # print (line) #************************************************************************
#!/usr/bin/env python import time import os import RPi.GPIO as GPIO from boto import kinesis GPIO.setmode(GPIO.BCM) DEBUG = 1 # set values STREAM_NAME = 'rPi_v1' # connect to oregon conn = kinesis.connect_to_region('us-west-2') response = conn.describe_stream(STREAM_NAME) shard_ids = [] stream_name = None if response and 'StreamDescription' in response: stream_name = response['StreamDescription']['StreamName'] for shard_id in response['StreamDescription']['Shards']: shard_id = shard_id['ShardId'] shard_iterator = conn.get_shard_iterator(stream_name, shard_id, shard_iterator_type) shard_ids.append({'shard_id' : shard_id ,'shard_iterator' : shard_iterator['ShardIterator'] }) # read SPI data from MCP3008 chip, 8 possible adc's (0 thru 7) def readadc(adcnum, clockpin, mosipin, misopin, cspin): if ((adcnum > 7) or (adcnum < 0)): return -1 GPIO.output(cspin, True)
import json import time from boto import kinesis auth = { "aws_access_key_id":"AKIAJ7YDUGOXLHX2WWMQ", "aws_secret_access_key":"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" } connection = kinesis.connect_to_region('us-west-2', **auth) print connection.list_streams() #stream_name = u'ArchimedesTaskStream' stream_name = u'TestStream' """ tries = 0 while tries < 10: tries += 1 time.sleep(1) try: response = connection.describe_stream(stream_name) if response['StreamDescription']['StreamStatus'] == 'ACTIVE': break except : logger.error('error while trying to describe kinesis stream : %s') else: raise TimeoutError('Stream is still not active, aborting...') shard_ids = [] stream_name = None if response and 'StreamDescription' in response:
#! /usr/bin/python #import boto import boto.kinesis as boto_kinesis import datetime import time #kinesis = boto.connect_kinesis() #streams = kinesis.list_streams() #stream = kinesis.describe_stream('js-test') kinesis = boto_kinesis.connect_to_region('us-west-2') for i in xrange(1,100001): time.sleep(0.3) ct = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') #print 'The current time is \t'+ct print kinesis.put_record('js-test','The current time is\t'+ct, ct)
from boto import kinesis import datetime import json import csv import configparser config = configparser.ConfigParser() ini = config.read('conf2.ini') AWS_ACCESS_KEY_ID = config.get('AWS Credentials', 'key') AWS_SECRET_ACCESS_KEY = config.get('AWS Credentials', 'secret') kinesis = kinesis.connect_to_region("eu-west-1", aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY) print kinesis.list_streams() print kinesis.describe_stream("rawdata") initial_uid = 32767376 bruce = "awesome" while bruce == "awesome": with open("Kinesis_Test_Data.csv", 'rb') as source_file: contents = csv.reader(source_file, delimiter=',', quotechar='|') for event in contents: data = dict() initial_uid = initial_uid + 1 data['uid'] = initial_uid
# Markus Schmidberger, [email protected] # July 14, 2015 # Python Connector to Kinesis in same region # #################### # # Copyright 2014, Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Amazon Software License (the "License"). # You may not use this file except in compliance with the License. # A copy of the License is located at # # http://aws.amazon.com/asl/ # # or in the "license" file accompanying this file. This file is distributed # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. # ##################### from boto import kinesis import urllib # we assume that Kinesis is running in same region REGION = urllib.urlopen('http://169.254.169.254/latest/meta-data/placement/availability-zone').read()[:-1] STREAMNAME = 'IoTSensorDemo' # Kinesis connection kinesisConn = kinesis.connect_to_region(REGION)
"direct_object": { "filesystem_metrics": { "size": size, "free": free, "available": avail } }, "at": get_event_time(), "on": { "server": { "hostname": get_hostname() } } }) def write_event(conn, stream_name): event_id, event_payload = create_event() event_json = json.dumps(event_payload) conn.put_record(stream_name, event_json, event_id) return event_id if __name__ == '__main__': # a conn = kinesis.connect_to_region(region_name="us-east-1", profile_name="ulp") while True: # b event_id = write_event(conn, "events") print "Wrote event: {}".format(event_id) time.sleep(10) # c
import random import time from datetime import datetime from boto import kinesis def random_generator(size=6, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for x in range(size)) #connecting to Kinesis stream region = 'eu-west-1' kinesisStreamName = 'BotoDemo' kinesis = kinesis.connect_to_region(region) # generating data and feeding kinesis. while True: y = random_generator(10,"techsummit2015") urls = ['foo.com','amazon.com','testing.com','google.com','sydney.com'] x = random.randint(0,4) userid = random.randint(25,35)+1200 now = datetime.now() timeformatted = str(now.month) + "/" + str(now.day) + "/" + str(now.year) + " " + str(now.hour) + ":" +str(now.minute) + ":" + str(now.second)
def sim(header,linesACC,linesORI,SIMIP,RUNS,REGION,STREAMNAME,RANDOMSTART): # Kinesis connection kinesisConn = kinesis.connect_to_region(REGION) # for every line create json and send to Kinesis records = [] start = time.time() for k in range(RUNS): count = 0 if RANDOMSTART == True: startline = random.randint(0,len(linesACC)) else: startline = 0 for lineACC in linesACC[startline:]: jsonstr = '' liACC = lineACC.split(',') i = 0 for l in liACC[1:]: l=l.replace("\n","").replace("\"","") if( headerACC[i] == "device" or headerACC[i] == "sensorname" or headerACC[i] == "cognitoId"): dat = '"' + headerACC[i] + '":"' + l +'"' else: dat = '"' + headerACC[i] + '":' + l jsonstr = jsonstr + "," + dat i = i +1 jsonstr = "{"+ jsonstr[1:] + "}" jsonobj = json.loads(jsonstr) ts = time.time() jsonobj["recordTime"] = ts jsonobj["cognitoId"] = "sim:" + str(abs(hash(SIMIP))) record = {'Data': json.dumps(jsonobj),'PartitionKey':str('screenAccG')} records.append(record) lineORI = linesORI[startline+count] jsonstr = '' liORI = lineORI.split(',') i = 0 for l in liORI[1:]: l=l.replace("\n","").replace("\"","") if( headerORI[i] == "device" or headerORI[i] == "sensorname" or headerORI[i] == "cognitoId"): dat = '"' + headerORI[i] + '":"' + l +'"' else: dat = '"' + headerORI[i] + '":' + l jsonstr = jsonstr + "," + dat i = i +1 jsonstr = "{"+ jsonstr[1:] + "}" jsonobj = json.loads(jsonstr) ts = time.time() jsonobj["recordTime"] = ts jsonobj["cognitoId"] = "sim:" + str(abs(hash(SIMIP))) record = {'Data': json.dumps(jsonobj),'PartitionKey':str('screenAdjustedEvent')} records.append(record) count = count + 1 if( count % (SENDTIME/0.3) == 0): kinesisConn.put_records(records, STREAMNAME) records=[] end = time.time() if ( (end - start) < 1.0/3): print("we are fast - make a break") time.sleep( (1.0/3) - (end-start) ) start = time.time()
from boto import kinesis import time import json import pickle # AWS Connection Credentials aws_access_key = '*****' aws_access_secret = '*****' # Selected Kinesis Stream from where to pull data stream = 'test-dev' # Aws Authentication auth = {"aws_access_key_id": aws_access_key, "aws_secret_access_key": aws_access_secret} conn = kinesis.connect_to_region('eu-west-1',**auth) # Targeted file to be pushed to S3 bucket fileName = "KinesisDataTest1.txt" f = open("C:\\Users\\test\\Desktop\\data\\Kinesis_S3Data.json", "w+") # Describe stream and get shards tries = 0 while tries < 10: tries += 1 time.sleep(1) response = conn.describe_stream(stream) if response['StreamDescription']['StreamStatus'] == 'ACTIVE': break else: raise TimeoutError('Stream is still not active, aborting...')