def records_iterator(self, limit=100):
        """
        Read the latest batch of records from a Kinesis stream

        """
        shard_it = kinesis.get_shard_iterator(self.stream_name,
                                              self.first_shard_id,
                                              "LATEST")["ShardIterator"]
        while True:
            t0 = time.time()
            out = kinesis.get_records(shard_it, limit=limit)
            shard_it = out["NextShardIterator"]
            yield out

            t1 = time.time()
            if t1 - t0 < 0.2:
                time.sleep(0.2 - (t1 - t0))
def main():
    from boto import kinesis
    kinesis = kinesis.connect_to_region("us-east-2")
    shard_id = 'shardId-000000000000'  #we only have one shard!
    shard_it = kinesis.get_shard_iterator("end-stream", shard_id,
                                          "LATEST")["ShardIterator"]

    while 1 == 1:
        out = kinesis.get_records(shard_it, limit=1)
        shard_it = out["NextShardIterator"]
        if len(out["Records"]) > 0:
            print(out["Records"][0]["Data"])
            data_dict = json.loads(out["Records"][0]["Data"])
            timestamp, userName, device_name, detail_result = extract_data_from_kinesis_stream(
                data_dict)
            # print(shard_it)
            write_to_db(timestamp, userName, device_name, detail_result)
        time.sleep(0.1)
Beispiel #3
0
def main():
    from boto import kinesis
    kinesis = kinesis.connect_to_region("us-east-2")
    shard_id = 'shardId-000000000000'  #we only have one shard!
    shard_it = kinesis.get_shard_iterator("end-stream", shard_id,
                                          "LATEST")["ShardIterator"]
    timestamps = [i for i in range(20)]
    bodyorientationpitches = [i for i in range(20)]
    steps = [i for i in range(20)]

    while 1 == 1:
        out = kinesis.get_records(shard_it, limit=1)
        shard_it = out["NextShardIterator"]
        if len(out["Records"]) > 0:
            print(out["Records"][0]["Data"])
            data_dict = json.loads(out["Records"][0]["Data"])
            timestamp, userName, device_name, detail_result = extract_data_from_kinesis_stream(
                data_dict)

            timestamps.insert(0, timestamp)
            timestamps = timestamps[:-1]

            bodyorientationpitches.insert(
                0, detail_result["IMU"]["BodyOrientationPitch"])
            bodyorientationpitches = bodyorientationpitches[:-1]

            steps.insert(0, detail_result["IMU"]["StepCount"])
            steps = steps[:-1]

            x1 = np.array(timestamps)
            y1 = np.array(bodyorientationpitches)

            x2 = np.array(timestamps)
            y2 = np.array(steps)

            plot_vals(x1, y1, x2, y2)

mydb = mysql.connector.connect(
  host=HOST,
  user=USER,
  passwd=PASSWORD,
  db=DB
)

cursor = mydb.cursor()

kinesis = kinesis.connect_to_region('us-east-1')

shard_id = 'shardId-0000000000000'

shard_it = kinesis.get_shard_iterator(MYSTREAM, shard_id, 'LATEST')['ShardIterator']



while 1==1:
  print("getting record")
  out = kinesis.get_records(shard_it,limit=1)
  for i in out['Records']:
    recordId = json.loads(i['Data'])['SaleId']['N']
    productId = json.loads(i['Data'])['ProductId']['N']
    quantity = json.loads(i['Data'])['Quantity']['N']
    saleDate = json.loads(i['Data'])['SaleDate']['S']
    inv_quantity = 0 - int(quantity)

    add_record = "INSERT INTO FactInventory (idFactInventory, ProductID, QuantityChange, DateTime) VALUES (%s, %s, %s, %s)"
data_record = {
from boto import kinesis
import time

# Using boto connect to the region in which your kinesis stream is created
kinesis = kinesis.connect_to_region("eu-west-1")


shard_id = 'shardId-000000000000' 
# Iterator to go throough the latest stream values
shard_it = kinesis.get_shard_iterator("BotoDemo", shard_id, "LATEST")["ShardIterator"]

# Get thed ata 
while True:
    out = kinesis.get_records(shard_it, limit=2)
    shard_it = out["NextShardIterator"]
    print out
    time.sleep(0.3)
Beispiel #6
0
from boto import kinesis
import time

if __name__ == '__main__':
	kinesis = kinesis.connect_to_region("ap-southeast-1")
	print kinesis.describe_stream("payments")
	print kinesis.list_streams()

	shard_id = 'shardId-000000000000'
	shard_it = kinesis.get_shard_iterator("payments", shard_id, "LATEST")["ShardIterator"]
	while 1==1:
		out = kinesis.get_records(shard_it, limit=2)
		shard_it = out["NextShardIterator"]
		print out
		time.sleep(0.2)
from boto import kinesis
import time

# boto kinesis API
kinesis = kinesis.connect_to_region("eu-west-1")

# Stream to read from
stream = 'logbuffer-dev'

# Lookup shards of stream
shards = kinesis.describe_stream(stream)["StreamDescription"]["Shards"]
n_shards = len(shards)
iterators = []

# create iterators for each shard
for shard in shards:
	shard_id = shard["ShardId"]
#	iterators.append(kinesis.get_shard_iterator(stream, shard_id, "AT_SEQUENCE_NUMBER", shard["SequenceNumberRange"]["StartingSequenceNumber"])["ShardIterator"])
	iterators.append(kinesis.get_shard_iterator(stream, shard_id, "LATEST")["ShardIterator"])

# circular query of iterators
while 1==1:
	iterator = iterators.pop(0)
	out = kinesis.get_records(iterator, limit=500)
	iterators.append(out["NextShardIterator"])
	for record in out["Records"]:
		print(record["Data"])
	
	# Kinesis Limit: 5 transactions per second per shard for reads 
	time.sleep(0.2/n_shards)
  response = kinesis.describe_stream(stream_name)

  # Is the stream active?
  if response['StreamDescription']['StreamStatus'] == 'ACTIVE':
    # Yes, the stream is active and we are ready to start reading from it.  Get the shard ID.
    shard_id = response['StreamDescription']['Shards'][0]['ShardId']
    break

  # The stream is not active, wait for 15 seconds and try again
  time.sleep(15)
else:
  # The stream took too long to become active
  raise TimeoutError('Stream is still not active, aborting...')

# Get the shard iterator and get only new data (TRIM_HORIZON)
response = kinesis.get_shard_iterator(stream_name, shard_id, 'TRIM_HORIZON')
shard_iterator = response['ShardIterator']

# Loop forever
while True:
  # Get the next set of records
  response = kinesis.get_records(shard_iterator)

  # Get the next shard iterator
  shard_iterator = response['NextShardIterator']

  # Loop through all of the records and print them
  for line in response['Records']:
    print line

  # Sleep for a second so we don't hit the Kinesis API too fast
Beispiel #9
0
from boto import kinesis
import time

kinesis = kinesis.connect_to_region("eu-west-1")
shard_id = 'shardId-000000000000'  #we only have one shard!
shard_it = kinesis.get_shard_iterator("BotoDemo", shard_id,
                                      "LATEST")["ShardIterator"]
while 1 == 1:
    out = kinesis.get_records(shard_it, limit=2)
    shard_it = out["NextShardIterator"]
    print out
    time.sleep(0.2)
Beispiel #10
0
from boto import kinesis
import time

kinesis = kinesis.connect_to_region("us-east-2")
shard_id = 'shardId-000000000000'  #we only have one shard!
shard_it = kinesis.get_shard_iterator("end-stream", shard_id,
                                      "LATEST")["ShardIterator"]
while 1 == 1:
    out = kinesis.get_records(shard_it, limit=1)
    shard_it = out["NextShardIterator"]
    if len(out["Records"]) > 0:
        print(out["Records"][0]["Data"])
    # print(shard_it)
    time.sleep(0.3)

# {'MillisBehindLatest': 0,
# 'NextShardIterator': 'AAAAAAAAAAFF9/mBhBuL7fpnx+/w6YomDqex3tSNgI9ZI57g7i92nH2A/yTng4OQSTOtYbSHzwy+KA32ezEV/32oAYrMfkRuDQnnIvJvbaTBCDZdQSJLL+UTZthKkGUNoSwh8iGEohXHZjnra1R/Ky2JQwD/RU/WyWKVxhnsGiyHfltq72rgQ5y5raMhI4i0XGrY6Cc3kK4Ieu8V0rikCfNVl7Pjz26n',
# 'Records':
# [
# 	{
# 	'ApproximateArrivalTimestamp': 1597406867.724,
# 	'Data': '{"deviceID": 59, "userName": "******", "timeStamp": 1597406867, "param1": 9, "param2": 7, "param3": 2, "param4": 9}', 'PartitionKey': 'partitionkey', 'SequenceNumber': '49609830172400645887225606482176990396391791989677359106'}]}
Beispiel #11
0
partitionKey="IoTExample"
shardCount=1
iterator_type='LATEST'

stream=kinesis.describe_stream(streamName)
print(json.dumps(stream,sort_keys=True,indent=2,separators=(',',':')))
shards=stream['StreamDescription']['Shards']
print('# Shard Count:', len(shards))

def processRecords(records):
    for record in records:
        text=record['Data'].lower()
        print 'Processing record with data:' + text

i=0
response=kinesis.get_shard_iterator(streamName,shards[0]['ShardId'], 'TRIM_HORIZON',starting_sequence_number=None)
next_iterator=response['ShardIterator']
print('Gettinng next records using iterator:', next_iterator)
while i<4000:
    try:
       response=kinesis.get_records(next_iterator,limit=1)
             #print response
       if len(response['Records'])>0:
             #print 'Number of records fetched:' + str(len(response['Records']))
             processRecords(response['Records'])

       next_iterator=response['NextShardIterator']
       time.sleep(1)
       i=i+1

    except ProvisionedThroughputExceededException as ptee:
Beispiel #12
0
ws.write(i, 3, "clicou")

'''
      informando region que vamos conectar
'''
kinesis = kinesis.connect() .connect_to_region("us-east-1")

'''
      iniciando shard_id
'''
shard_id = 'shardId-000000000000'

'''
      Informando de onde vamos pegar os dados e como vamos iterar 
'''
shard_it = kinesis.get_shard_iterator("big-data-analytics-desafio", shard_id, "LATEST")["ShardIterator"]

result = {}
t = 0

'''
      iniciando while que vai parar de rodar apenas quando pegarmos a quantidade de registros validos informado no inicio
'''
while i == 0:
    '''
          atribuindo uma variavel com valores coletados e ja buscando o proximo registro
    '''
    out = kinesis.get_records(shard_it)
    shard_it = out["NextShardIterator"]

    '''
from boto import kinesis
import time

kinesis = kinesis.connect_to_region("us-west-2")
stream_name = "TestStream"
tries = 0
while tries < 100:
    tries += 1
    try:
        response = kinesis.describe_stream(stream_name)
        #print(response)
        if response['StreamDescription']['StreamStatus'] == 'ACTIVE':
            print("stream is active")
            shards = response['StreamDescription']['Shards']
            for shard in shards:
                shard_id = shard["ShardId"]
                print(repr(shard))
                shard_it = kinesis.get_shard_iterator(
                    stream_name, shard_id, "LATEST")["ShardIterator"]
                while True:
                    out = kinesis.get_records(shard_it, limit=2)
                    for o in out["Records"]:
                        print(o["Data"])
                        # You specific data processing goes here

                    shard_it = out["NextShardIterator"]
    except:
        print('error while trying to describe kinesis stream : %s')
bruce = "awesome"

while bruce == "awesome":
    with open("Kinesis_Test_Data.csv", 'rb') as source_file:
        contents = csv.reader(source_file, delimiter=',', quotechar='|')

        for event in contents:
            data = dict()

            initial_uid = initial_uid + 1
            data['uid'] = initial_uid
            data['event'] = event[0]
            data['timestamp'] = str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
            data['unit'] = event[1]
            data['package'] = event[2]
            data['price'] = event[3]
            data['platform'] = event[4]

            json_data = json.dumps(data, ensure_ascii=False)

            print json_data

            kinesis.put_record("rawdata", json_data, "partitionkey")


    shard_id = 'shardId-000000000000'
    shard_it = kinesis.get_shard_iterator("rawdata", shard_id, "LATEST")["ShardIterator"]
    print shard_it

from boto import kinesis

from settings import KINESIS_REGION, KINESIS_STREAM_NAME

FLUSH_INTERVAL = 5
BATCH_SIZE = 20
running = True


def process_messages(batch_msgs):
    print('messages processed: {}'.format(len(batch_msgs['Records'])))
    for msg in batch_msgs['Records']:
        print('message: "{}" offset: {}'.format(msg['Data'], msg['SequenceNumber']))


print('Connect to Kinesis Streams')
kinesis = kinesis.connect_to_region(region_name=KINESIS_REGION)
stream = kinesis.describe_stream(KINESIS_STREAM_NAME)
shardId = stream['StreamDescription']['Shards'][0]['ShardId']
shardIterator = kinesis.get_shard_iterator(KINESIS_STREAM_NAME, shardId, 'TRIM_HORIZON')

print('Kinesis consumer started!')
while running:
    msgs = kinesis.get_records(shardIterator['ShardIterator'], limit=BATCH_SIZE)

    process_messages(msgs)
    shardIterator['ShardIterator'] = msgs['NextShardIterator']

    print('\nnext batch in {} seconds...'.format(FLUSH_INTERVAL))
    time.sleep(FLUSH_INTERVAL)