-
Notifications
You must be signed in to change notification settings - Fork 0
/
EC2ProcessData.py
44 lines (34 loc) · 1.13 KB
/
EC2ProcessData.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
from boto import kinesis
import mysql.connector
import time
import json
mydb = mysql.connector.connect(
host=HOST,
user=USER,
passwd=PASSWORD,
db=DB
)
cursor = mydb.cursor()
kinesis = kinesis.connect_to_region('us-east-1')
shard_id = 'shardId-0000000000000'
shard_it = kinesis.get_shard_iterator(MYSTREAM, shard_id, 'LATEST')['ShardIterator']
while 1==1:
print("getting record")
out = kinesis.get_records(shard_it,limit=1)
for i in out['Records']:
recordId = json.loads(i['Data'])['SaleId']['N']
productId = json.loads(i['Data'])['ProductId']['N']
quantity = json.loads(i['Data'])['Quantity']['N']
saleDate = json.loads(i['Data'])['SaleDate']['S']
inv_quantity = 0 - int(quantity)
add_record = "INSERT INTO FactInventory (idFactInventory, ProductID, QuantityChange, DateTime) VALUES (%s, %s, %s, %s)"
data_record = {
'record_no': recordId,
'productID': productId,
'quantity': quantity,
'dateTime': saleDate,
}
cursor.execute(add_record, (str(recordId), str(productId), str(inv_quantity), str(saleDate)))
mydb.commit()
shard_it = out['NextShardIterator']
time.sleep(5)