def main(fileName):
# API key
with open('../../config.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
### INSERT YOUR CODE HERE ###
Dynamodb_table_name = "mataData"
dynamodb = aws.getResource('dynamodb', 'us-east-1')
table = dynamodb.Table('mtaData')
f = open(fileName, 'a')
f_csv = csv.writer(f)
# f_csv.writerow(columns)
for i in range(ITERATIONS):
data = mtaUpdates.mtaUpdates("")
tripUpdate, alerts, vehicle, timestamp = data.getTripUpdates()
#headers = ['Timestamp', 'tripId', 'Route', 'Day of the week', 'Time at which it reaches express station', 'Time at which it reaches the destination']
#f = open(fileName, 'a')
#f_csv = csv.writer(f)
row = matchdata(columns, tripUpdate, alerts, vehicle, timestamp)
#print row
f_csv.writerows(row)
#f.close()
print "file write success iter:", i
f.close()
def main(fileName):
# API key
with open('../key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
### INSERT YOUR CODE HERE ###
mtaUpdate = mtaUpdates.mtaUpdates(APIKEY)
try:
i = 0
while i < 150:
dataList = parseData(mtaUpdate)
appendToCsvFile(fileName, dataList)
print "CSV updated!"
# Data is only published by the MTA every 30 seconds
# No reason to hammer the server
time.sleep(30)
i = i + 1
except KeyboardInterrupt:
print "Exiting"
exit()
pushToS3(fileName)
def updateDb(APIKEY,dynamoTable):
while True:
# Create an object of type mtaUpdates
mta = mtaUpdates.mtaUpdates(APIKEY)
tripUpdates = mta.getTripUpdates()
writeData(tripUpdates,dynamoTable)
time.sleep(33)
def put(a):
try:
while True:
mta = mtaUpdates.mtaUpdates('34a41fb28d7324dcd8faedb0b98a1614')
message = mta.getTripUpdates()
map(a.add, message)
print 'add ', a.count1
a.count1 = 0
time.sleep(30)
except KeyboardInterrupt:
print "put stop"
def main():
# API key
with open('./key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
newRecord = mtaUpdates.mtaUpdates(APIKEY)
ITERATIONS = 50
while ITERATIONS > 0:
getUpdates(newRecord)
remove_duplicates()
time.sleep(60)
ITERATIONS -= 1
myS3 = S3(fileName2)
myS3.uploadData()
def update_data():
while (1):
print "update_data"
global this_table
info = mtaUpdates.mtaUpdates('695c5439ae651f2d066964b1b198f7d8')
data=info.getTripUpdates()
all_items = this_table.scan()["Items"]
with this_table.batch_writer() as batch:
batch.put_item(
Item={
'account_type': 'standard_user',
'username': '******',
'first_name': 'John',
'last_name': 'Doe',
'age': 25,
'address': {
'road': '1 Jefferson Street',
'city': 'Los Angeles',
'state': 'CA',
'zipcode': 90001
}
}
)
# repeat this every 30 sec
timestamp = data[3]
i = 0
for j in xrange(0,len(data[i])):
this_item = data[i][j]
if item_in_table(this_item,timestamp):
update_item_u(this_item,timestamp)
else:
add_item_u(this_item,timestamp)
i = 1
for j in xrange(0,len(data[i])):
this_item = data[i][j]
if item_in_table(this_item, timestamp):
update_item_v(this_item,timestamp)
else:
add_item_v(this_item,timestamp)
def task1():
print 'Adding!'
newUpdate = mtaUpdates.mtaUpdates(APIKEY)
tr = newUpdate.getTripUpdates()
for entity in tr:
response = table.put_item(
Item={
'tripId': entity[0],
'routeId': entity[1],
'startDate': entity[2],
'direction': entity[3],
'currentStopId': entity[6],
'currentStopStatus': entity[8],
'vehicleTimeStamp': entity[7],
'futureStopData': entity[4],
'timeStamp': entity[5]
})
time.sleep(30)
def update_data():
while (1):
print "update_data"
global this_table
info = mtaUpdates.mtaUpdates('695c5439ae651f2d066964b1b198f7d8')
data = info.getTripUpdates()
all_items = this_table.scan()["Items"]
with this_table.batch_writer() as batch:
batch.put_item(
Item={
'account_type': 'standard_user',
'username': '******',
'first_name': 'John',
'last_name': 'Doe',
'age': 25,
'address': {
'road': '1 Jefferson Street',
'city': 'Los Angeles',
'state': 'CA',
'zipcode': 90001
}
})
# repeat this every 30 sec
timestamp = data[3]
i = 0
for j in xrange(0, len(data[i])):
this_item = data[i][j]
if item_in_table(this_item, timestamp):
update_item_u(this_item, timestamp)
else:
add_item_u(this_item, timestamp)
i = 1
for j in xrange(0, len(data[i])):
this_item = data[i][j]
if item_in_table(this_item, timestamp):
update_item_v(this_item, timestamp)
else:
add_item_v(this_item, timestamp)
def main():
with open('./key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
newRecord = mtaUpdates.mtaUpdates(APIKEY)
station = prompt()
data = getUpdates(newRecord,station)
if len(data) < 1:
print "Sorry, there is no available prediction at this time"
return
# connect to kinesis
kinesis = aws.getClient('kinesis','us-east-1')
# with open('trainRecordFinal.csv','rb') as f:
# dataReader = csv.DictReader(f)
# for row in dataReader:
# print row
response = kinesis.describe_stream(
StreamName=KINESIS_STREAM_NAME,
)
print data
kinesis.put_record(StreamName=KINESIS_STREAM_NAME, Data=json.dumps(data), PartitionKey='0')
def update_data():
while (1):
print "update_data"
global this_table
info = mtaUpdates.mtaUpdates('695c5439ae651f2d066964b1b198f7d8')
data = info.getTripUpdates()
# repeat this every 30 sec
timestamp = data[3]
i = 0
for j in xrange(0, len(data[i])):
this_item = data[i][j]
if item_in_table(this_item, timestamp):
update_item_u(this_item, timestamp)
else:
add_item_u(this_item, timestamp)
i = 1
for j in xrange(0, len(data[i])):
this_item = data[i][j]
if item_in_table(this_item, timestamp):
update_item_v(this_item, timestamp)
else:
add_item_v(this_item, timestamp)
def main():
# API key
ITERATIONS = input("Please input the iterations you want to run (each iteration takes 30 sec)")
with open('../utils/key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
up = mtaUpdates.mtaUpdates(APIKEY)
line = []
try:
with open('rawData4.csv', 'r') as f:
f.close()
except:
with open('rawData4.csv','w') as csvfile:
f = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
f.writerow(columns)
csvfile.close()
with open('rawData4.csv', 'a') as csvfile:
for i in range(ITERATIONS):
print "Iteration:" + str(i)
updateList = up.getTripUpdates()
for update in updateList:
if update.direction == "N":
continue
if update.tripId[7] != '1' and update.tripId[7] != '2' and update.tripId[7] != '3':
continue
#print update.futureStops
line = []
now = TIMEZONE.localize(datetime.now())
last_midnight = now.replace(hour = 0, minute = 0, second = 0, microsecond = 0)
time_from_midnight = (now - last_midnight).seconds / 60
line.append(time_from_midnight)
line.append(update.tripId)
if update.tripId[7] == '1':
line.append('L')
elif update.tripId[7] == '2' or update.tripId[7] == '3':
line.append('E')
line.append(datetime.today().weekday())
#try:
#line.append(update.futureStops["117S"][0]["arrivalTime"])
#except:
#line.append(None)
try:
line.append(update.futureStops["120S"][0]["arrivalTime"])
except:
line.append(None)
try:
line.append(update.futureStops["127S"][0]["arrivalTime"])
except:
line.append(None)
line.append(False)
line.append(False)
#line.append(False)
if update.vehicleData != None:
v = update.vehicleData
if v.currentStopId == "120S":
line[4] = v.timestamp
line[6] = True
if v.currentStopId == "127S":
line[5] = v.timestamp
line[7] = True
#if v.currentStopId == "127S":
#line[6] = v.timestamp
#line[9] = True
spamwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(line)
time.sleep(30)
def main(fileName):
# API key
with open('../../key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
data = []
iteration = 0
while iteration < ITERATIONS:
iteration = iteration + 1
# Create an object of type mtaUpdates
mta = mtaUpdates.mtaUpdates(APIKEY)
tripUpdates = mta.getTripUpdates()
# Filter out the trains with no vehicle data and trains going North
# These are tentative trips and sometimes it gets cancelled
trips = [trip for trip in tripUpdates if trip.vehicleData is not None and trip.direction=='S']
# List of trains with route id = 1 and are either at 116th station or before it
localTrains = [lcl for lcl in trips if lcl.routeId == '1' and int(lcl.vehicleData.currentStopId[:-1]) <= 120]
# List of trains with route id 2 and are either at 96th or before it
# Note that 2 train's stop starts from 201S and goes till 227S and merges with 96th street(120S)
# range2 is a list of all stations on 2 train route before and including 96th
range2 = range(201,228)
range2.append(120)
expressTrains2 = [exp2 for exp2 in trips if exp2.routeId == '2' and int(exp2.vehicleData.currentStopId[:-1]) in range2]
# List of trains with route id 3 and are either at 96th or before it
# Note that 3 train's stop starts from 301S and goes till 302S and merges with 2 train lane at 135th street lenox 224S
# range3 is a list of all stations on 3 train route before and including 96th
range3 = range(224,228)
range3.extend([301,302,120])
expressTrains3 = [exp3 for exp3 in trips if exp3.routeId == '3' and int(exp3.vehicleData.currentStopId[:-1]) in range3]
expressTrains = expressTrains2 + expressTrains3
if DEBUG:
print "$$$@$@$$$$$$ iteration : %d $$#$@^$#$" %iteration
print "current time : ",time.time()
print "local"
for e in localTrains:
print e.tripId + " : " + e.vehicleData.currentStopId + " : " + e.vehicleData.currentStopStatus + " reaching 42 at : " + str(getTimeToReachDestination(e,'127S'))
print "expressTrains"
for l in expressTrains:
print l.tripId + " : " + l.vehicleData.currentStopId + " : " + l.vehicleData.currentStopStatus + " reaching 42 at : " + str(getTimeToReachDestination(l,'127S'))
# Here we are just collecting data. So we blindly copy all the items from the filtered list
# Note that there might be multiple entries for the same tripId, but we will have another method
# to clean up data
for l in localTrains:
item = {}
item['tripId'] = l.tripId
item['timeToReachExpressStation'] = getTimeToReachDestination(l,'120S')
# L for local trains
item['route'] = 'L'
item['timeToReachDestination'] = getTimeToReachDestination(l,'127S')
item['timestamp'] = time.time()
item['day'] = DAY
data.append(item)
for e in expressTrains:
item = {}
item['tripId'] = e.tripId
item['timeToReachExpressStation'] = getTimeToReachDestination(e,'120S')
# E for express trains
item['route'] = 'E'
item['timeToReachDestination'] = getTimeToReachDestination(e,'127S')
item['timestamp'] = time.time()
item['day'] = DAY
data.append(item)
# A new feed is generated every 30 seconds
time.sleep(40)
# if the file name given by user doesn't have .csv extension , add it
if '.csv' not in fileName:
fileName = fileName + '.csv'
# write data to file
with open(fileName ,'wb') as f:
w = csv.DictWriter(f,columns)
w.writeheader()
w.writerows(data)
f.close()
sts = boto.connect_sts()
assumedRoleObject = sts.assume_role_with_web_identity(ROLE_ARN, "XX",
oidc['Token'])
sys.path.append('../utils')
#s3_conn = boto3.client('kinesis',aws_access_key_id =assumedRoleObject.credentials.access_key,aws_secret_access_key = assumedRoleObject.credentials.secret_key,aws_session_token=assumedRoleObject.credentials.session_token)
client_dynamo = boto.dynamodb2.connect_to_region(
'us-east-1',
aws_access_key_id=assumedRoleObject.credentials.access_key,
aws_secret_access_key=assumedRoleObject.credentials.secret_key,
security_token=assumedRoleObject.credentials.session_token)
KINESIS_STREAM_NAME = 'mtaStream'
importmta = mtaUpdates.mtaUpdates('') #key value
def main(fileName):
# connect to kinesis
kinesis = aws.getClient('kinesis', 'us-east-1')
data = [] # list of dictionaries will be sent to kinesis
listcsv = list()
l = []
try:
while (True):
tu, timest = importmta.getTripUpdates()
print 'Entering feed\n'
listlcl = []
listexp = []
for t in tu:
def add():
try:
while True:
print 'Start to add!'
#global APIKEY
#time.sleep(30)
t = {}
t['routeId'] = 'N/A'
t['startDate'] = 'N/A'
t['direction'] = 'N/A'
t['futureStopData'] = 'N/A'
t['currentStopId'] = 'N/A'
t['currentStopStatus'] = 'N/A'
t['vehicleTimeStamp'] = 'N/A'
t['timeStamp'] = time.time()
if mutex.acquire():
m = mtaUpdates.mtaUpdates(APIKEY)
tmp = m.getTripUpdates()
trips = tmp[0]
#print trips
global mta
#with mta.batch_writer() as batch:
for key in trips:
#print key
for k in trips[key]:
if k == 'routeId':
t['routeId'] = trips[key]['routeId']
if k == 'startDate':
t['startDate'] = trips[key]['startDate']
if k == 'direction':
t['direction'] = trips[key]['direction']
if k == 'futureStopData':
t['futureStopData'] = trips[key]['futureStopData']
if k == 'currentStopId':
t['currentStopId'] = trips[key]['currentStopId']
if k == 'currentStopStatus':
t['currentStopStatus'] = trips[key]['currentStopStatus']
if k == 'vehicleTimeStamp':
t['vehicleTimeStamp'] = trips[key]['vehicleTimeStamp']
if k == 'timeStamp':
t['timeStamp'] = trips[key]['timeStamp']
#with mta.batch_writer() as batch:
mta.put_item(
Item={
'tripId': str(key),
'routeId': str(t['routeId']),
'startDate': t['startDate'],
'direction': t['direction'],
'futureStopData': t['futureStopData'],
'currentStopId': t['currentStopId'],
'vehicleTimeStamp': t['vehicleTimeStamp'],
'currentStopStatus': t['currentStopStatus'],
'timeStamp': Decimal(t['timeStamp'])
})
print " Add Done!"
mutex.release()
time.sleep(30)
except KeyboardInterrupt:
print 'Stop'
except:
print 'Wrong add'
from collections import OrderedDict
from threading import Thread
from datetime import datetime
from pytz import timezone
import decimal
import boto3
from boto3.dynamodb.conditions import Key,Attr
sys.path.append('../utils')
import tripupdate,vehicle,alert,mtaUpdates,aws
### YOUR CODE HERE ####
# Instantiate mtaUpdates Class with apikey
mta_updates = mtaUpdates.mtaUpdates('ae83cfea3a7fd03c01ad3e0ea835240e')
# establish dynamoDB client
dynamodb = aws.getResource('dynamodb','us-east-1')
print 'Created dynamoDB Client'
# use existing
table = dynamodb.Table('mtaData')
print 'Accessing "mtaData" Table'
def add_mtadata():
while(True):
print 'adding mtadata'
trips, timestamp = mta_updates.getTripUpdates()
for tripIDs in trips:
if trips[tripIDs].vehicleData != None:
try:
mtaData = Table.create('mtaData',
schema=[HashKey('tripId')],
connection=client_dynamo)
mtaData = Table('mtaData',
schema=[HashKey('tripId')],
connection=client_dynamo)
time.sleep(.12)
except:
mtaData = Table('mtaData',
schema=[HashKey('tripId')],
connection=client_dynamo)
#########before this is all AWS requirements
importmta = mtaUpdates.mtaUpdates('80e73d31e83802678e719e770763da90')
#tu,timest = importmta.getTripUpdates()
#print t[10].futureStops,v[0].currentStopNumber,timest
def up_thread():
listcsv = list()
l = []
try:
while (True):
tu, timest = importmta.getTripUpdates()
print 'Entering feed\n'
# with mtaData.batch_write() as batch:
for t in tu:
if t.routeId == '1' and t.direction == 'S':
# print(csvtable)
with open("mtadata_new.csv", "a", newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerows(csvtable)
csvfile.close()
except KeyboardInterrupt:
exit(0)
########################################################################
########################################################################
apikey = '38b00c0c7fe109238277ace1d6573388'
dynamodb = aws.getResource('dynamodb', 'us-east-1')
DB = dynamoMethods(DYNAMO_TABLE_NAME)
Updates = mtaUpdates.mtaUpdates(apikey)
#_thread.start_new_thread(add, ('thread1',))
#_thread.start_new_thread(purge, ('thread2',))
datathread = Thread(target=add, args=('datathread', ), daemon=True)
cleanthread = Thread(target=purge, args=('cleanthread', ), daemon=True)
csvthread = Thread(target=get_info, args=('csvthread', ), daemon=True)
# trueTimethread = Thread(target=get_trueArrivalTime, args=('trueTimethread'), daemon=True)
try:
while 1:
datathread.start()
cleanthread.start()
csvthread.start()
while 1:
pass
except KeyboardInterrupt:
import json, time, sys
from collections import OrderedDict
from threading import Thread
from datetime import datetime, timedelta
import boto3
from boto3.dynamodb.conditions import Key, Attr
import threading
sys.path.append('../utils')
import tripupdate, vehicle, alert, mtaUpdates, aws
with open('../key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
mtaUpdate = mtaUpdates.mtaUpdates(APIKEY)
tripUpdates = mtaUpdate.getTripUpdates()
dynamodb = aws.getResource('dynamodb', 'us-east-1')
table = dynamodb.Table('mtaData')
global e_flag
e_flag = False
def addData(data):
while (1):
print 'thread2'
print e_flag
if e_flag:
print "addData return"
def data_clean(mtaData):
try:
while(True):
print 'cleaning data'
response = mtaData.scan(dataTime__lte=(time.time()-120))
for r in response:
res = mtaData.delete_item(tripId=r['tripId'])
time.sleep(60)
except KeyboardInterrupt:
print "user attemptted to exit"
sys.exit()
if __name__ == "__main__":
apikey = '78b884010b517f078fdaeb9345beac62'
mtaUpdateData = mtaUpdates.mtaUpdates(apikey)
mtaData = connection()
print mtaData, "here"
adding_thread = threading.Thread(target= data_add, args=(mtaData, mtaUpdateData, ))
adding_thread.setDaemon(True)
adding_thread.start()
cleaning_thread = threading.Thread(target= data_clean, args=(mtaData,))
cleaning_thread.setDaemon(True)
cleaning_thread.start()
try:
while(1):
pass
except KeyboardInterrupt:
sys.exit()
