Example #1
0
def main(fileName):
    # API key
    with open('../../config.txt', 'rb') as keyfile:
        APIKEY = keyfile.read().rstrip('\n')
        keyfile.close()

	### INSERT YOUR CODE HERE ###
	Dynamodb_table_name = "mataData"
	dynamodb = aws.getResource('dynamodb', 'us-east-1')
	table = dynamodb.Table('mtaData')
	f = open(fileName, 'a')
	f_csv = csv.writer(f)
	# f_csv.writerow(columns)	
	for i in range(ITERATIONS):
		data = mtaUpdates.mtaUpdates("")
	

		tripUpdate, alerts, vehicle, timestamp = data.getTripUpdates()
	
		#headers = ['Timestamp', 'tripId', 'Route', 'Day of the week', 'Time at which it reaches express station', 'Time at which it reaches the destination']
		#f = open(fileName, 'a')
		#f_csv = csv.writer(f)
		row = matchdata(columns, tripUpdate, alerts, vehicle, timestamp)
		#print row
		f_csv.writerows(row)
		#f.close()	
		print "file write success iter:", i
	f.close()
Example #2
0
def main(fileName):
    # API key
    with open('../key.txt', 'rb') as keyfile:
        APIKEY = keyfile.read().rstrip('\n')
        keyfile.close()

    ### INSERT YOUR CODE HERE ###

    mtaUpdate = mtaUpdates.mtaUpdates(APIKEY)

    try:
        i = 0
        while i < 150:
            dataList = parseData(mtaUpdate)
            appendToCsvFile(fileName, dataList)
            print "CSV updated!"

            # Data is only published by the MTA every 30 seconds
            # No reason to hammer the server
            time.sleep(30)
            i = i + 1
    except KeyboardInterrupt:
        print "Exiting"
        exit()

    pushToS3(fileName)
def updateDb(APIKEY,dynamoTable):
    while True:
        # Create an object of type mtaUpdates
        mta = mtaUpdates.mtaUpdates(APIKEY)
        tripUpdates = mta.getTripUpdates()

        writeData(tripUpdates,dynamoTable)
        time.sleep(33)
Example #4
0
def put(a):
    try:
        while True:
            mta = mtaUpdates.mtaUpdates('34a41fb28d7324dcd8faedb0b98a1614')
            message = mta.getTripUpdates()
            map(a.add, message)
            print 'add ', a.count1
            a.count1 = 0
            time.sleep(30)
    except KeyboardInterrupt:
        print "put stop"
def main():
    # API key
	with open('./key.txt', 'rb') as keyfile:
		APIKEY = keyfile.read().rstrip('\n')
		keyfile.close()
	newRecord = mtaUpdates.mtaUpdates(APIKEY)
	ITERATIONS = 50
	while ITERATIONS > 0:
		getUpdates(newRecord)
		remove_duplicates()
		time.sleep(60)
		ITERATIONS -= 1
	myS3 = S3(fileName2)
	myS3.uploadData()
def update_data():
	while (1):
		print "update_data"
		global this_table
		info = mtaUpdates.mtaUpdates('695c5439ae651f2d066964b1b198f7d8')
		data=info.getTripUpdates()

		all_items = this_table.scan()["Items"]


	with this_table.batch_writer() as batch:
		batch.put_item(
			Item={
				'account_type': 'standard_user',
				'username': '******',
				'first_name': 'John',
				'last_name': 'Doe',
				'age': 25,
				'address': {
					'road': '1 Jefferson Street',
					'city': 'Los Angeles',
					'state': 'CA',
					'zipcode': 90001
				}
			}
		)


	# repeat this every 30 sec
		timestamp = data[3]
		i = 0
		for j in xrange(0,len(data[i])):
			this_item = data[i][j]
			if item_in_table(this_item,timestamp):
				update_item_u(this_item,timestamp)
			else:
				add_item_u(this_item,timestamp)

		i = 1
		for j in xrange(0,len(data[i])):
			this_item = data[i][j]
			if item_in_table(this_item, timestamp):
				update_item_v(this_item,timestamp)
			else:
				add_item_v(this_item,timestamp)
Example #7
0
def task1():
    print 'Adding!'
    newUpdate = mtaUpdates.mtaUpdates(APIKEY)
    tr = newUpdate.getTripUpdates()
    for entity in tr:
        response = table.put_item(
            Item={
                'tripId': entity[0],
                'routeId': entity[1],
                'startDate': entity[2],
                'direction': entity[3],
                'currentStopId': entity[6],
                'currentStopStatus': entity[8],
                'vehicleTimeStamp': entity[7],
                'futureStopData': entity[4],
                'timeStamp': entity[5]
            })
    time.sleep(30)
def update_data():
    while (1):
        print "update_data"
        global this_table
        info = mtaUpdates.mtaUpdates('695c5439ae651f2d066964b1b198f7d8')
        data = info.getTripUpdates()

        all_items = this_table.scan()["Items"]

    with this_table.batch_writer() as batch:
        batch.put_item(
            Item={
                'account_type': 'standard_user',
                'username': '******',
                'first_name': 'John',
                'last_name': 'Doe',
                'age': 25,
                'address': {
                    'road': '1 Jefferson Street',
                    'city': 'Los Angeles',
                    'state': 'CA',
                    'zipcode': 90001
                }
            })

        # repeat this every 30 sec
        timestamp = data[3]
        i = 0
        for j in xrange(0, len(data[i])):
            this_item = data[i][j]
            if item_in_table(this_item, timestamp):
                update_item_u(this_item, timestamp)
            else:
                add_item_u(this_item, timestamp)

        i = 1
        for j in xrange(0, len(data[i])):
            this_item = data[i][j]
            if item_in_table(this_item, timestamp):
                update_item_v(this_item, timestamp)
            else:
                add_item_v(this_item, timestamp)
def main():
    with open('./key.txt', 'rb') as keyfile:
        APIKEY = keyfile.read().rstrip('\n')
        keyfile.close()
    newRecord = mtaUpdates.mtaUpdates(APIKEY)
    station = prompt()
    data = getUpdates(newRecord,station)
    if len(data) < 1:
        print "Sorry, there is no available prediction at this time"
        return
    # connect to kinesis
    kinesis = aws.getClient('kinesis','us-east-1')
    # with open('trainRecordFinal.csv','rb') as f:
    #     dataReader = csv.DictReader(f)
    #     for row in dataReader:
    #         print row
    response = kinesis.describe_stream(
        StreamName=KINESIS_STREAM_NAME,
    )
    print data
    kinesis.put_record(StreamName=KINESIS_STREAM_NAME, Data=json.dumps(data), PartitionKey='0')
Example #10
0
def update_data():
    while (1):
        print "update_data"
        global this_table
        info = mtaUpdates.mtaUpdates('695c5439ae651f2d066964b1b198f7d8')
        data = info.getTripUpdates()

        # repeat this every 30 sec
        timestamp = data[3]
        i = 0
        for j in xrange(0, len(data[i])):
            this_item = data[i][j]
            if item_in_table(this_item, timestamp):
                update_item_u(this_item, timestamp)
            else:
                add_item_u(this_item, timestamp)

        i = 1
        for j in xrange(0, len(data[i])):
            this_item = data[i][j]
            if item_in_table(this_item, timestamp):
                update_item_v(this_item, timestamp)
            else:
                add_item_v(this_item, timestamp)
def main():
    # API key
    ITERATIONS = input("Please input the iterations you want to run (each iteration takes 30 sec)")
    with open('../utils/key.txt', 'rb') as keyfile:
        APIKEY = keyfile.read().rstrip('\n')
        keyfile.close()
	up = mtaUpdates.mtaUpdates(APIKEY)
	line = []
	try:
		with open('rawData4.csv', 'r') as f:
			f.close()
	except:
		with open('rawData4.csv','w') as csvfile:
			f = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
			f.writerow(columns)
			csvfile.close()

	with open('rawData4.csv', 'a') as csvfile:
		for i in range(ITERATIONS):
			print "Iteration:" + str(i)
			updateList = up.getTripUpdates()
			for update in updateList:
				if update.direction == "N":
					continue
				if update.tripId[7] != '1' and update.tripId[7] != '2' and update.tripId[7] != '3':
					continue
				#print update.futureStops
				line = []
				now = TIMEZONE.localize(datetime.now())
				last_midnight = now.replace(hour = 0, minute = 0, second = 0, microsecond = 0)
				time_from_midnight = (now - last_midnight).seconds / 60
				line.append(time_from_midnight)
				line.append(update.tripId)
				if update.tripId[7] == '1':
					line.append('L')
				elif update.tripId[7] == '2' or update.tripId[7] == '3':
					line.append('E')
				line.append(datetime.today().weekday())
				#try:
					#line.append(update.futureStops["117S"][0]["arrivalTime"])
				#except:
					#line.append(None)
				try:
					line.append(update.futureStops["120S"][0]["arrivalTime"])
				except:
					line.append(None)
				try:
					line.append(update.futureStops["127S"][0]["arrivalTime"])
				except:
					line.append(None)
				line.append(False)
				line.append(False)
				#line.append(False)
				if update.vehicleData != None:
					v = update.vehicleData
					if v.currentStopId == "120S":
						line[4] = v.timestamp
						line[6] = True
					if v.currentStopId == "127S":
						line[5] = v.timestamp
						line[7] = True
					#if v.currentStopId == "127S":
						#line[6] = v.timestamp
						#line[9] = True
		
				spamwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
				spamwriter.writerow(line)
			time.sleep(30)
def main(fileName):
    # API key
    with open('../../key.txt', 'rb') as keyfile:
        APIKEY = keyfile.read().rstrip('\n')
        keyfile.close()


    data = []

    iteration = 0
    while iteration < ITERATIONS:
	    iteration = iteration +  1


	    # Create an object of type mtaUpdates
	    mta = mtaUpdates.mtaUpdates(APIKEY)
	    tripUpdates = mta.getTripUpdates()

	    # Filter out the trains with no vehicle data and trains going North
	    # These are tentative trips and sometimes it gets cancelled
	    trips = [trip for trip in tripUpdates if trip.vehicleData is not None and trip.direction=='S']

	    # List of trains with route id = 1 and are either at 116th station or before it
	    localTrains = [lcl for lcl in trips if lcl.routeId == '1' and int(lcl.vehicleData.currentStopId[:-1]) <= 120] 

	    # List of trains with route id 2 and are either at 96th or before it
	    # Note that 2 train's stop starts from 201S and goes till 227S and merges with 96th street(120S)
	    # range2 is a list of all stations on 2 train route before and including 96th
	    range2 = range(201,228)
	    range2.append(120)

	    expressTrains2 = [exp2 for exp2 in trips if exp2.routeId == '2' and int(exp2.vehicleData.currentStopId[:-1]) in range2]

		# List of trains with route id 3 and are either at 96th or before it
	    # Note that 3 train's stop starts from 301S and goes till 302S and merges with 2 train lane at 135th street lenox 224S 
	    # range3 is a list of all stations on 3 train route before and including 96th
	    range3 = range(224,228)
	    range3.extend([301,302,120])
	    expressTrains3 = [exp3 for exp3 in trips if exp3.routeId == '3' and int(exp3.vehicleData.currentStopId[:-1]) in range3]

	    expressTrains = expressTrains2 + expressTrains3

	    if DEBUG:
			print "$$$@$@$$$$$$ iteration : %d $$#$@^$#$" %iteration
			print "current time : ",time.time()
			print "local"
			for e in localTrains:
				print e.tripId + " : " + e.vehicleData.currentStopId + " : " + e.vehicleData.currentStopStatus + " reaching 42 at : " + str(getTimeToReachDestination(e,'127S'))
			print "expressTrains"
			for l in expressTrains:
				print l.tripId + " : " + l.vehicleData.currentStopId + " : " + l.vehicleData.currentStopStatus +  " reaching 42 at : " + str(getTimeToReachDestination(l,'127S'))


		# Here we are just collecting data. So we blindly copy all the items from the filtered list
	    # Note that there might be multiple entries for the same tripId, but we will have another method 
	    # to clean up data

	    for l in localTrains:
			item = {}
			item['tripId'] = l.tripId
			item['timeToReachExpressStation'] = getTimeToReachDestination(l,'120S')
			# L for local trains
			item['route'] = 'L'
			item['timeToReachDestination'] = getTimeToReachDestination(l,'127S')
			item['timestamp'] = time.time()
			item['day'] = DAY
			data.append(item)

	    for e in expressTrains:
			item = {}
			item['tripId'] = e.tripId
			item['timeToReachExpressStation'] = getTimeToReachDestination(e,'120S')
			# E for express trains
			item['route'] = 'E'
			item['timeToReachDestination'] = getTimeToReachDestination(e,'127S')
			item['timestamp'] = time.time()
			item['day'] = DAY
			data.append(item)

		# A new feed is generated every 30 seconds
	    time.sleep(40)


	# if the file name given by user doesn't have .csv extension , add it
    if '.csv' not in fileName:
    	fileName = fileName + '.csv'

    # write data to file
    with open(fileName ,'wb') as f:

    	w = csv.DictWriter(f,columns)
    	w.writeheader()
    	w.writerows(data)
    	f.close()
Example #13
0
sts = boto.connect_sts()
assumedRoleObject = sts.assume_role_with_web_identity(ROLE_ARN, "XX",
                                                      oidc['Token'])

sys.path.append('../utils')

#s3_conn = boto3.client('kinesis',aws_access_key_id =assumedRoleObject.credentials.access_key,aws_secret_access_key = assumedRoleObject.credentials.secret_key,aws_session_token=assumedRoleObject.credentials.session_token)

client_dynamo = boto.dynamodb2.connect_to_region(
    'us-east-1',
    aws_access_key_id=assumedRoleObject.credentials.access_key,
    aws_secret_access_key=assumedRoleObject.credentials.secret_key,
    security_token=assumedRoleObject.credentials.session_token)

KINESIS_STREAM_NAME = 'mtaStream'
importmta = mtaUpdates.mtaUpdates('')  #key value


def main(fileName):
    # connect to kinesis
    kinesis = aws.getClient('kinesis', 'us-east-1')
    data = []  # list of dictionaries will be sent to kinesis
    listcsv = list()
    l = []
    try:
        while (True):
            tu, timest = importmta.getTripUpdates()
            print 'Entering feed\n'
            listlcl = []
            listexp = []
            for t in tu:
Example #14
0
def add():
    try:
        while True:
	    print 'Start to add!'
            #global APIKEY
            #time.sleep(30)

            t = {}
            t['routeId'] = 'N/A'
            t['startDate'] = 'N/A'
            t['direction'] = 'N/A'
            t['futureStopData'] = 'N/A'
            t['currentStopId'] = 'N/A'
            t['currentStopStatus'] = 'N/A'
            t['vehicleTimeStamp'] = 'N/A'
            t['timeStamp'] = time.time()

            if mutex.acquire():
                m = mtaUpdates.mtaUpdates(APIKEY)
                tmp = m.getTripUpdates()
		trips = tmp[0]
		#print trips
		global mta
                #with mta.batch_writer() as batch:
                for key in trips:
			#print key
                        for k in trips[key]:
                            if k == 'routeId':
                                t['routeId'] = trips[key]['routeId']
                            if k == 'startDate':
                                t['startDate'] = trips[key]['startDate']
                            if k == 'direction':
                                t['direction'] = trips[key]['direction']
                            if k == 'futureStopData':
                                t['futureStopData'] = trips[key]['futureStopData']
                            if k == 'currentStopId':
                                t['currentStopId'] = trips[key]['currentStopId']
                            if k == 'currentStopStatus':
                                t['currentStopStatus'] = trips[key]['currentStopStatus']
                            if k == 'vehicleTimeStamp':
                                t['vehicleTimeStamp'] = trips[key]['vehicleTimeStamp']
                            if k == 'timeStamp':
                                t['timeStamp'] = trips[key]['timeStamp']
                        #with mta.batch_writer() as batch:
                        mta.put_item(
                                Item={
                                    'tripId': str(key),
                                    'routeId': str(t['routeId']),
                                    'startDate': t['startDate'],
                                    'direction': t['direction'],
                                    'futureStopData': t['futureStopData'],
                                    'currentStopId': t['currentStopId'],
                                    'vehicleTimeStamp': t['vehicleTimeStamp'],
                                    'currentStopStatus': t['currentStopStatus'],
                                    'timeStamp': Decimal(t['timeStamp'])
                                    })
	        print " Add Done!"
                mutex.release()
	    time.sleep(30)
    except KeyboardInterrupt:
        print 'Stop'
    except:
        print 'Wrong add'
Example #15
0
from collections import OrderedDict
from threading import Thread
from datetime import datetime
from pytz import timezone
import decimal

import boto3
from boto3.dynamodb.conditions import Key,Attr

sys.path.append('../utils')
import tripupdate,vehicle,alert,mtaUpdates,aws

### YOUR CODE HERE ####

# Instantiate mtaUpdates Class with apikey	
mta_updates = mtaUpdates.mtaUpdates('ae83cfea3a7fd03c01ad3e0ea835240e')

# establish dynamoDB client
dynamodb = aws.getResource('dynamodb','us-east-1')
print 'Created dynamoDB Client'

# use existing
table = dynamodb.Table('mtaData')
print 'Accessing "mtaData" Table'

def add_mtadata():
    while(True):
        print 'adding mtadata'
        trips, timestamp = mta_updates.getTripUpdates()
        for tripIDs in trips:
            if trips[tripIDs].vehicleData != None:
Example #16
0
try:
    mtaData = Table.create('mtaData',
                           schema=[HashKey('tripId')],
                           connection=client_dynamo)
    mtaData = Table('mtaData',
                    schema=[HashKey('tripId')],
                    connection=client_dynamo)
    time.sleep(.12)
except:
    mtaData = Table('mtaData',
                    schema=[HashKey('tripId')],
                    connection=client_dynamo)

#########before this is all AWS requirements
importmta = mtaUpdates.mtaUpdates('80e73d31e83802678e719e770763da90')

#tu,timest =  importmta.getTripUpdates()
#print t[10].futureStops,v[0].currentStopNumber,timest


def up_thread():
    listcsv = list()
    l = []
    try:
        while (True):
            tu, timest = importmta.getTripUpdates()
            print 'Entering feed\n'
            #  with mtaData.batch_write() as batch:
            for t in tu:
                if t.routeId == '1' and t.direction == 'S':
            # print(csvtable)
            with open("mtadata_new.csv", "a", newline='') as csvfile:
                writer = csv.writer(csvfile)
                writer.writerows(csvtable)
                csvfile.close()

    except KeyboardInterrupt:
        exit(0)


########################################################################
########################################################################
apikey = '38b00c0c7fe109238277ace1d6573388'
dynamodb = aws.getResource('dynamodb', 'us-east-1')
DB = dynamoMethods(DYNAMO_TABLE_NAME)
Updates = mtaUpdates.mtaUpdates(apikey)
#_thread.start_new_thread(add, ('thread1',))
#_thread.start_new_thread(purge, ('thread2',))
datathread = Thread(target=add, args=('datathread', ), daemon=True)
cleanthread = Thread(target=purge, args=('cleanthread', ), daemon=True)
csvthread = Thread(target=get_info, args=('csvthread', ), daemon=True)
# trueTimethread = Thread(target=get_trueArrivalTime, args=('trueTimethread'), daemon=True)

try:
    while 1:
        datathread.start()
        cleanthread.start()
        csvthread.start()
        while 1:
            pass
except KeyboardInterrupt:
Example #18
0
import json, time, sys
from collections import OrderedDict
from threading import Thread
from datetime import datetime, timedelta
import boto3
from boto3.dynamodb.conditions import Key, Attr
import threading

sys.path.append('../utils')
import tripupdate, vehicle, alert, mtaUpdates, aws

with open('../key.txt', 'rb') as keyfile:
    APIKEY = keyfile.read().rstrip('\n')
    keyfile.close()

mtaUpdate = mtaUpdates.mtaUpdates(APIKEY)
tripUpdates = mtaUpdate.getTripUpdates()

dynamodb = aws.getResource('dynamodb', 'us-east-1')
table = dynamodb.Table('mtaData')

global e_flag
e_flag = False


def addData(data):
    while (1):
        print 'thread2'
        print e_flag
        if e_flag:
            print "addData return"
Example #19
0
def data_clean(mtaData):
    try:
        while(True):
            print 'cleaning data'
            response = mtaData.scan(dataTime__lte=(time.time()-120))
            for r in response:
                res = mtaData.delete_item(tripId=r['tripId'])
            time.sleep(60)
    except KeyboardInterrupt:
        print "user attemptted to exit"
        sys.exit()


if __name__ == "__main__":
    apikey = '78b884010b517f078fdaeb9345beac62'
    mtaUpdateData = mtaUpdates.mtaUpdates(apikey)
    mtaData = connection()
    print mtaData, "here"


    adding_thread = threading.Thread(target= data_add, args=(mtaData, mtaUpdateData, ))
    adding_thread.setDaemon(True)
    adding_thread.start()
    cleaning_thread = threading.Thread(target= data_clean, args=(mtaData,))
    cleaning_thread.setDaemon(True)
    cleaning_thread.start()
    try:
        while(1):
            pass
    except KeyboardInterrupt:
        sys.exit()