def doGTFSRTPolling(self):
print("doGTFSRTPolling", time.ctime())
r = self.session.get(self.gtfsrtFeedURL)
if r.status_code != 200:
print("GTFS RT feed returned with " + str(r.status_code))
return
feedmsg = gtfs_realtime_pb2.FeedMessage()
try:
feedmsg.ParseFromString(r.content)
for entity in feedmsg.entity:
if not entity.HasField('vehicle'):
continue
# Don't send message if route_id is missing from entity
# HasField() function does not work as expected, therefore we need check it like this
if "route_id" not in map(lambda x: x.name, entity.vehicle.trip.DESCRIPTOR.fields):
continue
nfeedmsg = gtfs_realtime_pb2.FeedMessage()
nfeedmsg.header.gtfs_realtime_version = "1.0"
nfeedmsg.header.incrementality = nfeedmsg.header.DIFFERENTIAL
nfeedmsg.header.timestamp = int(time.time())
nent = nfeedmsg.entity.add()
nent.CopyFrom(entity)
trip_id = entity.vehicle.trip.trip_id
route_id = utils.parse_route_id(self.feedName, entity.vehicle.trip.route_id, trip_id, self.OTPData)
direction_id = utils.parse_direction_id(self.feedName, entity.vehicle.trip.direction_id, trip_id, self.OTPData)
trip_headsign = entity.vehicle.vehicle.label
# headsigns with / cause problems in topics
if '/' in trip_headsign:
trip_headsign = ''
latitude = "{:.6f}".format(entity.vehicle.position.latitude) # Force coordinates to have 6 numbers
latitude_head = latitude[:2]
longitude = "{:.6f}".format(entity.vehicle.position.longitude)
longitude_head = longitude[:2]
geohash_head = latitude_head + ";" + longitude_head
geohash_firstdeg = latitude[3] + "" + longitude[3]
geohash_seconddeg = latitude[4] + "" + longitude[4]
geohash_thirddeg = latitude[5] + "" + longitude[5]
stop_id = entity.vehicle.stop_id
start_time = entity.vehicle.trip.start_time[0:5] # hh:mm
vehicle_id = entity.vehicle.vehicle.id
short_name = utils.parse_short_name(self.feedName, trip_id, route_id, self.OTPData)
color = utils.parse_color(self.feedName, trip_id, route_id, self.OTPData)
mode = utils.parse_mode(self.feedName, trip_id, route_id, self.OTPData)
# gtfsrt/vp/<feed_name>/<agency_id>/<agency_name>/<mode>/<route_id>/<direction_id>/<trip_headsign>/<trip_id>/<next_stop>/<start_time>/<vehicle_id>/<geohash_head>/<geohash_firstdeg>/<geohash_seconddeg>/<geohash_thirddeg>/<short_name>/<color>/
# GTFS RT feed used for testing was missing some information so those are empty
full_topic = '{0}/{1}///{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}/{11}/{12}/{13}/{14}/{15}/'.format(
self.baseMqttTopic, self.feedName, mode, route_id, direction_id,
trip_headsign, trip_id, stop_id, start_time, vehicle_id, geohash_head, geohash_firstdeg,
geohash_seconddeg, geohash_thirddeg, short_name, color)
sernmesg = nfeedmsg.SerializeToString()
self.client.publish(full_topic, sernmesg)
except:
print(r.content)
raise
def main():
json_rt = json.load(urllib2.urlopen(sys.argv[1]))
gtfs_realtime = gtfs_realtime_pb2.FeedMessage()
json_format.ParseDict(json_rt, gtfs_realtime)
with open(sys.argv[2], 'wb') as f:
f.write(gtfs_realtime.SerializeToString())
return
def generate_feed(self):
feed = gtfs_realtime_pb2.FeedMessage()
feed_header = feed.header
feed_header.gtfs_realtime_version = "1.0"
ts = 0
trip_updates = copy.deepcopy(self.trip_updates)
vehicle_positions = copy.deepcopy(self.vehicle_positions)
for trip_id in trip_updates:
trip_update_orig = trip_updates[trip_id]
entity = feed.entity.add()
entity.id = trip_id
new_trip_update = entity.trip_update
new_trip_update.CopyFrom(trip_update_orig)
if new_trip_update.timestamp > ts:
ts = new_trip_update.timestamp
for veh_id in vehicle_positions:
veh_pos_orig = vehicle_positions[veh_id]
entity = feed.entity.add()
entity.id = veh_id
new_veh_pos = entity.vehicle
new_veh_pos.CopyFrom(veh_pos_orig)
if new_veh_pos.timestamp > ts:
ts = new_veh_pos.timestamp
feed_header.timestamp = ts
return feed
def getTripUpdates(self):
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(self.feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " + str(e)
timestamp = feed.header.timestamp
nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE)
aaa = 0
for entity in feed.entity:
# Trip update represents a change in timetable
if entity.HasField('trip_update'):
self.tripUpdates.append([])
update = tripupdate.tripupdate()
self.tripUpdates[len(self.tripUpdates) - 1].append(
entity.trip_update.trip.trip_id)
self.tripUpdates[len(self.tripUpdates) - 1].append(
entity.trip_update.trip.route_id)
self.tripUpdates[len(self.tripUpdates) - 1].append(
entity.trip_update.trip.start_date)
self.tripUpdates[len(self.tripUpdates) - 1].append(
entity.trip_update.stop_time_update[0].stop_id[-1])
for stop in entity.trip_update.stop_time_update:
update.futureStops[stop.stop_id] = [{
'arrivaltime':
stop.arrival.time or None
}, {
'departuretime':
stop.departure.time or None
}]
self.tripUpdates[len(self.tripUpdates) - 1].append(
update.futureStops)
self.tripUpdates[len(self.tripUpdates) - 1].append(timestamp)
self.tripUpdates[len(self.tripUpdates) - 1].append("-1")
self.tripUpdates[len(self.tripUpdates) - 1].append(-1)
self.tripUpdates[len(self.tripUpdates) - 1].append(-1)
if entity.HasField('vehicle'):
self.tripUpdates[len(self.tripUpdates) -
1][6] = entity.vehicle.stop_id
#print "$$$$$$$$$$" +newItem.currentStopId+ "$$$$$$$$$$"
self.tripUpdates[len(self.tripUpdates) -
1][7] = entity.vehicle.timestamp
#print "!!!!!!!!!!" +str(newItem.vehicleTimeStamp)+ "!!!!!!!!!!"
self.tripUpdates[len(self.tripUpdates) -
1][8] = entity.vehicle.current_status
#print "**********" +str(newItem.currentStopStatus)+ "**********"
if entity.HasField('alert'):
a = alert.alert()
a.alertMessage = entity.alert.header_text.translation
#a.alertMessage = entity.alert.header_text.translation.text
self.alerts.append(a)
#### INSERT ALERT CODE HERE #####
return self.tripUpdates
def get_feed():
print "Retrieving feed..."
raw_gtfs = requests.get(MTA_ENDPOINT)
new_feed = gtfs.FeedMessage()
new_feed.ParseFromString(raw_gtfs.content)
print "Retrieved feed."
return new_feed
def vehiclePosition():
# PROTOCAL BUFFER!!! https://developers.google.com/protocol-buffers/docs/pythontutorial
# Create feed
feed = gtfs_realtime_pb2.FeedMessage()
# header
feed.header.gtfs_realtime_version = '1.0'
feed.header.timestamp = long((datetime.utcnow() - datetime(1970,1,1)).total_seconds())
# create an entity for each active trip id
lastBusLocations = db['checkins'].aggregate([{ "$match":
{ "tripId": { '$ne': None },
"location": { '$exists': True } } },
{ "$group":
{ "_id": { "bus": "$busId" },
"trip": { "$last": "$tripId" },
"time": { "$last": "$time" },
"location": { "$last": "$location" } } } ])
#return json.dumps(lastLocations, default=dthandler)
for bus in lastBusLocations['result']:
# add the trip entity
entity = feed.entity.add()
entity.id = 'bus' + str(bus['_id']['bus'])
entity.vehicle.trip.trip_id = bus['trip']
entity.vehicle.vehicle.id = str(bus['_id']['bus'])
entity.vehicle.vehicle.label = str(bus['_id']['bus'])
entity.vehicle.position.longitude = float(bus['location'][0])
entity.vehicle.position.latitude = float(bus['location'][1])
entity.vehicle.timestamp = long((bus['time'] - datetime(1970,1,1)).total_seconds())
if request.args.get('debug'):
return text_format.MessageToString(feed)
return feed.SerializeToString()
def getTripUpdates(self):
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(self.feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " +str(e)
timestamp = feed.header.timestamp
nytime = datetime.fromtimestamp(timestamp,self.TIMEZONE)
for entity in feed.entity:
# Trip update represents a change in timetable
if entity.trip_update and entity.trip_update.trip.trip_id:
update = tripupdate.tripupdate()
##### INSERT TRIPUPDATE CODE HERE ####
if entity.vehicle and entity.vehicle.trip.trip_id:
v = vehicle.vehicle()
##### INSERT VEHICLE CODE HERE #####
if entity.alert:
a = alert.alert()=
#### INSERT ALERT CODE HERE #####
return self.tripUpdates
def load_feed():
key = os.environ['MTAKEY']
req = urllib2.Request(url(), None, {'x-api-key': key})
feed = gtfs_realtime_pb2.FeedMessage()
body = urllib2.urlopen(req).read()
feed.ParseFromString(body)
return feed
def __fetch_and_ingest_from_url(self, url):
real_time_data_file = urllib2.urlopen(url)
logging.info("Parsing data...")
real_time_data = real_time_data_file.read()
real_time_data_obj = gtfs_realtime_pb2.FeedMessage()
real_time_data_obj.ParseFromString(real_time_data)
real_time_data_file.close()
self.__ingest_real_time_data(real_time_data_obj)
def getTripUpdates(self):
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(self.feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " +str(e)
timestamp = feed.header.timestamp
print "Timestamp: ", timestamp
nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE)
print "NYC Time: ", nytime
self.timestamp = timestamp
self.tripUpdates = []
self.alerts = []
for entity in feed.entity:
# Trip update represents a change in timetable
if entity.trip_update and entity.trip_update.trip.trip_id:
# Assign the tripupdate fields
t = tripupdate.tripupdate()
t.tripId = entity.trip_update.trip.trip_id
t.routeId = entity.trip_update.trip.route_id
t.startDate = entity.trip_update.trip.start_date
t.direction = entity.trip_update.trip.direction_id
# There can be many StopTimeUpdate messages
for st_update in entity.trip_update.stop_time_update:
times = []
times.append({"arrivalTime": st_update.arrival.time})
times.append({"departureTime": st_update.departure.time})
t.futureStops[st_update.stop_id] = times
self.tripUpdates.append(t)
if entity.vehicle and entity.vehicle.trip.trip_id:
v = vehicle.vehicle()
v.currentStopNumber = entity.vehicle.current_stop_sequence
v.currentStopId = entity.vehicle.stop_id
v.timestamp = entity.vehicle.timestamp
v.currentStopStatus = entity.vehicle.current_status
# Using the trip_id on a vehicle, associate that vehicle with the trip update object
vehicleTripId = entity.vehicle.trip.trip_id
self.setVehicleDataOnTripUpdate(vehicleTripId, v)
if entity.HasField('alert') and entity.alert:
a = alert.alert()
a.alertMessage = entity.alert.header_text.translation[0].text
for informed_entity in entity.alert.informed_entity:
tripId = informed_entity.trip.trip_id
a.tripId.append(tripId)
a.routeId[tripId] = informed_entity.trip.route_id
a.startDate[tripId] = informed_entity.trip.start_time
self.alerts.append(a)
return self.tripUpdates
def getTripUpdates(self):
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(self.feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " + str(e)
timestamp = feed.header.timestamp
#nytime = datetime.fromtimestamp(timestamp,self.TIMEZONE)
for entity in feed.entity:
#Trip update represents a change in timetable
if entity.trip_update and entity.trip_update.trip.trip_id:
trip_id = entity.trip_update.trip.trip_id
update = tripupdate.tripupdate()
update.tripId = entity.trip_update.trip.trip_id
update.routeId = entity.trip_update.trip.route_id
update.startDate = entity.trip_update.trip.start_date
# parse trip id for direction
if (re.search('N', str(entity.trip_update.trip.trip_id))):
update.direction = 'N'
else:
update.direction = 'S'
### stop_time_update is a LIST ###
for stop_time_updates in entity.trip_update.stop_time_update:
departureTime = None
arrivalTime = stop_time_updates.arrival.time
stopId = stop_time_updates.stop_id
if stop_time_updates.departure:
departureTime = stop_time_updates.departure.time
update.futureStops[stopId] = [{
"arrivalTime":
arrivalTime,
"departureTime":
departureTime
}]
self.tripUpdates[trip_id] = update
if entity.vehicle and entity.vehicle.trip.trip_id:
v = vehicle.vehicle()
v.currentStopId = entity.vehicle.stop_id
v.currentStopNumber = entity.vehicle.current_stop_sequence
v.timestamp = entity.vehicle.timestamp
v.currentStopStatus = entity.vehicle.current_status
update.vehicleData = v
self.tripUpdates[str(entity.vehicle.trip.trip_id)] = update
return self.tripUpdates, timestamp
def testParseFromString(self):
feed = gtfs_realtime_pb2.FeedMessage()
data_dir = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(data_dir, 'vehicle_position.pb'), 'rb') as f:
feed.ParseFromString(f.read())
self.assertEquals(1, len(feed.entity))
entity = feed.entity[0]
self.assertEquals('1', entity.id)
self.assert_(entity.HasField('vehicle'))
def buildHeader():
fm = gtfsrtpb.FeedMessage()
fh = fm.header
fh.gtfs_realtime_version = "1.0"
fh.incrementality = 0 # Determina si la búsqueda actual es incremental (valor por defecto).
fh.timestamp = int(
time.mktime(datetime.now().timetuple())
) # Momento en que se creó el contenido de este feed (en tiempo del servidor). En tiempo UNIX.
return fm
def getTripUpdates(self):
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(self.feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " + str(e)
timestamp = feed.header.timestamp
nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE)
def buildHeader():
fm = gtfsrtpb.FeedMessage()
fh = fm.header
fh.gtfs_realtime_version = "0.5"
# Determines whether the current search is incremental (default value).
fh.incrementality = 0
# Time when the content of this feed was created (in server time).
# In UNIX time.
fh.timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
return fm
def tripUpdate():
# PROTOCAL BUFFER!!! https://developers.google.com/protocol-buffers/docs/pythontutorial
# Create feed
feed = gtfs_realtime_pb2.FeedMessage()
# header
feed.header.gtfs_realtime_version = '1.0'
feed.header.timestamp = long((datetime.utcnow() - datetime(1970,1,1)).total_seconds())
# create an entity for each active trip id
activeTrips = db['checkins'].aggregate([{ "$match":
{ "tripId": { '$ne': None },
"adherence": { '$exists': True },
"lastStopSequence": { '$exists': True },
"lastStopSequenceOBA": { '$exists': True } } },
{ "$sort": { "time": 1 } },
{ "$group":
{ "_id": { "trip": "$tripId", "seq": "$lastStopSequence" },
"time": { "$last": "$time" },
"bus": { "$last": "$busId" },
"adherence": { "$last": "$adherence" },
"seqOBA": { "$last": "$lastStopSequenceOBA" } } },
{ "$sort": { "_id.seq": 1 } },
{ "$group":
{ "_id": { "trip": "$_id.trip" },
"time": { "$last": "$time" },
"bus": { "$last": "$bus" },
"timeChecks" :
{ "$push" :
{ "seq": "$_id.seq",
"time" : "$time",
"adherence": "$adherence",
"seqOBA": "$seqOBA" } } } } ])
#return json.dumps(activeTrips, default=dthandler)
for trip in activeTrips['result']:
# add the trip entity
entity = feed.entity.add()
entity.id = 'trip' + trip['_id']['trip']
entity.trip_update.trip.trip_id = trip['_id']['trip']
entity.trip_update.vehicle.id = str(trip['bus'])
entity.trip_update.vehicle.label = str(trip['bus'])
entity.trip_update.timestamp = long((trip['time'] - datetime(1970,1,1)).total_seconds())
# add the stop time updates
for update in trip['timeChecks']:
stopTime = entity.trip_update.stop_time_update.add()
stopTime.stop_sequence = update['seq'] if not request.args.get('oba') else update['seqOBA']
stopTime.arrival.delay = update['adherence'] * -60 # convert minutes to seconds
if request.args.get('debug'):
return text_format.MessageToString(feed)
return feed.SerializeToString()
def createRealtimeInstance():
global realtime
global realtimeAge
if realtimeAge and ((time.time() - realtimeAge) < 120):
print "No update"
return
print "update"
realtimeAge = time.time()
realtime = gtfs_realtime_pb2.FeedMessage()
realtime.ParseFromString(
urlopen('http://rtu.york.ca/gtfsrealtime/TripUpdates').read())
def produce_trip_updates(self):
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(self.mta_api_url)
feed.ParseFromString(response.content)
for entity in feed.entity:
if entity.HasField('trip_update'):
update_json = MessageToJson(entity.trip_update)
self.kafka_producer.produce(self.kafka_topic,
update_json.encode('utf-8'))
self.kafka_producer.flush()
def make_message(self):
"""Make a full gtfs_realtime message with a header and entities."""
lat, long, spd, bearing = self.next_pvt()
self.make_header()
self.make_one_entity(lat, long, spd, bearing)
message = gtfs_rt.FeedMessage()
message.header.CopyFrom(self.header)
# TBD: let this loop over many entities.
message.entity.extend([
self.entity,
])
self.message = message
def load_entity(url):
fm = gtfs_realtime_pb2.FeedMessage()
with requests.get(url) as r:
fm.ParseFromString(r.content)
# Check the feed version
if fm.header.gtfs_realtime_version != '1.0':
logging.warning(
'Warning: feed version has changed. Expected 1.0, found %s',
fm.header.gtfs_realtime_version)
return fm.entity
def doGTFSRTPolling(self):
print("doGTFSRTPolling", time.ctime())
r = requests.get(self.gtfsrtFeedURL)
feedmsg = gtfs_realtime_pb2.FeedMessage()
try:
feedmsg.ParseFromString(r.content)
for e in feedmsg.entity:
nfeedmsg = gtfs_realtime_pb2.FeedMessage()
nfeedmsg.header.gtfs_realtime_version = "1.0"
nfeedmsg.header.incrementality = nfeedmsg.header.DIFFERENTIAL
nfeedmsg.header.timestamp = int(time.time())
nent = nfeedmsg.entity.add()
nent.CopyFrom(e)
sernmesg = nfeedmsg.SerializeToString()
self.client.publish("gtfsrt/tre/vp", sernmesg)
except:
print(r.content)
raise
def main(fileName):
# connect to kinesis
with open('../utils/key.txt', 'rb') as keyfile:
APIKEY = keyfile.read().rstrip('\n')
keyfile.close()
feedurl = 'http://datamine.mta.info/mta_esi.php?feed_id=1&key='
feedurl = feedurl + APIKEY
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " + str(e)
row = []
nearest = 0
#nearest_tripId =''
for entity in feed.entity:
if entity.vehicle and entity.vehicle.trip.trip_id and entity.vehicle.stop_id:
vehicleData = entity.vehicle
tripId = vehicleData.trip.trip_id
stopId = vehicleData.stop_id
if (tripId[7] == '1') and (stopId[3] == 'S') and (int(
str(stopId)[0:3]) <= 117) and (int(str(stopId)[0:3]) >
nearest):
nearest = int(stopId[0:3])
nearest_tripId = tripId
push = int(str(nearest_tripId)[0:6]) / 100
row.append(push)
row.append(datetime.today().weekday())
with open(fileName, 'a') as csvfile:
spamwriter = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(row)
kinesis = aws.getClient('kinesis', 'us-east-1')
data = [] # list of dictionaries will be sent to kinesis
with open(fileName, 'rb') as f:
dataReader = csv.DictReader(f)
for row in dataReader:
#print row
kinesis.put_record(StreamName=KINESIS_STREAM_NAME,
Data=json.dumps(row),
PartitionKey='0')
break
f.close()
def poll_mbta_vehicle_positions():
fetch_timestamp = int(time.time())
# fetch feed from remote server
fp = urlopen(FEED_URL)
gtfsrt_message = fp.read()
# parse it using protocol buffers
pp = grt.FeedMessage()
pp.ParseFromString(gtfsrt_message)
try:
info = Info.objects.all()[0]
if info.buspos_timestamp is not None and pp.header.timestamp <= info.buspos_timestamp:
self.stdout.write("nothing to do - already have the latest data\n")
return
except IndexError:
info = Info(buspos_timestamp=pp.header.timestamp)
ret = []
print "new timestamp %s > %s old timestamp" % (pp.header.timestamp,
info.timestamp)
info.buspos_timestamp = pp.header.timestamp
info.save()
print pp.header
for i, entity in enumerate(pp.entity):
print ".",
vprec = VehicleUpdate()
if entity.vehicle.HasField('trip'):
vprec.trip_id = entity.vehicle.trip.trip_id
vprec.start_date = entity.vehicle.trip.start_date
vprec.schedule_relationship = entity.vehicle.trip.schedule_relationship
if entity.vehicle.HasField('position'):
vprec.latitude = entity.vehicle.position.latitude
vprec.longitude = entity.vehicle.position.longitude
vprec.current_stop_sequence = entity.vehicle.current_stop_sequence
vprec.data_timestamp = entity.vehicle.timestamp
vprec.fetch_timestamp = fetch_timestamp
vprec.save()
print
print "%s - %s\n" % (datetime.datetime.fromtimestamp(fetch_timestamp), i)
def getTripUpdates(self):
feed = gtfs_realtime_pb2.FeedMessage()
try:
with contextlib.closing(urllib2.urlopen(self.feedurl)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server " + str(e)
timestamp = feed.header.timestamp
nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE)
for entity in feed.entity:
# Trip update represents a change in timetable
update = None
if entity.trip_update and entity.trip_update.trip.trip_id:
update = tripupdate.tripupdate()
update.tripId = entity.trip_update.trip.trip_id
update.routeId = entity.trip_update.trip.route_id
update.startDate = entity.trip_update.trip.start_date
update.direction = entity.trip_update.trip.trip_id[
entity.trip_update.trip.trip_id.find("..") + 2]
update.vehicleData = entity.vehicle
for e in entity.trip_update.stop_time_update:
arrivalTime = e.arrival.time
departureTime = e.departure.time
v1 = {"arrivalTime": arrivalTime}
v2 = {"departureTime": departureTime}
value = [v1, v2]
update.futureStops[e.stop_id] = value
v = None
if entity.vehicle and entity.vehicle.trip.trip_id:
v = vehicle.vehicle()
v.currentStopNumber = entity.vehicle.current_stop_sequence
v.currentStopId = entity.vehicle.stop_id
v.timestamp = entity.vehicle.timestamp
v.currentStopStatus = entity.vehicle.current_status
a = None
if entity.alert:
a = alert.alert()
for trip in entity.alert.informed_entity:
a.tripId.append(trip.trip_id)
a.routeId[trip.trip_id] = trip.route_id
a.startDate[trip.trip_id] = trip.start_date
self.tripUpdates.append([update, v, a])
return self.tripUpdates
def populate_pb(disruption):
feed_message = gtfs_realtime_pb2.FeedMessage()
feed_message.header.gtfs_realtime_version = '1.0'
feed_message.header.incrementality = gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL
feed_message.header.timestamp = get_pos_time(datetime.datetime.utcnow())
feed_entity = feed_message.entity.add()
feed_entity.id = disruption.id
feed_entity.is_deleted = (disruption.status == "archived")
if not feed_entity.is_deleted:
disruption_pb = feed_entity.Extensions[chaos_pb2.disruption]
populate_disruption(disruption, disruption_pb)
return feed_message
def grab_feeds(path_name):
feed_list = []
feednum = 0
# unpack protobufs into FeedMessages
for fn in os.listdir(path_name):
path = os.path.join(path_name, fn)
if os.path.isfile(path):
f = open(path, 'r')
feed = gtfs.FeedMessage()
feed.ParseFromString(f.read())
feed_list.append(feed)
print('GRABBING FROM FILE: ' + path + ', IN FEED NUMBER: ' +
str(feednum))
feednum += 1
return feed_list
def update(self):
"""
:param self: Get trip updates from mta real time feed
:return:
"""
# fetch feed from GTFS
feed = gtfs_realtime_pb2.FeedMessage()
# check for valid response
try:
with contextlib.closing(urllib2.urlopen(self.feed_url)) as response:
d = feed.ParseFromString(response.read())
except (urllib2.URLError, google.protobuf.message.DecodeError) as e:
print "Error while connecting to mta server: " +str(e)
# parse for timestamp
timestamp = feed.header.timestamp
nytime = datetime.datetime.fromtimestamp(timestamp, TIMEZONE)
# parse feed
for entity in feed.entity:
# trip update represents a change in timetable
if entity.trip_update and entity.trip_update.trip.trip_id:
# create new trip update
tu = TripUpdate(entity.trip_update)
tu.set_timestamp(timestamp)
# add update to the trip dictionary
self.trip_updates[tu.trip_id] = tu
# vehicle update gets added to pre-existing trip update
if entity.vehicle and entity.vehicle.trip.trip_id:
# create vehicle
v = Vehicle(entity.vehicle)
# add vehicle to trip_update
tid = entity.vehicle.trip.trip_id
if self.trip_updates[tid]:
self.trip_updates[tid].set_vehicle(v)
# push new alert
if entity.alert and entity.alert.informed_entity:
a = Alert(entity.alert)
self.alerts.append(a)
def buildGTFSRTMessage(self, alerts=0, fuzzy=False, debug=False):
trains = self.train_dt.getTrainDataCopy()
msg = gtfs_realtime_pb2.FeedMessage()
msg.header.gtfs_realtime_version = "1.0"
msg.header.incrementality = msg.header.FULL_DATASET
#return pprint.pformat(trains)
for t in trains:
self.matchTrainsAndCreateMessage(msg,
trains[t],
alerts=alerts,
fuzzy=fuzzy,
debug=debug)
return msg
def load_message(url):
fm = gtfs_realtime_pb2.FeedMessage()
with requests.get(url) as r:
try:
fm.ParseFromString(r.content)
except (RuntimeWarning, google.protobuf.message.DecodeError) as e:
logging.error("ERROR: %s in %s", e, url)
return fm, e
# Check the feed version
if fm.entity and fm.header.gtfs_realtime_version != "1.0":
logging.warning(
"WARNING: feed version has changed. Expected 1.0, found %s",
fm.header.gtfs_realtime_version,
)
logging.warning("file: %s", url)
return fm, None
def getData(key):
Max_Tries = 30
failed = False
for i in range(Max_Tries):
try:
fetch_url = "http://datamine.mta.info/mta_esi.php?key=%s&feed_id=1" % key
logging.debug("Updating data: url: %s" % fetch_url)
data = gtfs_realtime_pb2.FeedMessage()
u = urllib2.urlopen(fetch_url)
data.ParseFromString(u.read())
if failed:
logging.warn("Update successful.")
return data
except Exception as e:
sleep_time = 5
logging.warn("Update failed: %s" % e)
logging.debug("sleeping %s seconds." % sleep_time)
time.sleep(sleep_time)
