def doGTFSRTPolling(self): print("doGTFSRTPolling", time.ctime()) r = self.session.get(self.gtfsrtFeedURL) if r.status_code != 200: print("GTFS RT feed returned with " + str(r.status_code)) return feedmsg = gtfs_realtime_pb2.FeedMessage() try: feedmsg.ParseFromString(r.content) for entity in feedmsg.entity: if not entity.HasField('vehicle'): continue # Don't send message if route_id is missing from entity # HasField() function does not work as expected, therefore we need check it like this if "route_id" not in map(lambda x: x.name, entity.vehicle.trip.DESCRIPTOR.fields): continue nfeedmsg = gtfs_realtime_pb2.FeedMessage() nfeedmsg.header.gtfs_realtime_version = "1.0" nfeedmsg.header.incrementality = nfeedmsg.header.DIFFERENTIAL nfeedmsg.header.timestamp = int(time.time()) nent = nfeedmsg.entity.add() nent.CopyFrom(entity) trip_id = entity.vehicle.trip.trip_id route_id = utils.parse_route_id(self.feedName, entity.vehicle.trip.route_id, trip_id, self.OTPData) direction_id = utils.parse_direction_id(self.feedName, entity.vehicle.trip.direction_id, trip_id, self.OTPData) trip_headsign = entity.vehicle.vehicle.label # headsigns with / cause problems in topics if '/' in trip_headsign: trip_headsign = '' latitude = "{:.6f}".format(entity.vehicle.position.latitude) # Force coordinates to have 6 numbers latitude_head = latitude[:2] longitude = "{:.6f}".format(entity.vehicle.position.longitude) longitude_head = longitude[:2] geohash_head = latitude_head + ";" + longitude_head geohash_firstdeg = latitude[3] + "" + longitude[3] geohash_seconddeg = latitude[4] + "" + longitude[4] geohash_thirddeg = latitude[5] + "" + longitude[5] stop_id = entity.vehicle.stop_id start_time = entity.vehicle.trip.start_time[0:5] # hh:mm vehicle_id = entity.vehicle.vehicle.id short_name = utils.parse_short_name(self.feedName, trip_id, route_id, self.OTPData) color = utils.parse_color(self.feedName, trip_id, route_id, self.OTPData) mode = utils.parse_mode(self.feedName, trip_id, route_id, self.OTPData) # gtfsrt/vp/<feed_name>/<agency_id>/<agency_name>/<mode>/<route_id>/<direction_id>/<trip_headsign>/<trip_id>/<next_stop>/<start_time>/<vehicle_id>/<geohash_head>/<geohash_firstdeg>/<geohash_seconddeg>/<geohash_thirddeg>/<short_name>/<color>/ # GTFS RT feed used for testing was missing some information so those are empty full_topic = '{0}/{1}///{2}/{3}/{4}/{5}/{6}/{7}/{8}/{9}/{10}/{11}/{12}/{13}/{14}/{15}/'.format( self.baseMqttTopic, self.feedName, mode, route_id, direction_id, trip_headsign, trip_id, stop_id, start_time, vehicle_id, geohash_head, geohash_firstdeg, geohash_seconddeg, geohash_thirddeg, short_name, color) sernmesg = nfeedmsg.SerializeToString() self.client.publish(full_topic, sernmesg) except: print(r.content) raise
def main(): json_rt = json.load(urllib2.urlopen(sys.argv[1])) gtfs_realtime = gtfs_realtime_pb2.FeedMessage() json_format.ParseDict(json_rt, gtfs_realtime) with open(sys.argv[2], 'wb') as f: f.write(gtfs_realtime.SerializeToString()) return
def generate_feed(self): feed = gtfs_realtime_pb2.FeedMessage() feed_header = feed.header feed_header.gtfs_realtime_version = "1.0" ts = 0 trip_updates = copy.deepcopy(self.trip_updates) vehicle_positions = copy.deepcopy(self.vehicle_positions) for trip_id in trip_updates: trip_update_orig = trip_updates[trip_id] entity = feed.entity.add() entity.id = trip_id new_trip_update = entity.trip_update new_trip_update.CopyFrom(trip_update_orig) if new_trip_update.timestamp > ts: ts = new_trip_update.timestamp for veh_id in vehicle_positions: veh_pos_orig = vehicle_positions[veh_id] entity = feed.entity.add() entity.id = veh_id new_veh_pos = entity.vehicle new_veh_pos.CopyFrom(veh_pos_orig) if new_veh_pos.timestamp > ts: ts = new_veh_pos.timestamp feed_header.timestamp = ts return feed
def getTripUpdates(self): feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(self.feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " + str(e) timestamp = feed.header.timestamp nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE) aaa = 0 for entity in feed.entity: # Trip update represents a change in timetable if entity.HasField('trip_update'): self.tripUpdates.append([]) update = tripupdate.tripupdate() self.tripUpdates[len(self.tripUpdates) - 1].append( entity.trip_update.trip.trip_id) self.tripUpdates[len(self.tripUpdates) - 1].append( entity.trip_update.trip.route_id) self.tripUpdates[len(self.tripUpdates) - 1].append( entity.trip_update.trip.start_date) self.tripUpdates[len(self.tripUpdates) - 1].append( entity.trip_update.stop_time_update[0].stop_id[-1]) for stop in entity.trip_update.stop_time_update: update.futureStops[stop.stop_id] = [{ 'arrivaltime': stop.arrival.time or None }, { 'departuretime': stop.departure.time or None }] self.tripUpdates[len(self.tripUpdates) - 1].append( update.futureStops) self.tripUpdates[len(self.tripUpdates) - 1].append(timestamp) self.tripUpdates[len(self.tripUpdates) - 1].append("-1") self.tripUpdates[len(self.tripUpdates) - 1].append(-1) self.tripUpdates[len(self.tripUpdates) - 1].append(-1) if entity.HasField('vehicle'): self.tripUpdates[len(self.tripUpdates) - 1][6] = entity.vehicle.stop_id #print "$$$$$$$$$$" +newItem.currentStopId+ "$$$$$$$$$$" self.tripUpdates[len(self.tripUpdates) - 1][7] = entity.vehicle.timestamp #print "!!!!!!!!!!" +str(newItem.vehicleTimeStamp)+ "!!!!!!!!!!" self.tripUpdates[len(self.tripUpdates) - 1][8] = entity.vehicle.current_status #print "**********" +str(newItem.currentStopStatus)+ "**********" if entity.HasField('alert'): a = alert.alert() a.alertMessage = entity.alert.header_text.translation #a.alertMessage = entity.alert.header_text.translation.text self.alerts.append(a) #### INSERT ALERT CODE HERE ##### return self.tripUpdates
def get_feed(): print "Retrieving feed..." raw_gtfs = requests.get(MTA_ENDPOINT) new_feed = gtfs.FeedMessage() new_feed.ParseFromString(raw_gtfs.content) print "Retrieved feed." return new_feed
def vehiclePosition(): # PROTOCAL BUFFER!!! https://developers.google.com/protocol-buffers/docs/pythontutorial # Create feed feed = gtfs_realtime_pb2.FeedMessage() # header feed.header.gtfs_realtime_version = '1.0' feed.header.timestamp = long((datetime.utcnow() - datetime(1970,1,1)).total_seconds()) # create an entity for each active trip id lastBusLocations = db['checkins'].aggregate([{ "$match": { "tripId": { '$ne': None }, "location": { '$exists': True } } }, { "$group": { "_id": { "bus": "$busId" }, "trip": { "$last": "$tripId" }, "time": { "$last": "$time" }, "location": { "$last": "$location" } } } ]) #return json.dumps(lastLocations, default=dthandler) for bus in lastBusLocations['result']: # add the trip entity entity = feed.entity.add() entity.id = 'bus' + str(bus['_id']['bus']) entity.vehicle.trip.trip_id = bus['trip'] entity.vehicle.vehicle.id = str(bus['_id']['bus']) entity.vehicle.vehicle.label = str(bus['_id']['bus']) entity.vehicle.position.longitude = float(bus['location'][0]) entity.vehicle.position.latitude = float(bus['location'][1]) entity.vehicle.timestamp = long((bus['time'] - datetime(1970,1,1)).total_seconds()) if request.args.get('debug'): return text_format.MessageToString(feed) return feed.SerializeToString()
def getTripUpdates(self): feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(self.feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " +str(e) timestamp = feed.header.timestamp nytime = datetime.fromtimestamp(timestamp,self.TIMEZONE) for entity in feed.entity: # Trip update represents a change in timetable if entity.trip_update and entity.trip_update.trip.trip_id: update = tripupdate.tripupdate() ##### INSERT TRIPUPDATE CODE HERE #### if entity.vehicle and entity.vehicle.trip.trip_id: v = vehicle.vehicle() ##### INSERT VEHICLE CODE HERE ##### if entity.alert: a = alert.alert()= #### INSERT ALERT CODE HERE ##### return self.tripUpdates
def load_feed(): key = os.environ['MTAKEY'] req = urllib2.Request(url(), None, {'x-api-key': key}) feed = gtfs_realtime_pb2.FeedMessage() body = urllib2.urlopen(req).read() feed.ParseFromString(body) return feed
def __fetch_and_ingest_from_url(self, url): real_time_data_file = urllib2.urlopen(url) logging.info("Parsing data...") real_time_data = real_time_data_file.read() real_time_data_obj = gtfs_realtime_pb2.FeedMessage() real_time_data_obj.ParseFromString(real_time_data) real_time_data_file.close() self.__ingest_real_time_data(real_time_data_obj)
def getTripUpdates(self): feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(self.feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " +str(e) timestamp = feed.header.timestamp print "Timestamp: ", timestamp nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE) print "NYC Time: ", nytime self.timestamp = timestamp self.tripUpdates = [] self.alerts = [] for entity in feed.entity: # Trip update represents a change in timetable if entity.trip_update and entity.trip_update.trip.trip_id: # Assign the tripupdate fields t = tripupdate.tripupdate() t.tripId = entity.trip_update.trip.trip_id t.routeId = entity.trip_update.trip.route_id t.startDate = entity.trip_update.trip.start_date t.direction = entity.trip_update.trip.direction_id # There can be many StopTimeUpdate messages for st_update in entity.trip_update.stop_time_update: times = [] times.append({"arrivalTime": st_update.arrival.time}) times.append({"departureTime": st_update.departure.time}) t.futureStops[st_update.stop_id] = times self.tripUpdates.append(t) if entity.vehicle and entity.vehicle.trip.trip_id: v = vehicle.vehicle() v.currentStopNumber = entity.vehicle.current_stop_sequence v.currentStopId = entity.vehicle.stop_id v.timestamp = entity.vehicle.timestamp v.currentStopStatus = entity.vehicle.current_status # Using the trip_id on a vehicle, associate that vehicle with the trip update object vehicleTripId = entity.vehicle.trip.trip_id self.setVehicleDataOnTripUpdate(vehicleTripId, v) if entity.HasField('alert') and entity.alert: a = alert.alert() a.alertMessage = entity.alert.header_text.translation[0].text for informed_entity in entity.alert.informed_entity: tripId = informed_entity.trip.trip_id a.tripId.append(tripId) a.routeId[tripId] = informed_entity.trip.route_id a.startDate[tripId] = informed_entity.trip.start_time self.alerts.append(a) return self.tripUpdates
def getTripUpdates(self): feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(self.feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " + str(e) timestamp = feed.header.timestamp #nytime = datetime.fromtimestamp(timestamp,self.TIMEZONE) for entity in feed.entity: #Trip update represents a change in timetable if entity.trip_update and entity.trip_update.trip.trip_id: trip_id = entity.trip_update.trip.trip_id update = tripupdate.tripupdate() update.tripId = entity.trip_update.trip.trip_id update.routeId = entity.trip_update.trip.route_id update.startDate = entity.trip_update.trip.start_date # parse trip id for direction if (re.search('N', str(entity.trip_update.trip.trip_id))): update.direction = 'N' else: update.direction = 'S' ### stop_time_update is a LIST ### for stop_time_updates in entity.trip_update.stop_time_update: departureTime = None arrivalTime = stop_time_updates.arrival.time stopId = stop_time_updates.stop_id if stop_time_updates.departure: departureTime = stop_time_updates.departure.time update.futureStops[stopId] = [{ "arrivalTime": arrivalTime, "departureTime": departureTime }] self.tripUpdates[trip_id] = update if entity.vehicle and entity.vehicle.trip.trip_id: v = vehicle.vehicle() v.currentStopId = entity.vehicle.stop_id v.currentStopNumber = entity.vehicle.current_stop_sequence v.timestamp = entity.vehicle.timestamp v.currentStopStatus = entity.vehicle.current_status update.vehicleData = v self.tripUpdates[str(entity.vehicle.trip.trip_id)] = update return self.tripUpdates, timestamp
def testParseFromString(self): feed = gtfs_realtime_pb2.FeedMessage() data_dir = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(data_dir, 'vehicle_position.pb'), 'rb') as f: feed.ParseFromString(f.read()) self.assertEquals(1, len(feed.entity)) entity = feed.entity[0] self.assertEquals('1', entity.id) self.assert_(entity.HasField('vehicle'))
def buildHeader(): fm = gtfsrtpb.FeedMessage() fh = fm.header fh.gtfs_realtime_version = "1.0" fh.incrementality = 0 # Determina si la búsqueda actual es incremental (valor por defecto). fh.timestamp = int( time.mktime(datetime.now().timetuple()) ) # Momento en que se creó el contenido de este feed (en tiempo del servidor). En tiempo UNIX. return fm
def getTripUpdates(self): feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(self.feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " + str(e) timestamp = feed.header.timestamp nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE)
def buildHeader(): fm = gtfsrtpb.FeedMessage() fh = fm.header fh.gtfs_realtime_version = "0.5" # Determines whether the current search is incremental (default value). fh.incrementality = 0 # Time when the content of this feed was created (in server time). # In UNIX time. fh.timestamp = int(time.mktime(datetime.datetime.now().timetuple())) return fm
def tripUpdate(): # PROTOCAL BUFFER!!! https://developers.google.com/protocol-buffers/docs/pythontutorial # Create feed feed = gtfs_realtime_pb2.FeedMessage() # header feed.header.gtfs_realtime_version = '1.0' feed.header.timestamp = long((datetime.utcnow() - datetime(1970,1,1)).total_seconds()) # create an entity for each active trip id activeTrips = db['checkins'].aggregate([{ "$match": { "tripId": { '$ne': None }, "adherence": { '$exists': True }, "lastStopSequence": { '$exists': True }, "lastStopSequenceOBA": { '$exists': True } } }, { "$sort": { "time": 1 } }, { "$group": { "_id": { "trip": "$tripId", "seq": "$lastStopSequence" }, "time": { "$last": "$time" }, "bus": { "$last": "$busId" }, "adherence": { "$last": "$adherence" }, "seqOBA": { "$last": "$lastStopSequenceOBA" } } }, { "$sort": { "_id.seq": 1 } }, { "$group": { "_id": { "trip": "$_id.trip" }, "time": { "$last": "$time" }, "bus": { "$last": "$bus" }, "timeChecks" : { "$push" : { "seq": "$_id.seq", "time" : "$time", "adherence": "$adherence", "seqOBA": "$seqOBA" } } } } ]) #return json.dumps(activeTrips, default=dthandler) for trip in activeTrips['result']: # add the trip entity entity = feed.entity.add() entity.id = 'trip' + trip['_id']['trip'] entity.trip_update.trip.trip_id = trip['_id']['trip'] entity.trip_update.vehicle.id = str(trip['bus']) entity.trip_update.vehicle.label = str(trip['bus']) entity.trip_update.timestamp = long((trip['time'] - datetime(1970,1,1)).total_seconds()) # add the stop time updates for update in trip['timeChecks']: stopTime = entity.trip_update.stop_time_update.add() stopTime.stop_sequence = update['seq'] if not request.args.get('oba') else update['seqOBA'] stopTime.arrival.delay = update['adherence'] * -60 # convert minutes to seconds if request.args.get('debug'): return text_format.MessageToString(feed) return feed.SerializeToString()
def createRealtimeInstance(): global realtime global realtimeAge if realtimeAge and ((time.time() - realtimeAge) < 120): print "No update" return print "update" realtimeAge = time.time() realtime = gtfs_realtime_pb2.FeedMessage() realtime.ParseFromString( urlopen('http://rtu.york.ca/gtfsrealtime/TripUpdates').read())
def produce_trip_updates(self): feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(self.mta_api_url) feed.ParseFromString(response.content) for entity in feed.entity: if entity.HasField('trip_update'): update_json = MessageToJson(entity.trip_update) self.kafka_producer.produce(self.kafka_topic, update_json.encode('utf-8')) self.kafka_producer.flush()
def make_message(self): """Make a full gtfs_realtime message with a header and entities.""" lat, long, spd, bearing = self.next_pvt() self.make_header() self.make_one_entity(lat, long, spd, bearing) message = gtfs_rt.FeedMessage() message.header.CopyFrom(self.header) # TBD: let this loop over many entities. message.entity.extend([ self.entity, ]) self.message = message
def load_entity(url): fm = gtfs_realtime_pb2.FeedMessage() with requests.get(url) as r: fm.ParseFromString(r.content) # Check the feed version if fm.header.gtfs_realtime_version != '1.0': logging.warning( 'Warning: feed version has changed. Expected 1.0, found %s', fm.header.gtfs_realtime_version) return fm.entity
def doGTFSRTPolling(self): print("doGTFSRTPolling", time.ctime()) r = requests.get(self.gtfsrtFeedURL) feedmsg = gtfs_realtime_pb2.FeedMessage() try: feedmsg.ParseFromString(r.content) for e in feedmsg.entity: nfeedmsg = gtfs_realtime_pb2.FeedMessage() nfeedmsg.header.gtfs_realtime_version = "1.0" nfeedmsg.header.incrementality = nfeedmsg.header.DIFFERENTIAL nfeedmsg.header.timestamp = int(time.time()) nent = nfeedmsg.entity.add() nent.CopyFrom(e) sernmesg = nfeedmsg.SerializeToString() self.client.publish("gtfsrt/tre/vp", sernmesg) except: print(r.content) raise
def main(fileName): # connect to kinesis with open('../utils/key.txt', 'rb') as keyfile: APIKEY = keyfile.read().rstrip('\n') keyfile.close() feedurl = 'http://datamine.mta.info/mta_esi.php?feed_id=1&key=' feedurl = feedurl + APIKEY feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " + str(e) row = [] nearest = 0 #nearest_tripId ='' for entity in feed.entity: if entity.vehicle and entity.vehicle.trip.trip_id and entity.vehicle.stop_id: vehicleData = entity.vehicle tripId = vehicleData.trip.trip_id stopId = vehicleData.stop_id if (tripId[7] == '1') and (stopId[3] == 'S') and (int( str(stopId)[0:3]) <= 117) and (int(str(stopId)[0:3]) > nearest): nearest = int(stopId[0:3]) nearest_tripId = tripId push = int(str(nearest_tripId)[0:6]) / 100 row.append(push) row.append(datetime.today().weekday()) with open(fileName, 'a') as csvfile: spamwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow(row) kinesis = aws.getClient('kinesis', 'us-east-1') data = [] # list of dictionaries will be sent to kinesis with open(fileName, 'rb') as f: dataReader = csv.DictReader(f) for row in dataReader: #print row kinesis.put_record(StreamName=KINESIS_STREAM_NAME, Data=json.dumps(row), PartitionKey='0') break f.close()
def poll_mbta_vehicle_positions(): fetch_timestamp = int(time.time()) # fetch feed from remote server fp = urlopen(FEED_URL) gtfsrt_message = fp.read() # parse it using protocol buffers pp = grt.FeedMessage() pp.ParseFromString(gtfsrt_message) try: info = Info.objects.all()[0] if info.buspos_timestamp is not None and pp.header.timestamp <= info.buspos_timestamp: self.stdout.write("nothing to do - already have the latest data\n") return except IndexError: info = Info(buspos_timestamp=pp.header.timestamp) ret = [] print "new timestamp %s > %s old timestamp" % (pp.header.timestamp, info.timestamp) info.buspos_timestamp = pp.header.timestamp info.save() print pp.header for i, entity in enumerate(pp.entity): print ".", vprec = VehicleUpdate() if entity.vehicle.HasField('trip'): vprec.trip_id = entity.vehicle.trip.trip_id vprec.start_date = entity.vehicle.trip.start_date vprec.schedule_relationship = entity.vehicle.trip.schedule_relationship if entity.vehicle.HasField('position'): vprec.latitude = entity.vehicle.position.latitude vprec.longitude = entity.vehicle.position.longitude vprec.current_stop_sequence = entity.vehicle.current_stop_sequence vprec.data_timestamp = entity.vehicle.timestamp vprec.fetch_timestamp = fetch_timestamp vprec.save() print print "%s - %s\n" % (datetime.datetime.fromtimestamp(fetch_timestamp), i)
def getTripUpdates(self): feed = gtfs_realtime_pb2.FeedMessage() try: with contextlib.closing(urllib2.urlopen(self.feedurl)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server " + str(e) timestamp = feed.header.timestamp nytime = datetime.fromtimestamp(timestamp, self.TIMEZONE) for entity in feed.entity: # Trip update represents a change in timetable update = None if entity.trip_update and entity.trip_update.trip.trip_id: update = tripupdate.tripupdate() update.tripId = entity.trip_update.trip.trip_id update.routeId = entity.trip_update.trip.route_id update.startDate = entity.trip_update.trip.start_date update.direction = entity.trip_update.trip.trip_id[ entity.trip_update.trip.trip_id.find("..") + 2] update.vehicleData = entity.vehicle for e in entity.trip_update.stop_time_update: arrivalTime = e.arrival.time departureTime = e.departure.time v1 = {"arrivalTime": arrivalTime} v2 = {"departureTime": departureTime} value = [v1, v2] update.futureStops[e.stop_id] = value v = None if entity.vehicle and entity.vehicle.trip.trip_id: v = vehicle.vehicle() v.currentStopNumber = entity.vehicle.current_stop_sequence v.currentStopId = entity.vehicle.stop_id v.timestamp = entity.vehicle.timestamp v.currentStopStatus = entity.vehicle.current_status a = None if entity.alert: a = alert.alert() for trip in entity.alert.informed_entity: a.tripId.append(trip.trip_id) a.routeId[trip.trip_id] = trip.route_id a.startDate[trip.trip_id] = trip.start_date self.tripUpdates.append([update, v, a]) return self.tripUpdates
def populate_pb(disruption): feed_message = gtfs_realtime_pb2.FeedMessage() feed_message.header.gtfs_realtime_version = '1.0' feed_message.header.incrementality = gtfs_realtime_pb2.FeedHeader.DIFFERENTIAL feed_message.header.timestamp = get_pos_time(datetime.datetime.utcnow()) feed_entity = feed_message.entity.add() feed_entity.id = disruption.id feed_entity.is_deleted = (disruption.status == "archived") if not feed_entity.is_deleted: disruption_pb = feed_entity.Extensions[chaos_pb2.disruption] populate_disruption(disruption, disruption_pb) return feed_message
def grab_feeds(path_name): feed_list = [] feednum = 0 # unpack protobufs into FeedMessages for fn in os.listdir(path_name): path = os.path.join(path_name, fn) if os.path.isfile(path): f = open(path, 'r') feed = gtfs.FeedMessage() feed.ParseFromString(f.read()) feed_list.append(feed) print('GRABBING FROM FILE: ' + path + ', IN FEED NUMBER: ' + str(feednum)) feednum += 1 return feed_list
def update(self): """ :param self: Get trip updates from mta real time feed :return: """ # fetch feed from GTFS feed = gtfs_realtime_pb2.FeedMessage() # check for valid response try: with contextlib.closing(urllib2.urlopen(self.feed_url)) as response: d = feed.ParseFromString(response.read()) except (urllib2.URLError, google.protobuf.message.DecodeError) as e: print "Error while connecting to mta server: " +str(e) # parse for timestamp timestamp = feed.header.timestamp nytime = datetime.datetime.fromtimestamp(timestamp, TIMEZONE) # parse feed for entity in feed.entity: # trip update represents a change in timetable if entity.trip_update and entity.trip_update.trip.trip_id: # create new trip update tu = TripUpdate(entity.trip_update) tu.set_timestamp(timestamp) # add update to the trip dictionary self.trip_updates[tu.trip_id] = tu # vehicle update gets added to pre-existing trip update if entity.vehicle and entity.vehicle.trip.trip_id: # create vehicle v = Vehicle(entity.vehicle) # add vehicle to trip_update tid = entity.vehicle.trip.trip_id if self.trip_updates[tid]: self.trip_updates[tid].set_vehicle(v) # push new alert if entity.alert and entity.alert.informed_entity: a = Alert(entity.alert) self.alerts.append(a)
def buildGTFSRTMessage(self, alerts=0, fuzzy=False, debug=False): trains = self.train_dt.getTrainDataCopy() msg = gtfs_realtime_pb2.FeedMessage() msg.header.gtfs_realtime_version = "1.0" msg.header.incrementality = msg.header.FULL_DATASET #return pprint.pformat(trains) for t in trains: self.matchTrainsAndCreateMessage(msg, trains[t], alerts=alerts, fuzzy=fuzzy, debug=debug) return msg
def load_message(url): fm = gtfs_realtime_pb2.FeedMessage() with requests.get(url) as r: try: fm.ParseFromString(r.content) except (RuntimeWarning, google.protobuf.message.DecodeError) as e: logging.error("ERROR: %s in %s", e, url) return fm, e # Check the feed version if fm.entity and fm.header.gtfs_realtime_version != "1.0": logging.warning( "WARNING: feed version has changed. Expected 1.0, found %s", fm.header.gtfs_realtime_version, ) logging.warning("file: %s", url) return fm, None
def getData(key): Max_Tries = 30 failed = False for i in range(Max_Tries): try: fetch_url = "http://datamine.mta.info/mta_esi.php?key=%s&feed_id=1" % key logging.debug("Updating data: url: %s" % fetch_url) data = gtfs_realtime_pb2.FeedMessage() u = urllib2.urlopen(fetch_url) data.ParseFromString(u.read()) if failed: logging.warn("Update successful.") return data except Exception as e: sleep_time = 5 logging.warn("Update failed: %s" % e) logging.debug("sleeping %s seconds." % sleep_time) time.sleep(sleep_time)