def main(): # need the GTFS Python bindings from google.transit import gtfs_realtime_pb2 import urllib import json import socket import time # create socket connection to hostname/port on which a TCP GeoEvent input is running tcpSocket = socket.create_connection(("localhost", 5565)) # polling model - run, wait 5 seconds, run, wait, run, wait, etc while True: feed = gtfs_realtime_pb2.FeedMessage() # this particular feed is from OV Api (http://www.openov.nl/) response = urllib.urlopen( 'http://gtfs.openov.nl/gtfs-rt/vehiclePositions.pb') # read the Protocal Buffers (.pb) file feed.ParseFromString(response.read()) # loop through feed entities for entity in feed.entity: # check for a vehicle in feed entity if entity.HasField('vehicle'): # build a simple id,lon,lat message to send to GeoEvent. msg = str(entity.vehicle.vehicle.label) + "," + \ str(entity.vehicle.position.longitude) + "," + \ str(entity.vehicle.position.latitude) + "\n" # send message tcpSocket.send(msg) #print(msg) time.sleep(5)
def make_vehicle_list(pb_file_list, folder_path): vehicle_list = [] bad_vehicle_header_list = [] dirname = folder_path for pb_file in pb_file_list: pb_file_path = os.path.join(dirname, pb_file) with open(pb_file_path, 'rb') as f: feed = gtfs_realtime_pb2.FeedMessage() feed.ParseFromString(f.read()) json_obj = MessageToJson(feed) json_parsed = json.loads(json_obj) if 'entity' in json_parsed.keys(): for vehicles_idx in range(len(json_parsed['entity'])): vehicle_dict = {} j_out = json_parsed['entity'][vehicles_idx] if 'vehicle' in j_out.keys(): if 'position' in j_out['vehicle'] and 'trip' in j_out[ 'vehicle']: vehicle_dict['vehicle_id'] = j_out['vehicle'][ 'vehicle']['id'] vehicle_dict['timestamp'] = j_out['vehicle'][ 'timestamp'] vehicle_dict['trip_id'] = j_out['vehicle']['trip'][ 'tripId'] vehicle_dict['route_id'] = j_out['vehicle']['trip'][ 'routeId'] vehicle_dict['vehicle_lat'] = j_out['vehicle'][ 'position']['latitude'] vehicle_dict['vehicle_long'] = j_out['vehicle'][ 'position']['longitude'] #trip_id = j_out['vehicle']['trip']['tripId'] #route_id = j_out['vehicle']['trip']['routeId'] #vehicle_dict['shape_id'] = get_shape_id_from_triproute(trip_id, route_id, schedule_df) vehicle_list.append(vehicle_dict) else: bad_vehicle_header_list.append(json_parsed['header']) else: bad_vehicle_header_list.append(json_parsed['header']) else: bad_vehicle_header_list.append(json_parsed['header']) return vehicle_list, bad_vehicle_header_list
def parser(url): feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(url) feed.ParseFromString(response.content) count = len(feed.entity) for val in feed.entity: for idx in range(count): try: values = { "route": val.trip_update.trip.route_id, "arrival": val.trip_update.stop_time_update[idx].arrival.time, "stop_id": val.trip_update.stop_time_update[idx].stop_id } feed_list.append(values) except Exception as e: # logging.debug(e) pass return feed_list
def _parse(self, data) -> pd.DataFrame: feed = gtfs_realtime_pb2.FeedMessage() feed.ParseFromString(data) feed_timestamp = feed.header.timestamp def gen(): for entity in feed.entity: trip_id = entity.vehicle.trip.trip_id timestamp = entity.vehicle.timestamp # stop_id, peron_id = parse_stop_id(entity.vehicle.stop_id) stop_sequence = entity.vehicle.current_stop_sequence block_id, trip_num, service_id = parse_trip_id(trip_id) yield block_id, trip_num, service_id, stop_sequence, timestamp, feed_timestamp columns = [ 'block_id', 'trip_num', 'service_id', 'stop_sequence', 'timestamp', 'feed_timestamp' ] vehicle_positions_df = pd.DataFrame(gen(), columns=columns) # vehicle_positions_df.set_index(['block_id', 'trip_num', 'service_id'], inplace=True) return vehicle_positions_df
class Gtfsrt(BaseAPI): """API for real time General Transit Feed Specifications: https://api.actransit.org/transit/gtfsrt/""" api = "gtfsrt" protobuf = gtfs_realtime_pb2.FeedMessage() def __repr__(self): return "ACTransit({})".format(self.__class__.__name__) @api_method def alerts(self): pass @api_method def tripupdates(self): pass @api_method def vehicles(self): pass
def run(): # Get gps data feed = gtfs_realtime_pb2.FeedMessage() # PMPML GTFS username and password username = "******" password = "******" response = requests.get( 'http://117.232.125.138/tms/data/gtfs/vehicle-positions.pb', auth=(username, password)) feed.ParseFromString(response.content) feed = MessageToDict(feed) for entity in feed['entity']: data = entity['vehicle']['position'] if entity['vehicle']['vehicle']['id'] == "ANT74": print(entity['vehicle']['vehicle']['id'], " = ", data) data.update({"route": entity['vehicle']['trip']['tripId']}) gps_result.update({entity['vehicle']['vehicle']['id']: data}) print(entity)
def _refresh_vehicle_position_data(self): positions = dict() occupancies = dict() feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(self._positions_url).content feed.ParseFromString(response) for entity in feed.entity: if entity.HasField('vehicle'): # Handle position = entity.vehicle.position trip_id = entity.vehicle.trip.trip_id positions[trip_id] = { "latitude": position.latitude, "longitude": position.longitude, "bearing": position.bearing, "speed": position.speed * 3.6, # m/s to kph } occupancies[trip_id] = entity.vehicle.occupancy_status self._occupancies = occupancies self._positions = positions self._positions_last_updated = int(time.time())
def _get_vehicle_positions(self): from google.transit import gtfs_realtime_pb2 feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(self._vehicle_position_url, headers=self._headers) if response.status_code != 200: _LOGGER.error("updating vehicle positions got {}:{}.".format( response.status_code, response.content)) feed.ParseFromString(response.content) positions = {} for entity in feed.entity: vehicle = entity.vehicle if not vehicle.trip.route_id: # Vehicle is not in service continue positions[vehicle.vehicle.id] = vehicle.position return positions
def _update_trains(self, id): try: #print("Updating train info for ID: {0}...".format(id)) feed = gtfs_realtime_pb2.FeedMessage() #print("Retriving info from MTA datamine...") response = requests.get(self.url + id, headers={"x-api-key": self.key}) feed.ParseFromString(response.content) # Parse Entire Feed for e in feed.entity: # Record Route of Trip route = e.vehicle.trip.route_id # Exclude particular routes and record the Stop if route in self.exclude: continue elif route == "L": i = int(e.vehicle.current_stop_sequence) if i < 10: stop = "L0{0}".format(i) else: stop = "L{0}".format(i) else: stop = e.vehicle.stop_id[:3] if stop in self.combine.keys(): stop = self.combine[stop] if stop not in self.trains.keys(): # Check if stop is already in dict self.trains[stop] = [route] elif route in self.trains[stop]: # Check if route is already in dict's list continue else: self.trains[stop].append(route) except ConnectionError as e: print("Unable to connect to the MTA Data Stream...") except DecodeError: print("Unable to decode the message from " + id) except KeyboardInterrupt: print("Exiting...") sys.exit()
def create_gtfs_feed(vehicle_states: dict) -> gtfs_realtime_pb2.FeedMessage: """Transform out internal state into a FeedMessage structure that GTFS expects""" # https://developers.google.com/transit/gtfs-realtime/reference#message-feedmessage""" feed_message = gtfs_realtime_pb2.FeedMessage() feed_message.header.gtfs_realtime_version = "2.0" feed_message.header.incrementality = gtfs_realtime_pb2.FeedHeader.FULL_DATASET feed_message.header.timestamp = int(datetime.now().timestamp()) created_entries = 0 for board_id, state in vehicle_states.items(): if state.route_id in ROUTE_ID_MAP: # create the FeedEntity and populate it with the values from our own state structure feed_entity = feed_message.entity.add() feed_entity.id = f"{created_entries}-{board_id}" # https://developers.google.com/transit/gtfs-realtime/reference#message-vehicledescriptor feed_entity.vehicle.vehicle.id = state.rtu_id feed_entity.vehicle.vehicle.label = state.board_name feed_entity.vehicle.timestamp = int(state.last_seen) # https://developers.google.com/transit/gtfs-realtime/reference#message-position feed_entity.vehicle.position.longitude = state.lon feed_entity.vehicle.position.latitude = state.lat feed_entity.vehicle.position.speed = state.speed feed_entity.vehicle.position.bearing = state.direction # https://developers.google.com/transit/gtfs-realtime/reference#message-tripdescriptor # note that the route_id is different here, ROUTE_ID_MAP[entity.route_id] is what we # usually call "name_concise" feed_entity.vehicle.trip.route_id = ROUTE_ID_MAP[state.route_id] created_entries += 1 else: if state.route_id not in UNKNOWN_ROUTES: LOG.warning( "Skipping board %s from unknown upstream_route_id=%s", board_id, state.route_id) UNKNOWN_ROUTES.add(state.route_id) LOG.debug("Produced %i entries in the feed", created_entries) return feed_message
def _update_route_statuses(self, vehicle_positions): """Get the latest data.""" from google.transit import gtfs_realtime_pb2 class StopDetails: def __init__(self, arrival_time, position): self.arrival_time = arrival_time self.position = position feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(self._trip_update_url, headers=self._headers) if response.status_code != 200: _LOGGER.error("updating route status got {}:{}".format( response.status_code, response.content)) feed.ParseFromString(response.content) departure_times = {} for entity in feed.entity: if entity.HasField('trip_update'): route_id = entity.trip_update.trip.route_id if route_id not in departure_times: departure_times[route_id] = {} for stop in entity.trip_update.stop_time_update: stop_id = stop.stop_id if not departure_times[route_id].get(stop_id): departure_times[route_id][stop_id] = [] # Use stop departure time; fall back on stop arrival time if not available details = StopDetails( datetime.datetime.fromtimestamp(stop.arrival.time), vehicle_positions.get(entity.trip_update.vehicle.id)) departure_times[route_id][stop_id].append(details) # Sort by arrival time for route in departure_times: for stop in departure_times[route]: departure_times[route][stop].sort(key=lambda t: t.arrival_time) self.info = departure_times
def GetTimes(): for tr in trains: tr.arrival_times = [] feed = gtfs_realtime_pb2.FeedMessage() response = urllib.request.urlopen( 'http://datamine.mta.info/mta_esi.php?key=581ad7b399d1eddcccec3a31d0e6e00d&feed_id=1' ) feed.ParseFromString(response.read()) now = datetime.now() for entity in feed.entity: if entity.HasField('trip_update'): for stu in entity.trip_update.stop_time_update: for tr in trains: if stu.stop_id == tr.stop_id and entity.trip_update.trip.route_id[ 0] == tr.route_id[0]: mintoarrive = (datetime.fromtimestamp(stu.arrival.time) - now).total_seconds() / 60 if mintoarrive >= 0: tr.arrival_times.append(round(mintoarrive, 0)) break for tr in trains: tr.arrival_times.sort()
def loopdwlmta(x): ## send request to api mta.info using my own key feed = gtfs_realtime_pb2.FeedMessage() response = urllib.urlopen( 'http://datamine.mta.info/mta_esi.php?key=5a8f6f4375e9f4bdea9b0c86afeaf911' ) feed.ParseFromString(response.read()) ## looping the content downloaded tmpSys = datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S') i = 1 for entity in feed.entity: if entity.HasField('trip_update'): ## trip update: 3 data characterisitcs TripIdi = entity.trip_update.trip.trip_id Routei = entity.trip_update.trip.route_id ## looking to record first 2 predicted arrival time ## looking to record first 2 predicted arrival time ArriveTime = [] stop_id = [] tripcomplete = [] p3trip = min(len(entity.trip_update.stop_time_update), 3) j = 0 while j < p3trip: tripcomplete = [] #print j ArrivalTime = entity.trip_update.stop_time_update[ j].arrival.time stop_id = entity.trip_update.stop_time_update[j].stop_id station_nth = j tripcomplete.extend([ tmpSys, TripIdi, Routei, station_nth, stop_id, ArrivalTime ]) #print tripcomplete print tripcomplete, type(tripcomplete) tripcomplstr = ','.join([str(i) for i in tripcomplete]) savefile(tripcomplstr) j += 1
def _get_vehicle_positions(self): from google.transit import gtfs_realtime_pb2 feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(self._vehicle_position_url, headers=self._headers) if response.status_code == 200: _LOGGER.info("Successfully updated vehicle positions - {}".format(response.status_code)) else: _LOGGER.error("updating vehicle positions got {}:{}.".format(response.status_code, response.content)) feed.ParseFromString(response.content) positions = {} for entity in feed.entity: vehicle = entity.vehicle if not vehicle.trip.trip_id: # Vehicle is not in service continue _LOGGER.debug("......Adding position for trip id {} position latitude {} longitude {} ".format(vehicle.trip.trip_id,vehicle.position.latitude,vehicle.position.longitude)) positions[vehicle.trip.trip_id] = vehicle.position return positions
def getStops(_, part): feed = gtfs_realtime_pb2.FeedMessage() for fn, contents in part: hr = int(fn[-5:-3]) dt = fn[-8:-6] # RUSH HOUR ONLY # the gtfs reports are 5 hours behind # from the file contents # therefore, add 5 hours if hr in [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0, 1]: #RUSH HOUR ONLY feed.ParseFromString(contents) for entity in feed.entity: if entity.HasField('vehicle'): vehicle = protobuf_to_dict(entity.vehicle) if vehicle.get('current_status') != 1: continue try: line = vehicle['trip']['route_id'].rstrip('X') # fold express into normal if line not in ['1', '2', '3', '4', '5', '6', 'GS', 'L', 'SI']: print 'weird line', line continue if 'stop_id' in vehicle: stop = vehicle['stop_id'] else: # L and SI stop at every station, need to use stop = '%d%s' % (vehicle['current_stop_sequence'], vehicle['trip']['trip_id'][-1]) #key = (line, stop) timestamp = vehicle['timestamp'] # datetime.datetime.utcfromtimestamp(vehicle['timestamp']) # merge 2 and 3 line if line in ['2','3']: line = '23' if line in ['4','5']: line = '45' yield ((dt,line,stop),[timestamp]) #yield ((dt,line,stop),timestamp) #yield (line, timestamp) except: print 'weird vehicle', vehicle continue
def makeProtoBuffer(trips, locations): ''' Create a protobuffer object from the trips object. Trips is expected to be a dictionary keyed to trip_id strings. The values are dictionaries with `stop_sequence` and `dev` items. ''' feed = gtfs_realtime_pb2.FeedMessage() # Set Header feed.header.gtfs_realtime_version = "2.0" feed.header.incrementality = 0 feed.header.timestamp = int(time.time()) # Create an trip update entity for each trip for trip_id, trip in trips.items(): entity = gtfs_realtime_pb2.FeedEntity() entity.id = trip_id entity.trip_update.trip.trip_id = trip_id stop_time_update = entity.trip_update.StopTimeUpdate() stop_time_update.stop_sequence = trip['stop_sequence'] stop_time_update.arrival.delay = trip['dev'] entity.trip_update.stop_time_update.append(stop_time_update) feed.entity.append(entity) # Create vehicle location for each vehicle location for vehicle_id, info in locations.items(): entity = gtfs_realtime_pb2.FeedEntity() entity.id = vehicle_id entity.vehicle.timestamp = info['timestamp'] entity.vehicle.trip.trip_id = info['trip']['trip_id'] entity.vehicle.vehicle.id = vehicle_id entity.vehicle.position.latitude = info['position']['latitude'] entity.vehicle.position.longitude = info['position']['longitude'] entity.vehicle.position.bearing = info['position']['bearing'] entity.vehicle.position.speed = info['position']['speed'] feed.entity.append(entity) return feed
def getTripStatus(trip_id, startStationName, endStationName): feed = gtfs_realtime_pb2.FeedMessage() response = requests.get( 'http://datamine.mta.info/mta_esi.php?key=%s&feed_id=1' % key) feed.ParseFromString(response.content) departureTime = None arrivalTime = None current_stop_sequence = None current_status = None timestamp = None for entity in feed.entity: if entity.HasField('trip_update'): if entity.trip_update.trip.trip_id == trip_id: for stop_time_update in entity.trip_update.stop_time_update: if stop_time_update.stop_id == startStationName: departureTime = int(stop_time_update.departure.time) # print entity if stop_time_update.stop_id == endStationName: arrivalTime = int(stop_time_update.arrival.time) # print stop_time_update if entity.HasField('vehicle'): if entity.vehicle.trip.trip_id == trip_id: current_stop_sequence = int( entity.vehicle.current_stop_sequence) current_status = str(entity.vehicle.current_status) timestamp = int(entity.vehicle.timestamp) if entity.HasField('alert'): for informed_entity in entity.alert.informed_entity: if informed_entity.trip.trip_id == trip_id: file.write(str(entity.alert.informed_entity)) if entity.alert.HasField('header_text'): file.write(str(entity.alert.header_text)) return (departureTime, arrivalTime, current_stop_sequence, current_status, timestamp)
def get_times(): feed = gtfs_realtime_pb2.FeedMessage() response = requests.get('http://datamine.mta.info/mta_esi.php?key={}&feed_id=2'.format(api_key)) feed.ParseFromString(response.content) jsonObj = MessageToJson(feed) data = json.loads(jsonObj) stops = { "S": [], "N": [] } time_now = int(time.time()) if( 'entity' in data): for entity in data['entity']: if('tripUpdate' in entity): for sched in entity['tripUpdate']['stopTimeUpdate']: if( stop_id in sched['stopId']): stops[sched['stopId'].replace(stop_id, "")].append( {"time": round((int(sched['arrival']['time']) - time_now) / 60) ,"direction": smap[entity['tripUpdate']['stopTimeUpdate'][-1]["stopId"]] } ) stops['S'] = sorted(stops['S'], key=lambda k: k['time']) stops['N'] = sorted(stops['N'], key=lambda k: k['time']) else: print('Malformatted MTA data', data) with open('data.json', 'w') as outfile: json.dump(jsonObj, outfile) # for entity in data['entity']: # if('tripUpdate' in entity): # for sched in entity['tripUpdate']['stopTimeUpdate']: # if( stop_id in sched['stopId']): # stops[sched['stopId'].replace(stop_id, "")].append( round((int(sched['arrival']['time']) - time_now) / 60) ) return stops
async def get_mb_records(s: ClientSession) -> MetroBus: """ Get records from MB api """ date_updated = datetime.now() feed = gtfs_realtime_pb2.FeedMessage() async with s.get(url=os.getenv("URL_API_MB")) as response: response.encoding = "utf-8" if response.status != 200: return feed.ParseFromString(await response.read()) for entity in feed.entity: yield MetroBusRaw( vehicle_id=entity.vehicle.vehicle.id, vehicle_label=entity.vehicle.vehicle.label, vehicle_status=entity.vehicle.current_status, date_updated=date_updated, position_latitude=entity.vehicle.position.latitude, position_longitude=entity.vehicle.position.longitude, )
def process_dump_contents(feed, contents, trip_dates, stops, fetchtime, feed_tz, start_date_str, start_date_str_after_midnight): global global_stats global_stats = { 'trip_updates_found': 0, 'stop_updates_found': 0, 'stop_updates_stored': 0, 'missing_trips': 0, 'unscheduled_trips': 0, 'unscheduled_tripdates': 0, 'missing_stops': 0, 'missing_trip_id': 0, 'missing_tripdates': 0, 'missing_tripdates_but_trip_existed': 0, 'missing_routes': 0, } feed_message = gtfs_realtime_pb2.FeedMessage() feed_message.ParseFromString(contents) threshold = int((fetchtime + timedelta(minutes=5)).timestamp()) for entity in feed_message.entity: if entity.HasField('trip_update'): process_trip_update(feed, trip_dates, stops, feed_tz, entity.trip_update, threshold, start_date_str, start_date_str_after_midnight) for stat in global_stats: print(f'{stat}: {global_stats[stat]}')
def MTAAPICall(route, stopcodes): # Helper function to make request to MTA GFTS API feed = gtfs_realtime_pb2.FeedMessage() response = requests.get( 'http://datamine.mta.info/mta_esi.php?key={}&feed_id=1'.format( mta_api_key)) feed.ParseFromString(response.content) currentTime = feed.header.timestamp # Time of API call arrivals = [] # List of subway arrival times stationcode = 'N/A' # If station code not found, return N/A for entity in feed.entity: # For each trip in the response if entity.trip_update.trip.route_id == route: # Matches the correct subway line for stop in ( entity.trip_update.stop_time_update ): # Parse each stop for all active subways in the correct subway line for possibleStation in stopcodes: # Parse each station code for the correct station (each station can have multiple station codes) if stop.stop_id == (possibleStation ): # Matches the direction and station stationcode = stop.stop_id # If station code is matched, assign the correct station code arrivals.append(stop.arrival.time ) # Add arrival time of train to list return currentTime, arrivals, stationcode # Return API call time, list of arriving times, and correct station code
def pull_septa_rail(): ssl._create_default_https_context = ssl._create_unverified_context tripfeed = gtfs_realtime_pb2.FeedMessage() #request for trip feed tripresponse = requests.get( 'http://www3.septa.org/gtfsrt/septarail-pa-us/Trip/rtTripUpdates.pb', allow_redirects=True) tripfeed.ParseFromString(tripresponse.content) # Check if request is working #print('There are {} trip updates in the dataset.'.format(len(tripfeed.entity))) if len(tripfeed.entity) == 0: return None dict_obj_trip = MessageToDict(tripfeed) collector = [] for trip in dict_obj_trip['entity']: if trip['tripUpdate'].get('stopTimeUpdate') is not None: row = OrderedDict() row['trip_id'] = trip['tripUpdate'].get('trip', {}).get('tripId') row['id'] = trip.get('id') row['delay'] = trip['tripUpdate']['stopTimeUpdate'][0].get( 'arrival', {}).get('delay') row['trip_stop_id'] = trip['tripUpdate']['stopTimeUpdate'][0].get( 'stopId') row['trip_stop_seq'] = trip['tripUpdate']['stopTimeUpdate'][0].get( 'stopSequence') row['timestamp'] = dict_obj_trip['header']['timestamp'] collector.append(row) df = pd.DataFrame(collector) df['humantime'] = df.apply( lambda row: datetime.datetime.fromtimestamp(int(row['timestamp'])), axis=1) df['agency'] = 'SEPTA' df['mode'] = "Rail" df['region'] = "New York/Philadelphia" return df
def find_station(station_ids, trains): feed = gtfs_realtime_pb2.FeedMessage() with open("secret.json") as f: key = json.load(f)["mta"] final_times = {} # Get a set of the appropriate MTA feed_id. This saves a ton of time in # terms of network requests. feeds = {FEED_ID[line] for line in trains} for feed_id in feeds: response = requests.get( f"http://datamine.mta.info/mta_esi.php?key={key}&feed_id={feed_id}" ) feed.ParseFromString(response.content) # Turn feed into a regular dictionary. d = protobuf_to_dict(feed) times = collections.defaultdict( lambda: collections.defaultdict(lambda: []) ) for i, val in enumerate(d["entity"]): if "trip_update" in val: for item in val["trip_update"]["stop_time_update"]: # Only looking for specific stations. if (item["stop_id"] in station_ids): if ("arrival" in item): times[ val["trip_update"]["trip"]["route_id"] ][item["stop_id"]].append(item["arrival"]["time"]) if ("departure" in item): times[ val["trip_update"]["trip"]["route_id"] ][item["stop_id"]].append( item["departure"]["time"] ) # The times dictionary might be edited again, so update the final_times # dictionary now. final_times.update(times) return final_times
def fetch(self): stop_info = copy.deepcopy(self.init_data) for feed_id in self.feed_ids: current_time = datetime.datetime.now() feed = gtfs_realtime_pb2.FeedMessage() response = requests.get( "http://datamine.mta.info/mta_esi.php?key=%s&feed_id=%d" % (self.api_key, feed_id)) feed.ParseFromString(response.content) for entity in feed.entity: if entity.trip_update: for update in entity.trip_update.stop_time_update: if update.stop_id in self.stop_ids: stop_info[update.stop_id][ "line"] = entity.trip_update.trip.route_id nyct_extension = entity.trip_update.trip.Extensions[ nyct_trip_descriptor] stop_info[update.stop_id][ "direction"] = self.get_direction( nyct_extension) train_time = update.arrival.time if train_time <= 0: train_time = update.departure.time train_time = datetime.datetime.fromtimestamp( train_time) train_time = math.trunc( ((train_time - current_time).total_seconds()) / 60) if train_time >= 0: times = stop_info[update.stop_id].get( "next_train_times", []) times.append(train_time) times.sort() stop_info[ update.stop_id]["next_train_times"] = times return stop_info
def refresh(line_code): """Retrieves fresh data from the MTA. Args: line_code (str): code for the line or lines whose trips will be retrieved. Returns: list: list of dicts with train data. """ """""" feed = gtfs_realtime_pb2.FeedMessage() response = requests.get( f'https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs{str(line_code)}', headers={'x-api-key': os.environ.get('API_KEY')}) feed.ParseFromString(response.content) # Taking the transit data from its specific format into a dictionary subway_feed = protobuf_to_dict(feed) subway_feed = [r for r in subway_feed['entity'] if 'alert' not in r.keys()] consol_feed = list( map( lambda dict_tuple: dict(ChainMap(*dict_tuple[1])), groupby(sorted(subway_feed, key=lambda t: extract_trip_details(t)), key=lambda t: extract_trip_details(t)))) return consol_feed
def RequestsWrite(APIkey, feed_id): ''' This function takes APIkey and feed_id as an input, and Requests MTA subway real-time status, and Writes a gtfs file. ''' url = 'http://datamine.mta.info/mta_esi.php?key=' + APIkey + '&feed_id=' + str( feed_id) response = requests.get(url) feed = gtfs_realtime_pb2.FeedMessage() feed.ParseFromString(response.content) timestamp = datetime.fromtimestamp(feed.header.timestamp) FolderName = '%04d'%(timestamp.year) + '%02d'%(timestamp.month) + '/' \ + '%04d'%(timestamp.year) + '%02d'%(timestamp.month) + '%02d'%(timestamp.day) if not os.path.isdir(FolderName): os.makedirs(FolderName) file = open( FolderName + "/gtfs_" + str(feed_id) + '_' + str(timestamp).replace(" ", "-").replace(":", "-") + ".gtfs", "wb") file.write(response.content) file.close()
def produce_trip_updates(self): # initialise the feed message parser from Google feed = gtfs_realtime_pb2.FeedMessage() # get the response from the api response = requests.get(self.act_lr_api_url) # pass the response to the Parser feed.ParseFromString(response.content) # loop through feed entity for entity in feed.entity: if entity.HasField('trip_update'): # only get trip updates trip_id = entity.trip_update.trip.trip_id print('**Trip ID**: ', trip_id) stop_time_updates = entity.trip_update.stop_time_update print('Count of updates: ', len(stop_time_updates)) print('--------') update_json = MessageToDict(entity.trip_update) update_json['id'] = int(entity.id) update_json = json.dumps(update_json) self.kafka_producer.produce(self.kafka_topic, update_json.encode('utf-8')) self.kafka_producer.flush()
def recording(today_trip,previous): #returnlist={} trip_start_time={} vehicle_list = get_vehicle() count=0 filename = 'dc-circulator-'+'tripupdates'+ext fm = g.FeedMessage() fm.header.gtfs_realtime_version = '2.0' fm.header.incrementality = g._FEEDHEADER_INCREMENTALITY.values_by_name['FULL_DATASET'].number for ve in vehicle_list: schedule_relation = 0 if ve.tripID==0: #skip the trip ID = 0 records continue route = routes_dict[ve.route_id] if get_gtfsid(ve.nextStopID): stop_ID = get_gtfsid(ve.nextStopID) else: #skip the next stop ID not in ddot list record continue eta = get_eta(ve.route_id,ve.nextStopID) if eta: eta2= time.strftime('%H:%M:%S', time.localtime(int(eta))) else: #skip no eta records continue if ve_to_garage_distance(ve.lat,ve.lon): #skip vehicles that are in/near garage continue start_time, start_time2 = get_trip_info(ve.tripID,ve.nextStopID) # start_time is the raw start time align wit optibus system, start_time2 is the one round to 10 mins if start_time2=='0': start_time = start_time2 = previous[ve.vehicle_id][-1] start_date = str(get_service_day()) #get rid of the toooo old eta time # wait for bishop peak responds t = datetime.datetime.strptime(start_date+start_time, '%Y%m%d%H:%M:%S') if time.mktime(t.timetuple())>eta: continue #if it's terminal stops and not sure about the trip id, reocrd the trip id serve the route trip_list=[] for trip_id,tup in today_trip.iteritems(): if tup.route_id == route: trip_list.append(trip_id) if int(stop_ID) in seq_dict[trip_list[0]] and int(stop_ID) in seq_dict[trip_list[1]]: trip_id_final=str(trip_list[0])+" "+str(trip_list[1]) seq = str(seq_dict[trip_list[0]].index(int(stop_ID))) + " " + str(seq_dict[trip_list[1]].index(int(stop_ID))) #write to protobuf as start of next trip but do no return seq_to_proto='0' for i in trip_list: if seq_dict[i].index(int(stop_ID))==0: trip_id_to_proto = str(i) direction_proto = today_trip[i].direction_id if trip_id_to_proto not in trip_start_time.keys(): trip_start_time[trip_id_to_proto] = [start_time2] else: if start_time2 in trip_start_time[trip_id_to_proto]:#which is duplicate schedule_relation=1 else: trip_start_time[trip_id_to_proto].append(start_time2) tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction_proto, \ trip_id_to_proto, ve.tripID, start_time,start_time2, start_date,\ ve.lat, ve.lon, schedule_relation,\ stop_ID, seq_to_proto, eta, ve.timestamp) elif int(stop_ID) in seq_dict[trip_list[0]]: trip_id_final=str(trip_list[0]) direction = today_trip[int(trip_id_final)].direction_id seq = get_seq(trip_id_final, stop_ID) if ve.vehicle_id in previous.keys(): if previous[ve.vehicle_id].trip_id == trip_id_final: start_time2 = previous[ve.vehicle_id].start_time2 if trip_id_final not in trip_start_time.keys(): trip_start_time[trip_id_final] = [start_time2] else: if start_time2 in trip_start_time[trip_id_final]:#which is duplicate schedule_relation=1 else: trip_start_time[trip_id_final].append(start_time2) tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction, \ trip_id_final, ve.tripID, start_time, start_time2, start_date,\ ve.lat, ve.lon, schedule_relation, \ stop_ID, seq, eta, ve.timestamp) elif int(stop_ID) in seq_dict[trip_list[1]]: trip_id_final=str(trip_list[1]) direction = today_trip[int(trip_id_final)].direction_id seq = get_seq(trip_id_final, stop_ID) if ve.vehicle_id in previous.keys(): if previous[ve.vehicle_id].trip_id == trip_id_final: start_time2 = previous[ve.vehicle_id].start_time2 if trip_id_final not in trip_start_time.keys(): trip_start_time[trip_id_final] = [start_time2] else: if start_time2 in trip_start_time[trip_id_final]:#which is duplicate schedule_relation=1 else: trip_start_time[trip_id_final].append(start_time2) tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction, \ trip_id_final, ve.tripID, start_time,start_time2, start_date,\ ve.lat, ve.lon, schedule_relation,\ stop_ID, seq, eta, ve.timestamp) #keep the start_time consistant through out the whole trip instead of updating based on input if ve.vehicle_id in previous.keys(): if previous[ve.vehicle_id].trip_id!= trip_id_final and len(trip_id_final) >2:#9 to 9 10 if previous[ve.vehicle_id].seq == 0: if trip_id_final not in trip_start_time.keys(): trip_start_time[trip_id_final] = [start_time2] else: if start_time2 in trip_start_time[trip_id_final]:#which is duplicate schedule_relation=1 else: trip_start_time[trip_id_final].append(start_time2) tuplelist[ve.vehicle_id] = previous[ve.vehicle_id]._replace(bishop_trip = ve.tripID, start_time=start_time, start_time2=start_time2,lat=ve.lat,lon=ve.lon, eta=eta, timestamp=ve.timestamp, schedule_relation=schedule_relation)#relation?? else: if ve.tripID == previous[ve.vehicle_id].bishop_trip: seq = get_seq(previous[ve.vehicle_id].trip_id, stop_ID) tuplelist[ve.vehicle_id] = previous[ve.vehicle_id]._replace(lat=ve.lat,lon=ve.lon,seq=seq,eta=eta,timestamp=ve.timestamp) else: seq='0' trip_list.remove(int(previous[ve.vehicle_id].trip_id)) # renew the trip id # trip_list.remove(int(previous[ve.vehicle_id].trip_id_final)) trip_id_final = str(trip_list[0]) direction = today_trip[int(trip_id_final)].direction_id if trip_id_final not in trip_start_time.keys(): trip_start_time[trip_id_final] = [start_time2] else: if start_time2 in trip_start_time[trip_id_final]:#which is duplicate schedule_relation=1 else: trip_start_time[trip_id_final].append(start_time2) tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction, \ trip_id_final, ve.tripID, start_time,start_time2, start_date,\ ve.lat, ve.lon, schedule_relation,\ stop_ID, seq, eta, ve.timestamp) if ve.vehicle_id in tuplelist.keys(): write_protobuf(tuplelist[ve.vehicle_id],fm) count+=1
import requests import os from google.transit import gtfs_realtime_pb2 MTA_KEY = os.environ.get('MTA_KEY') feed = gtfs_realtime_pb2.FeedMessage() r = requests.get("http://datamine.mta.info/mta_esi.php?key=" + MTA_KEY + "&feed_id=1", allow_redirects=True) feed.ParseFromString(r.content) for i in range(len(feed.entity)): print(feed.entity[i])
def get_realtime_data(link): feed = gtfs_realtime_pb2.FeedMessage() response = requests.get(link) feed.ParseFromString(response.content) return feed