def main():
# need the GTFS Python bindings
from google.transit import gtfs_realtime_pb2
import urllib
import json
import socket
import time
# create socket connection to hostname/port on which a TCP GeoEvent input is running
tcpSocket = socket.create_connection(("localhost", 5565))
# polling model - run, wait 5 seconds, run, wait, run, wait, etc
while True:
feed = gtfs_realtime_pb2.FeedMessage()
# this particular feed is from OV Api (http://www.openov.nl/)
response = urllib.urlopen(
'http://gtfs.openov.nl/gtfs-rt/vehiclePositions.pb')
# read the Protocal Buffers (.pb) file
feed.ParseFromString(response.read())
# loop through feed entities
for entity in feed.entity:
# check for a vehicle in feed entity
if entity.HasField('vehicle'):
# build a simple id,lon,lat message to send to GeoEvent.
msg = str(entity.vehicle.vehicle.label) + "," + \
str(entity.vehicle.position.longitude) + "," + \
str(entity.vehicle.position.latitude) + "\n"
# send message
tcpSocket.send(msg)
#print(msg)
time.sleep(5)
def make_vehicle_list(pb_file_list, folder_path):
vehicle_list = []
bad_vehicle_header_list = []
dirname = folder_path
for pb_file in pb_file_list:
pb_file_path = os.path.join(dirname, pb_file)
with open(pb_file_path, 'rb') as f:
feed = gtfs_realtime_pb2.FeedMessage()
feed.ParseFromString(f.read())
json_obj = MessageToJson(feed)
json_parsed = json.loads(json_obj)
if 'entity' in json_parsed.keys():
for vehicles_idx in range(len(json_parsed['entity'])):
vehicle_dict = {}
j_out = json_parsed['entity'][vehicles_idx]
if 'vehicle' in j_out.keys():
if 'position' in j_out['vehicle'] and 'trip' in j_out[
'vehicle']:
vehicle_dict['vehicle_id'] = j_out['vehicle'][
'vehicle']['id']
vehicle_dict['timestamp'] = j_out['vehicle'][
'timestamp']
vehicle_dict['trip_id'] = j_out['vehicle']['trip'][
'tripId']
vehicle_dict['route_id'] = j_out['vehicle']['trip'][
'routeId']
vehicle_dict['vehicle_lat'] = j_out['vehicle'][
'position']['latitude']
vehicle_dict['vehicle_long'] = j_out['vehicle'][
'position']['longitude']
#trip_id = j_out['vehicle']['trip']['tripId']
#route_id = j_out['vehicle']['trip']['routeId']
#vehicle_dict['shape_id'] = get_shape_id_from_triproute(trip_id, route_id, schedule_df)
vehicle_list.append(vehicle_dict)
else:
bad_vehicle_header_list.append(json_parsed['header'])
else:
bad_vehicle_header_list.append(json_parsed['header'])
else:
bad_vehicle_header_list.append(json_parsed['header'])
return vehicle_list, bad_vehicle_header_list
def parser(url):
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(url)
feed.ParseFromString(response.content)
count = len(feed.entity)
for val in feed.entity:
for idx in range(count):
try:
values = {
"route": val.trip_update.trip.route_id,
"arrival":
val.trip_update.stop_time_update[idx].arrival.time,
"stop_id":
val.trip_update.stop_time_update[idx].stop_id
}
feed_list.append(values)
except Exception as e:
# logging.debug(e)
pass
return feed_list
def _parse(self, data) -> pd.DataFrame:
feed = gtfs_realtime_pb2.FeedMessage()
feed.ParseFromString(data)
feed_timestamp = feed.header.timestamp
def gen():
for entity in feed.entity:
trip_id = entity.vehicle.trip.trip_id
timestamp = entity.vehicle.timestamp
# stop_id, peron_id = parse_stop_id(entity.vehicle.stop_id)
stop_sequence = entity.vehicle.current_stop_sequence
block_id, trip_num, service_id = parse_trip_id(trip_id)
yield block_id, trip_num, service_id, stop_sequence, timestamp, feed_timestamp
columns = [
'block_id', 'trip_num', 'service_id', 'stop_sequence', 'timestamp',
'feed_timestamp'
]
vehicle_positions_df = pd.DataFrame(gen(), columns=columns)
# vehicle_positions_df.set_index(['block_id', 'trip_num', 'service_id'], inplace=True)
return vehicle_positions_df
class Gtfsrt(BaseAPI):
"""API for real time General Transit Feed Specifications:
https://api.actransit.org/transit/gtfsrt/"""
api = "gtfsrt"
protobuf = gtfs_realtime_pb2.FeedMessage()
def __repr__(self):
return "ACTransit({})".format(self.__class__.__name__)
@api_method
def alerts(self):
pass
@api_method
def tripupdates(self):
pass
@api_method
def vehicles(self):
pass
def run():
# Get gps data
feed = gtfs_realtime_pb2.FeedMessage()
# PMPML GTFS username and password
username = "******"
password = "******"
response = requests.get(
'http://117.232.125.138/tms/data/gtfs/vehicle-positions.pb',
auth=(username, password))
feed.ParseFromString(response.content)
feed = MessageToDict(feed)
for entity in feed['entity']:
data = entity['vehicle']['position']
if entity['vehicle']['vehicle']['id'] == "ANT74":
print(entity['vehicle']['vehicle']['id'], " = ", data)
data.update({"route": entity['vehicle']['trip']['tripId']})
gps_result.update({entity['vehicle']['vehicle']['id']: data})
print(entity)
def _refresh_vehicle_position_data(self):
positions = dict()
occupancies = dict()
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(self._positions_url).content
feed.ParseFromString(response)
for entity in feed.entity:
if entity.HasField('vehicle'):
# Handle
position = entity.vehicle.position
trip_id = entity.vehicle.trip.trip_id
positions[trip_id] = {
"latitude": position.latitude,
"longitude": position.longitude,
"bearing": position.bearing,
"speed": position.speed * 3.6, # m/s to kph
}
occupancies[trip_id] = entity.vehicle.occupancy_status
self._occupancies = occupancies
self._positions = positions
self._positions_last_updated = int(time.time())
def _get_vehicle_positions(self):
from google.transit import gtfs_realtime_pb2
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(self._vehicle_position_url,
headers=self._headers)
if response.status_code != 200:
_LOGGER.error("updating vehicle positions got {}:{}.".format(
response.status_code, response.content))
feed.ParseFromString(response.content)
positions = {}
for entity in feed.entity:
vehicle = entity.vehicle
if not vehicle.trip.route_id:
# Vehicle is not in service
continue
positions[vehicle.vehicle.id] = vehicle.position
return positions
def _update_trains(self, id):
try:
#print("Updating train info for ID: {0}...".format(id))
feed = gtfs_realtime_pb2.FeedMessage()
#print("Retriving info from MTA datamine...")
response = requests.get(self.url + id,
headers={"x-api-key": self.key})
feed.ParseFromString(response.content)
# Parse Entire Feed
for e in feed.entity:
# Record Route of Trip
route = e.vehicle.trip.route_id
# Exclude particular routes and record the Stop
if route in self.exclude:
continue
elif route == "L":
i = int(e.vehicle.current_stop_sequence)
if i < 10:
stop = "L0{0}".format(i)
else:
stop = "L{0}".format(i)
else:
stop = e.vehicle.stop_id[:3]
if stop in self.combine.keys():
stop = self.combine[stop]
if stop not in self.trains.keys():
# Check if stop is already in dict
self.trains[stop] = [route]
elif route in self.trains[stop]:
# Check if route is already in dict's list
continue
else:
self.trains[stop].append(route)
except ConnectionError as e:
print("Unable to connect to the MTA Data Stream...")
except DecodeError:
print("Unable to decode the message from " + id)
except KeyboardInterrupt:
print("Exiting...")
sys.exit()
def create_gtfs_feed(vehicle_states: dict) -> gtfs_realtime_pb2.FeedMessage:
"""Transform out internal state into a FeedMessage structure that GTFS expects"""
# https://developers.google.com/transit/gtfs-realtime/reference#message-feedmessage"""
feed_message = gtfs_realtime_pb2.FeedMessage()
feed_message.header.gtfs_realtime_version = "2.0"
feed_message.header.incrementality = gtfs_realtime_pb2.FeedHeader.FULL_DATASET
feed_message.header.timestamp = int(datetime.now().timestamp())
created_entries = 0
for board_id, state in vehicle_states.items():
if state.route_id in ROUTE_ID_MAP:
# create the FeedEntity and populate it with the values from our own state structure
feed_entity = feed_message.entity.add()
feed_entity.id = f"{created_entries}-{board_id}"
# https://developers.google.com/transit/gtfs-realtime/reference#message-vehicledescriptor
feed_entity.vehicle.vehicle.id = state.rtu_id
feed_entity.vehicle.vehicle.label = state.board_name
feed_entity.vehicle.timestamp = int(state.last_seen)
# https://developers.google.com/transit/gtfs-realtime/reference#message-position
feed_entity.vehicle.position.longitude = state.lon
feed_entity.vehicle.position.latitude = state.lat
feed_entity.vehicle.position.speed = state.speed
feed_entity.vehicle.position.bearing = state.direction
# https://developers.google.com/transit/gtfs-realtime/reference#message-tripdescriptor
# note that the route_id is different here, ROUTE_ID_MAP[entity.route_id] is what we
# usually call "name_concise"
feed_entity.vehicle.trip.route_id = ROUTE_ID_MAP[state.route_id]
created_entries += 1
else:
if state.route_id not in UNKNOWN_ROUTES:
LOG.warning(
"Skipping board %s from unknown upstream_route_id=%s",
board_id, state.route_id)
UNKNOWN_ROUTES.add(state.route_id)
LOG.debug("Produced %i entries in the feed", created_entries)
return feed_message
def _update_route_statuses(self, vehicle_positions):
"""Get the latest data."""
from google.transit import gtfs_realtime_pb2
class StopDetails:
def __init__(self, arrival_time, position):
self.arrival_time = arrival_time
self.position = position
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(self._trip_update_url, headers=self._headers)
if response.status_code != 200:
_LOGGER.error("updating route status got {}:{}".format(
response.status_code, response.content))
feed.ParseFromString(response.content)
departure_times = {}
for entity in feed.entity:
if entity.HasField('trip_update'):
route_id = entity.trip_update.trip.route_id
if route_id not in departure_times:
departure_times[route_id] = {}
for stop in entity.trip_update.stop_time_update:
stop_id = stop.stop_id
if not departure_times[route_id].get(stop_id):
departure_times[route_id][stop_id] = []
# Use stop departure time; fall back on stop arrival time if not available
details = StopDetails(
datetime.datetime.fromtimestamp(stop.arrival.time),
vehicle_positions.get(entity.trip_update.vehicle.id))
departure_times[route_id][stop_id].append(details)
# Sort by arrival time
for route in departure_times:
for stop in departure_times[route]:
departure_times[route][stop].sort(key=lambda t: t.arrival_time)
self.info = departure_times
def GetTimes():
for tr in trains:
tr.arrival_times = []
feed = gtfs_realtime_pb2.FeedMessage()
response = urllib.request.urlopen(
'http://datamine.mta.info/mta_esi.php?key=581ad7b399d1eddcccec3a31d0e6e00d&feed_id=1'
)
feed.ParseFromString(response.read())
now = datetime.now()
for entity in feed.entity:
if entity.HasField('trip_update'):
for stu in entity.trip_update.stop_time_update:
for tr in trains:
if stu.stop_id == tr.stop_id and entity.trip_update.trip.route_id[
0] == tr.route_id[0]:
mintoarrive = (datetime.fromtimestamp(stu.arrival.time)
- now).total_seconds() / 60
if mintoarrive >= 0:
tr.arrival_times.append(round(mintoarrive, 0))
break
for tr in trains:
tr.arrival_times.sort()
def loopdwlmta(x): ## send request to api mta.info using my own key
feed = gtfs_realtime_pb2.FeedMessage()
response = urllib.urlopen(
'http://datamine.mta.info/mta_esi.php?key=5a8f6f4375e9f4bdea9b0c86afeaf911'
)
feed.ParseFromString(response.read())
## looping the content downloaded
tmpSys = datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S')
i = 1
for entity in feed.entity:
if entity.HasField('trip_update'):
## trip update: 3 data characterisitcs
TripIdi = entity.trip_update.trip.trip_id
Routei = entity.trip_update.trip.route_id
## looking to record first 2 predicted arrival time
## looking to record first 2 predicted arrival time
ArriveTime = []
stop_id = []
tripcomplete = []
p3trip = min(len(entity.trip_update.stop_time_update), 3)
j = 0
while j < p3trip:
tripcomplete = []
#print j
ArrivalTime = entity.trip_update.stop_time_update[
j].arrival.time
stop_id = entity.trip_update.stop_time_update[j].stop_id
station_nth = j
tripcomplete.extend([
tmpSys, TripIdi, Routei, station_nth, stop_id, ArrivalTime
])
#print tripcomplete
print tripcomplete, type(tripcomplete)
tripcomplstr = ','.join([str(i) for i in tripcomplete])
savefile(tripcomplstr)
j += 1
def _get_vehicle_positions(self):
from google.transit import gtfs_realtime_pb2
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(self._vehicle_position_url, headers=self._headers)
if response.status_code == 200:
_LOGGER.info("Successfully updated vehicle positions - {}".format(response.status_code))
else:
_LOGGER.error("updating vehicle positions got {}:{}.".format(response.status_code, response.content))
feed.ParseFromString(response.content)
positions = {}
for entity in feed.entity:
vehicle = entity.vehicle
if not vehicle.trip.trip_id:
# Vehicle is not in service
continue
_LOGGER.debug("......Adding position for trip id {} position latitude {} longitude {} ".format(vehicle.trip.trip_id,vehicle.position.latitude,vehicle.position.longitude))
positions[vehicle.trip.trip_id] = vehicle.position
return positions
def getStops(_, part):
feed = gtfs_realtime_pb2.FeedMessage()
for fn, contents in part:
hr = int(fn[-5:-3])
dt = fn[-8:-6]
# RUSH HOUR ONLY
# the gtfs reports are 5 hours behind
# from the file contents
# therefore, add 5 hours
if hr in [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 0, 1]: #RUSH HOUR ONLY
feed.ParseFromString(contents)
for entity in feed.entity:
if entity.HasField('vehicle'):
vehicle = protobuf_to_dict(entity.vehicle)
if vehicle.get('current_status') != 1:
continue
try:
line = vehicle['trip']['route_id'].rstrip('X') # fold express into normal
if line not in ['1', '2', '3', '4', '5', '6', 'GS', 'L', 'SI']:
print 'weird line', line
continue
if 'stop_id' in vehicle:
stop = vehicle['stop_id']
else:
# L and SI stop at every station, need to use
stop = '%d%s' % (vehicle['current_stop_sequence'], vehicle['trip']['trip_id'][-1])
#key = (line, stop)
timestamp = vehicle['timestamp'] # datetime.datetime.utcfromtimestamp(vehicle['timestamp'])
# merge 2 and 3 line
if line in ['2','3']:
line = '23'
if line in ['4','5']:
line = '45'
yield ((dt,line,stop),[timestamp])
#yield ((dt,line,stop),timestamp)
#yield (line, timestamp)
except:
print 'weird vehicle', vehicle
continue
def makeProtoBuffer(trips, locations):
'''
Create a protobuffer object from the trips object.
Trips is expected to be a dictionary keyed to trip_id strings.
The values are dictionaries with `stop_sequence` and `dev` items.
'''
feed = gtfs_realtime_pb2.FeedMessage()
# Set Header
feed.header.gtfs_realtime_version = "2.0"
feed.header.incrementality = 0
feed.header.timestamp = int(time.time())
# Create an trip update entity for each trip
for trip_id, trip in trips.items():
entity = gtfs_realtime_pb2.FeedEntity()
entity.id = trip_id
entity.trip_update.trip.trip_id = trip_id
stop_time_update = entity.trip_update.StopTimeUpdate()
stop_time_update.stop_sequence = trip['stop_sequence']
stop_time_update.arrival.delay = trip['dev']
entity.trip_update.stop_time_update.append(stop_time_update)
feed.entity.append(entity)
# Create vehicle location for each vehicle location
for vehicle_id, info in locations.items():
entity = gtfs_realtime_pb2.FeedEntity()
entity.id = vehicle_id
entity.vehicle.timestamp = info['timestamp']
entity.vehicle.trip.trip_id = info['trip']['trip_id']
entity.vehicle.vehicle.id = vehicle_id
entity.vehicle.position.latitude = info['position']['latitude']
entity.vehicle.position.longitude = info['position']['longitude']
entity.vehicle.position.bearing = info['position']['bearing']
entity.vehicle.position.speed = info['position']['speed']
feed.entity.append(entity)
return feed
def getTripStatus(trip_id, startStationName, endStationName):
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(
'http://datamine.mta.info/mta_esi.php?key=%s&feed_id=1' % key)
feed.ParseFromString(response.content)
departureTime = None
arrivalTime = None
current_stop_sequence = None
current_status = None
timestamp = None
for entity in feed.entity:
if entity.HasField('trip_update'):
if entity.trip_update.trip.trip_id == trip_id:
for stop_time_update in entity.trip_update.stop_time_update:
if stop_time_update.stop_id == startStationName:
departureTime = int(stop_time_update.departure.time)
# print entity
if stop_time_update.stop_id == endStationName:
arrivalTime = int(stop_time_update.arrival.time)
# print stop_time_update
if entity.HasField('vehicle'):
if entity.vehicle.trip.trip_id == trip_id:
current_stop_sequence = int(
entity.vehicle.current_stop_sequence)
current_status = str(entity.vehicle.current_status)
timestamp = int(entity.vehicle.timestamp)
if entity.HasField('alert'):
for informed_entity in entity.alert.informed_entity:
if informed_entity.trip.trip_id == trip_id:
file.write(str(entity.alert.informed_entity))
if entity.alert.HasField('header_text'):
file.write(str(entity.alert.header_text))
return (departureTime, arrivalTime, current_stop_sequence, current_status,
timestamp)
def get_times():
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get('http://datamine.mta.info/mta_esi.php?key={}&feed_id=2'.format(api_key))
feed.ParseFromString(response.content)
jsonObj = MessageToJson(feed)
data = json.loads(jsonObj)
stops = {
"S": [],
"N": []
}
time_now = int(time.time())
if( 'entity' in data):
for entity in data['entity']:
if('tripUpdate' in entity):
for sched in entity['tripUpdate']['stopTimeUpdate']:
if( stop_id in sched['stopId']):
stops[sched['stopId'].replace(stop_id, "")].append( {"time": round((int(sched['arrival']['time']) - time_now) / 60) ,"direction": smap[entity['tripUpdate']['stopTimeUpdate'][-1]["stopId"]] } )
stops['S'] = sorted(stops['S'], key=lambda k: k['time'])
stops['N'] = sorted(stops['N'], key=lambda k: k['time'])
else:
print('Malformatted MTA data', data)
with open('data.json', 'w') as outfile:
json.dump(jsonObj, outfile)
# for entity in data['entity']:
# if('tripUpdate' in entity):
# for sched in entity['tripUpdate']['stopTimeUpdate']:
# if( stop_id in sched['stopId']):
# stops[sched['stopId'].replace(stop_id, "")].append( round((int(sched['arrival']['time']) - time_now) / 60) )
return stops
async def get_mb_records(s: ClientSession) -> MetroBus:
"""
Get records from MB api
"""
date_updated = datetime.now()
feed = gtfs_realtime_pb2.FeedMessage()
async with s.get(url=os.getenv("URL_API_MB")) as response:
response.encoding = "utf-8"
if response.status != 200:
return
feed.ParseFromString(await response.read())
for entity in feed.entity:
yield MetroBusRaw(
vehicle_id=entity.vehicle.vehicle.id,
vehicle_label=entity.vehicle.vehicle.label,
vehicle_status=entity.vehicle.current_status,
date_updated=date_updated,
position_latitude=entity.vehicle.position.latitude,
position_longitude=entity.vehicle.position.longitude,
)
def process_dump_contents(feed, contents, trip_dates, stops, fetchtime, feed_tz, start_date_str, start_date_str_after_midnight):
global global_stats
global_stats = {
'trip_updates_found': 0,
'stop_updates_found': 0,
'stop_updates_stored': 0,
'missing_trips': 0,
'unscheduled_trips': 0,
'unscheduled_tripdates': 0,
'missing_stops': 0,
'missing_trip_id': 0,
'missing_tripdates': 0,
'missing_tripdates_but_trip_existed': 0,
'missing_routes': 0,
}
feed_message = gtfs_realtime_pb2.FeedMessage()
feed_message.ParseFromString(contents)
threshold = int((fetchtime + timedelta(minutes=5)).timestamp())
for entity in feed_message.entity:
if entity.HasField('trip_update'):
process_trip_update(feed, trip_dates, stops, feed_tz, entity.trip_update, threshold, start_date_str, start_date_str_after_midnight)
for stat in global_stats:
print(f'{stat}: {global_stats[stat]}')
def MTAAPICall(route, stopcodes):
# Helper function to make request to MTA GFTS API
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(
'http://datamine.mta.info/mta_esi.php?key={}&feed_id=1'.format(
mta_api_key))
feed.ParseFromString(response.content)
currentTime = feed.header.timestamp # Time of API call
arrivals = [] # List of subway arrival times
stationcode = 'N/A' # If station code not found, return N/A
for entity in feed.entity: # For each trip in the response
if entity.trip_update.trip.route_id == route: # Matches the correct subway line
for stop in (
entity.trip_update.stop_time_update
): # Parse each stop for all active subways in the correct subway line
for possibleStation in stopcodes: # Parse each station code for the correct station (each station can have multiple station codes)
if stop.stop_id == (possibleStation
): # Matches the direction and station
stationcode = stop.stop_id # If station code is matched, assign the correct station code
arrivals.append(stop.arrival.time
) # Add arrival time of train to list
return currentTime, arrivals, stationcode # Return API call time, list of arriving times, and correct station code
def pull_septa_rail():
ssl._create_default_https_context = ssl._create_unverified_context
tripfeed = gtfs_realtime_pb2.FeedMessage()
#request for trip feed
tripresponse = requests.get(
'http://www3.septa.org/gtfsrt/septarail-pa-us/Trip/rtTripUpdates.pb',
allow_redirects=True)
tripfeed.ParseFromString(tripresponse.content)
# Check if request is working
#print('There are {} trip updates in the dataset.'.format(len(tripfeed.entity)))
if len(tripfeed.entity) == 0:
return None
dict_obj_trip = MessageToDict(tripfeed)
collector = []
for trip in dict_obj_trip['entity']:
if trip['tripUpdate'].get('stopTimeUpdate') is not None:
row = OrderedDict()
row['trip_id'] = trip['tripUpdate'].get('trip', {}).get('tripId')
row['id'] = trip.get('id')
row['delay'] = trip['tripUpdate']['stopTimeUpdate'][0].get(
'arrival', {}).get('delay')
row['trip_stop_id'] = trip['tripUpdate']['stopTimeUpdate'][0].get(
'stopId')
row['trip_stop_seq'] = trip['tripUpdate']['stopTimeUpdate'][0].get(
'stopSequence')
row['timestamp'] = dict_obj_trip['header']['timestamp']
collector.append(row)
df = pd.DataFrame(collector)
df['humantime'] = df.apply(
lambda row: datetime.datetime.fromtimestamp(int(row['timestamp'])),
axis=1)
df['agency'] = 'SEPTA'
df['mode'] = "Rail"
df['region'] = "New York/Philadelphia"
return df
def find_station(station_ids, trains):
feed = gtfs_realtime_pb2.FeedMessage()
with open("secret.json") as f:
key = json.load(f)["mta"]
final_times = {}
# Get a set of the appropriate MTA feed_id. This saves a ton of time in
# terms of network requests.
feeds = {FEED_ID[line] for line in trains}
for feed_id in feeds:
response = requests.get(
f"http://datamine.mta.info/mta_esi.php?key={key}&feed_id={feed_id}"
)
feed.ParseFromString(response.content)
# Turn feed into a regular dictionary.
d = protobuf_to_dict(feed)
times = collections.defaultdict(
lambda: collections.defaultdict(lambda: [])
)
for i, val in enumerate(d["entity"]):
if "trip_update" in val:
for item in val["trip_update"]["stop_time_update"]:
# Only looking for specific stations.
if (item["stop_id"] in station_ids):
if ("arrival" in item):
times[
val["trip_update"]["trip"]["route_id"]
][item["stop_id"]].append(item["arrival"]["time"])
if ("departure" in item):
times[
val["trip_update"]["trip"]["route_id"]
][item["stop_id"]].append(
item["departure"]["time"]
)
# The times dictionary might be edited again, so update the final_times
# dictionary now.
final_times.update(times)
return final_times
def fetch(self):
stop_info = copy.deepcopy(self.init_data)
for feed_id in self.feed_ids:
current_time = datetime.datetime.now()
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(
"http://datamine.mta.info/mta_esi.php?key=%s&feed_id=%d" %
(self.api_key, feed_id))
feed.ParseFromString(response.content)
for entity in feed.entity:
if entity.trip_update:
for update in entity.trip_update.stop_time_update:
if update.stop_id in self.stop_ids:
stop_info[update.stop_id][
"line"] = entity.trip_update.trip.route_id
nyct_extension = entity.trip_update.trip.Extensions[
nyct_trip_descriptor]
stop_info[update.stop_id][
"direction"] = self.get_direction(
nyct_extension)
train_time = update.arrival.time
if train_time <= 0:
train_time = update.departure.time
train_time = datetime.datetime.fromtimestamp(
train_time)
train_time = math.trunc(
((train_time - current_time).total_seconds()) /
60)
if train_time >= 0:
times = stop_info[update.stop_id].get(
"next_train_times", [])
times.append(train_time)
times.sort()
stop_info[
update.stop_id]["next_train_times"] = times
return stop_info
def refresh(line_code):
"""Retrieves fresh data from the MTA.
Args:
line_code (str): code for the line or lines whose trips will be retrieved.
Returns:
list: list of dicts with train data.
""" """"""
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(
f'https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs{str(line_code)}',
headers={'x-api-key': os.environ.get('API_KEY')})
feed.ParseFromString(response.content)
# Taking the transit data from its specific format into a dictionary
subway_feed = protobuf_to_dict(feed)
subway_feed = [r for r in subway_feed['entity'] if 'alert' not in r.keys()]
consol_feed = list(
map(
lambda dict_tuple: dict(ChainMap(*dict_tuple[1])),
groupby(sorted(subway_feed, key=lambda t: extract_trip_details(t)),
key=lambda t: extract_trip_details(t))))
return consol_feed
def RequestsWrite(APIkey, feed_id):
'''
This function takes APIkey and feed_id as an input, and
Requests MTA subway real-time status, and Writes a gtfs file.
'''
url = 'http://datamine.mta.info/mta_esi.php?key=' + APIkey + '&feed_id=' + str(
feed_id)
response = requests.get(url)
feed = gtfs_realtime_pb2.FeedMessage()
feed.ParseFromString(response.content)
timestamp = datetime.fromtimestamp(feed.header.timestamp)
FolderName = '%04d'%(timestamp.year) + '%02d'%(timestamp.month) + '/' \
+ '%04d'%(timestamp.year) + '%02d'%(timestamp.month) + '%02d'%(timestamp.day)
if not os.path.isdir(FolderName):
os.makedirs(FolderName)
file = open(
FolderName + "/gtfs_" + str(feed_id) + '_' +
str(timestamp).replace(" ", "-").replace(":", "-") + ".gtfs", "wb")
file.write(response.content)
file.close()
def produce_trip_updates(self):
# initialise the feed message parser from Google
feed = gtfs_realtime_pb2.FeedMessage()
# get the response from the api
response = requests.get(self.act_lr_api_url)
# pass the response to the Parser
feed.ParseFromString(response.content)
# loop through feed entity
for entity in feed.entity:
if entity.HasField('trip_update'): # only get trip updates
trip_id = entity.trip_update.trip.trip_id
print('**Trip ID**: ', trip_id)
stop_time_updates = entity.trip_update.stop_time_update
print('Count of updates: ', len(stop_time_updates))
print('--------')
update_json = MessageToDict(entity.trip_update)
update_json['id'] = int(entity.id)
update_json = json.dumps(update_json)
self.kafka_producer.produce(self.kafka_topic,
update_json.encode('utf-8'))
self.kafka_producer.flush()
def recording(today_trip,previous):
#returnlist={}
trip_start_time={}
vehicle_list = get_vehicle()
count=0
filename = 'dc-circulator-'+'tripupdates'+ext
fm = g.FeedMessage()
fm.header.gtfs_realtime_version = '2.0'
fm.header.incrementality = g._FEEDHEADER_INCREMENTALITY.values_by_name['FULL_DATASET'].number
for ve in vehicle_list:
schedule_relation = 0
if ve.tripID==0:
#skip the trip ID = 0 records
continue
route = routes_dict[ve.route_id]
if get_gtfsid(ve.nextStopID):
stop_ID = get_gtfsid(ve.nextStopID)
else:
#skip the next stop ID not in ddot list record
continue
eta = get_eta(ve.route_id,ve.nextStopID)
if eta: eta2= time.strftime('%H:%M:%S', time.localtime(int(eta)))
else:
#skip no eta records
continue
if ve_to_garage_distance(ve.lat,ve.lon):
#skip vehicles that are in/near garage
continue
start_time, start_time2 = get_trip_info(ve.tripID,ve.nextStopID) # start_time is the raw start time align wit optibus system, start_time2 is the one round to 10 mins
if start_time2=='0':
start_time = start_time2 = previous[ve.vehicle_id][-1]
start_date = str(get_service_day())
#get rid of the toooo old eta time # wait for bishop peak responds
t = datetime.datetime.strptime(start_date+start_time, '%Y%m%d%H:%M:%S')
if time.mktime(t.timetuple())>eta:
continue
#if it's terminal stops and not sure about the trip id, reocrd the trip id serve the route
trip_list=[]
for trip_id,tup in today_trip.iteritems():
if tup.route_id == route:
trip_list.append(trip_id)
if int(stop_ID) in seq_dict[trip_list[0]] and int(stop_ID) in seq_dict[trip_list[1]]:
trip_id_final=str(trip_list[0])+" "+str(trip_list[1])
seq = str(seq_dict[trip_list[0]].index(int(stop_ID))) + " " + str(seq_dict[trip_list[1]].index(int(stop_ID)))
#write to protobuf as start of next trip but do no return
seq_to_proto='0'
for i in trip_list:
if seq_dict[i].index(int(stop_ID))==0:
trip_id_to_proto = str(i)
direction_proto = today_trip[i].direction_id
if trip_id_to_proto not in trip_start_time.keys():
trip_start_time[trip_id_to_proto] = [start_time2]
else:
if start_time2 in trip_start_time[trip_id_to_proto]:#which is duplicate
schedule_relation=1
else:
trip_start_time[trip_id_to_proto].append(start_time2)
tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction_proto, \
trip_id_to_proto, ve.tripID, start_time,start_time2, start_date,\
ve.lat, ve.lon, schedule_relation,\
stop_ID, seq_to_proto, eta, ve.timestamp)
elif int(stop_ID) in seq_dict[trip_list[0]]:
trip_id_final=str(trip_list[0])
direction = today_trip[int(trip_id_final)].direction_id
seq = get_seq(trip_id_final, stop_ID)
if ve.vehicle_id in previous.keys():
if previous[ve.vehicle_id].trip_id == trip_id_final:
start_time2 = previous[ve.vehicle_id].start_time2
if trip_id_final not in trip_start_time.keys():
trip_start_time[trip_id_final] = [start_time2]
else:
if start_time2 in trip_start_time[trip_id_final]:#which is duplicate
schedule_relation=1
else:
trip_start_time[trip_id_final].append(start_time2)
tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction, \
trip_id_final, ve.tripID, start_time, start_time2, start_date,\
ve.lat, ve.lon, schedule_relation, \
stop_ID, seq, eta, ve.timestamp)
elif int(stop_ID) in seq_dict[trip_list[1]]:
trip_id_final=str(trip_list[1])
direction = today_trip[int(trip_id_final)].direction_id
seq = get_seq(trip_id_final, stop_ID)
if ve.vehicle_id in previous.keys():
if previous[ve.vehicle_id].trip_id == trip_id_final:
start_time2 = previous[ve.vehicle_id].start_time2
if trip_id_final not in trip_start_time.keys():
trip_start_time[trip_id_final] = [start_time2]
else:
if start_time2 in trip_start_time[trip_id_final]:#which is duplicate
schedule_relation=1
else:
trip_start_time[trip_id_final].append(start_time2)
tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction, \
trip_id_final, ve.tripID, start_time,start_time2, start_date,\
ve.lat, ve.lon, schedule_relation,\
stop_ID, seq, eta, ve.timestamp)
#keep the start_time consistant through out the whole trip instead of updating based on input
if ve.vehicle_id in previous.keys():
if previous[ve.vehicle_id].trip_id!= trip_id_final and len(trip_id_final) >2:#9 to 9 10
if previous[ve.vehicle_id].seq == 0:
if trip_id_final not in trip_start_time.keys():
trip_start_time[trip_id_final] = [start_time2]
else:
if start_time2 in trip_start_time[trip_id_final]:#which is duplicate
schedule_relation=1
else:
trip_start_time[trip_id_final].append(start_time2)
tuplelist[ve.vehicle_id] = previous[ve.vehicle_id]._replace(bishop_trip = ve.tripID, start_time=start_time, start_time2=start_time2,lat=ve.lat,lon=ve.lon, eta=eta, timestamp=ve.timestamp, schedule_relation=schedule_relation)#relation??
else:
if ve.tripID == previous[ve.vehicle_id].bishop_trip:
seq = get_seq(previous[ve.vehicle_id].trip_id, stop_ID)
tuplelist[ve.vehicle_id] = previous[ve.vehicle_id]._replace(lat=ve.lat,lon=ve.lon,seq=seq,eta=eta,timestamp=ve.timestamp)
else:
seq='0'
trip_list.remove(int(previous[ve.vehicle_id].trip_id)) # renew the trip id # trip_list.remove(int(previous[ve.vehicle_id].trip_id_final))
trip_id_final = str(trip_list[0])
direction = today_trip[int(trip_id_final)].direction_id
if trip_id_final not in trip_start_time.keys():
trip_start_time[trip_id_final] = [start_time2]
else:
if start_time2 in trip_start_time[trip_id_final]:#which is duplicate
schedule_relation=1
else:
trip_start_time[trip_id_final].append(start_time2)
tuplelist[ve.vehicle_id] = VehicleInfo(ve.vehicle_id, route, direction, \
trip_id_final, ve.tripID, start_time,start_time2, start_date,\
ve.lat, ve.lon, schedule_relation,\
stop_ID, seq, eta, ve.timestamp)
if ve.vehicle_id in tuplelist.keys():
write_protobuf(tuplelist[ve.vehicle_id],fm)
count+=1
import requests
import os
from google.transit import gtfs_realtime_pb2
MTA_KEY = os.environ.get('MTA_KEY')
feed = gtfs_realtime_pb2.FeedMessage()
r = requests.get("http://datamine.mta.info/mta_esi.php?key=" + MTA_KEY +
"&feed_id=1",
allow_redirects=True)
feed.ParseFromString(r.content)
for i in range(len(feed.entity)):
print(feed.entity[i])
def get_realtime_data(link):
feed = gtfs_realtime_pb2.FeedMessage()
response = requests.get(link)
feed.ParseFromString(response.content)
return feed
