def get_realtime(agency, mode):
URL = transit_agencies.get(agency, mode)
if URL == None:
return None
response = urllib.urlopen(URL)
feed = gtfs_realtime_pb2.FeedMessage()
feed.ParseFromString(response.read())
return feed
def get_realtime(agency, mode): URL = transit_agencies.get(agency, mode) if URL == None: return None response = urllib.urlopen(URL) feed = gtfs_realtime_pb2.FeedMessage() feed.ParseFromString(response.read()) return feed
def get_static(agency):
feed = {}
pathname = "./agencies/" + agency + "/"
#IF LOCAL FILES EXISTS
if path.exists(pathname):
for f in os.listdir(pathname):
if f[-4:] == ".txt" or f[-4:] == ".csv": #if its a txt file
with open(pathname + "/" + str(f)) as csvfile:
print "f[:-4] " + f[:-4]
feed[f[:-4]] = csv2df(csvfile)
return feed
#PULL NEW INFORMATION
request = requests.get(transit_agencies.get(agency, "static"), stream=True)
if request.status_code != 200:
print "Error! Did not reach"
if not path.exists(pathname):
os.makedirs(pathname)
#Unzip GTFS static
buf = request.raw.read()
zipdata = StringIO()
zipdata.write(buf)
with open(pathname + "gtfs.zip", "w") as zipout:
zipout.write(buf)
z = zipfile.ZipFile(zipdata)
z.extractall(pathname)
# format static feed
for f in z.namelist():
with z.open(f) as csvfile:
feed[f[:-4]] = csv2df(csvfile).rename(columns=lambda s: str(s.decode('ascii', 'ignore')))
for f in REQUIRED_GTFS_FILES:
if f not in feed:
print "Incomplete GTFS dataset"
return None
z.close()
return feed
def interpret(in_tabletype, in_agency, in_route, static_feed, trip_update_feed, alert_feed, vehicle_position_feed):
tables = {}
trip2pattern = {}
trip2vehicle = {}
agency_id = transit_agencies.get(in_agency, "id") #for bart
route_id = in_route
if in_tabletype == "static" :
#Agencies
tablefunctions.agencies(tables, static_feed, arg_agency_id=agency_id)
#Routes
tablefunctions.routes(tables, static_feed, arg_agency_id=agency_id)
#Stops
tablefunctions.stops(tables, static_feed, arg_agency_id=agency_id)
#Route Stop Seq
tablefunctions.route_stop_seq(tables, static_feed,
arg_agency_id=agency_id, arg_route_id=route_id, trip2pattern=trip2pattern)
#Run Pattern
tablefunctions.runPattern(tables=tables, static_feed=static_feed, arg_agency_id=agency_id)
#Schedules: gets some data from RunPattern table
tablefunctions.schedules(tables, static_feed, arg_agency_id=agency_id, trip2pattern=trip2pattern)
#Table Points
tablefunctions.points(tables, static_feed, arg_agency_id=agency_id, trip2pattern=trip2pattern)
tablefunctions.big_points(tables, static_feed, arg_agency_id=agency_id, trip2pattern=trip2pattern)
#Table Route_point_Seq
tablefunctions.route_point_seq(tables, static_feed, arg_agency_id=agency_id, trip2pattern=trip2pattern)
elif in_tabletype == "transfer" :
### ----- Task 2 ----------
#Table Transfers
tablefunctions.transfers(tables, static_feed, arg_agency_id=agency_id)
elif in_tabletype == "realtime" :
### ---- Task 3 -----------------
#Table GPS FIXES
tablefunctions.gps_fixes(tables, static_feed, trip_update_feed, alert_feed, vehicle_position_feed,
arg_agency_id=agency_id, trip2pattern=trip2pattern)
#Table Transit ETA
tablefunctions.transit_eta(tables, static_feed, trip_update_feed, alert_feed, vehicle_position_feed,
arg_agency_id=agency_id, trip2pattern=trip2pattern)
def get_static(agency):
feed = {}
pathname = "./agencies/" + agency + "/"
#IF LOCAL FILES EXISTS
if path.exists(pathname):
for f in os.listdir(pathname):
if f[-4:] == ".txt" or f[-4:] == ".csv": #if its a txt file
with open(pathname + "/" + str(f)) as csvfile:
print "f[:-4] " + f[:-4]
feed[f[:-4]] = csv2df(csvfile)
return feed
#PULL NEW INFORMATION
request = requests.get(transit_agencies.get(agency, "static"), stream=True)
if request.status_code != 200:
print "Error! Did not reach"
if not path.exists(pathname):
os.makedirs(pathname)
#Unzip GTFS static
buf = request.raw.read()
zipdata = StringIO()
zipdata.write(buf)
with open(pathname + "gtfs.zip", "w") as zipout:
zipout.write(buf)
z = zipfile.ZipFile(zipdata)
z.extractall(pathname)
# format static feed
for f in z.namelist():
with z.open(f) as csvfile:
feed[f[:-4]] = csv2df(csvfile).rename(
columns=lambda s: str(s.decode('ascii', 'ignore')))
for f in REQUIRED_GTFS_FILES:
if f not in feed:
print "Incomplete GTFS dataset"
return None
z.close()
return feed
def get_static(agency, refresh):
pathname = "./agencies/" + agency + "/"
feed = {}
get_new_feed = False
if path.exists(pathname + "raw_csv/") and not refresh:
with open(pathname + "gtfs.zip", "r") as zipout:
checksum = hasher(zipout, hashlib.md5()) # did not work
# read csv files
for f in os.listdir(pathname + "raw_csv/"):
if f[-4:] == ".csv":
with open(pathname + "raw_csv/" + f, 'rb') as csvfile:
feed[f[:-4]] = df_helper.csv2df(csvfile)
# if required GTFS data is missing, request new data anyway
for f in REQUIRED_GTFS_FILES:
if f not in feed:
get_new_feed = True
# if feed_end_date has passed, request new data anyway
if not get_new_feed:
feed_end_date = feed["feed_info"].feed_end_date[
0] if "feed_info" in feed else feed["calendar"].end_date[0]
feed_end_date = datetime.strptime(str(feed_end_date), '%Y%m%d')
current_date = datetime.now()
logging.info("feed_end_date = %s",
feed_end_date.strftime("%Y%m%d"))
logging.info("current_date = %s", current_date.strftime("%Y%m%d"))
get_new_feed = feed_end_date < current_date
if not get_new_feed:
logging.debug("Read from local")
return feed, checksum
# request GTFS-Static
request = requests.get(transit_agencies.get(agency, "static"), stream=True)
# if unsuccessful
if request.status_code != 200:
return None
if not path.exists(pathname + "raw/"):
os.makedirs(pathname + "raw/")
# unzip GTFS static
buf = request.raw.read()
zipdata = StringIO()
zipdata.write(buf)
with open(pathname + "gtfs.zip", "w") as zipout:
zipout.write(buf)
with open(pathname + "gtfs.zip", "r") as zipout:
checksum = hasher(zipout, hashlib.md5())
z = zipfile.ZipFile(zipdata)
z.extractall(pathname + "raw/")
# format static feed
for f in z.namelist():
with z.open(f) as csvfile:
feed[f[:-4]] = df_helper.csv2df(csvfile).rename(
columns=lambda s: str(s.decode('ascii', 'ignore')))
for f in REQUIRED_GTFS_FILES:
if f not in feed:
logging.error("Incomplete GTFS dataset")
return None
z.close()
# write csv files
if not path.exists(pathname + "raw_csv/"):
os.makedirs(pathname + "raw_csv/")
for fn, df in feed.iteritems():
df.to_csv(pathname + "raw_csv/" + fn + ".csv", sep=',', index=False)
logging.debug("Read from online")
return feed, checksum
tables[table].to_csv(pathname + table + '.csv',
sep=',',
index=False)
else:
df_helper.df2sql(tables[table],
table,
login=login,
exist_flag=('replace' if refresh else 'append'))
# check if newer timestamp
# process entity
tables = {}
trip2pattern = {}
trip2vehicle = {}
agency_id = transit_agencies.get(agency, "id")
# Static Feed
# ---- Task 1 ----
# Agency
# int agency_id -> 'agency_id' int(10) unsigned
# required string agency_name -> 'agency_name' varchar(255)
# required string agency_url -> 'agency_url' varchar(255)
# required string agency_timezone -> 'agency_timezone' smallint(6)
# optional string agency_lang -> 'agency_lang' varchar(255)
# optional string agency_phone -> 'agency_phone' varchar(255)
# required string agency_timezone -> 'timezone_name' varchar(45)
# PRIMARY KEY ('agency_id')
# KEY ('agency_timezone')
if can_read_table('Agency') and not refresh:
logging.debug('Read from database')
return df_helper.sql2df(table, login)
def write_table(table):
if local:
tables[table].to_csv(pathname + table + '.csv', sep = ',', index = False)
else:
df_helper.df2sql(tables[table], table, login=login, exist_flag=('replace' if refresh else 'append'))
# check if newer timestamp
# process entity
tables = {}
trip2pattern = {}
trip2vehicle = {}
agency_id = transit_agencies.get(agency, "id")
# Static Feed
# ---- Task 1 ----
# Agency
# int agency_id -> 'agency_id' int(10) unsigned
# required string agency_name -> 'agency_name' varchar(255)
# required string agency_url -> 'agency_url' varchar(255)
# required string agency_timezone -> 'agency_timezone' smallint(6)
# optional string agency_lang -> 'agency_lang' varchar(255)
# optional string agency_phone -> 'agency_phone' varchar(255)
# required string agency_timezone -> 'timezone_name' varchar(45)
# PRIMARY KEY ('agency_id')
# KEY ('agency_timezone')
if can_read_table('Agency') and not refresh:
def get_static(agency, refresh):
pathname = "./agencies/" + agency + "/"
feed = {}
get_new_feed = False
if path.exists(pathname + "raw_csv/") and not refresh:
with open(pathname + "gtfs.zip", "r") as zipout:
checksum = hasher(zipout, hashlib.md5()) # did not work
# read csv files
for f in os.listdir(pathname + "raw_csv/"):
if f[-4:] == ".csv":
with open(pathname + "raw_csv/" + f, 'rb') as csvfile:
feed[f[:-4]] = df_helper.csv2df(csvfile)
# if required GTFS data is missing, request new data anyway
for f in REQUIRED_GTFS_FILES:
if f not in feed:
get_new_feed = True
# if feed_end_date has passed, request new data anyway
if not get_new_feed:
feed_end_date = feed["feed_info"].feed_end_date[0] if "feed_info" in feed else feed["calendar"].end_date[0]
feed_end_date = datetime.strptime(str(feed_end_date),'%Y%m%d')
current_date = datetime.now()
logging.info("feed_end_date = %s", feed_end_date.strftime("%Y%m%d"))
logging.info("current_date = %s", current_date.strftime("%Y%m%d"))
get_new_feed = feed_end_date < current_date
if not get_new_feed:
logging.debug("Read from local")
return feed, checksum
# request GTFS-Static
request = requests.get(transit_agencies.get(agency, "static"), stream = True)
# if unsuccessful
if request.status_code != 200:
return None
if not path.exists(pathname + "raw/"):
os.makedirs(pathname + "raw/")
# unzip GTFS static
buf = request.raw.read()
zipdata = StringIO()
zipdata.write(buf)
with open(pathname + "gtfs.zip", "w") as zipout:
zipout.write(buf)
with open(pathname + "gtfs.zip", "r") as zipout:
checksum = hasher(zipout, hashlib.md5())
z = zipfile.ZipFile(zipdata)
z.extractall(pathname + "raw/")
# format static feed
for f in z.namelist():
with z.open(f) as csvfile:
feed[f[:-4]] = df_helper.csv2df(csvfile).rename(columns = lambda s: str(s.decode('ascii', 'ignore')))
for f in REQUIRED_GTFS_FILES:
if f not in feed:
logging.error("Incomplete GTFS dataset")
return None
z.close()
# write csv files
if not path.exists(pathname + "raw_csv/"):
os.makedirs(pathname + "raw_csv/")
for fn, df in feed.iteritems():
df.to_csv(pathname + "raw_csv/" + fn + ".csv", sep = ',', index = False)
logging.debug("Read from online")
return feed, checksum