def Generate_Shapes_AGOL():
'''Generate preliminary shapes for each route by calculating the optimal
route along the network using the ArcGIS Online route services.'''
arcpy.AddMessage("Generating on-street route shapes via ArcGIS Online for routes of the following types, if they exist in your data:")
for rtype in route_type_Street_textlist:
arcpy.AddMessage(rtype)
arcpy.AddMessage("(This step may take a while for large GTFS datasets.)")
global NoRouteGenerated
NoRouteGenerated = []
Too_Many_Stops = []
global badStops
# ----- Generate a route for each sequence -----
arcpy.AddMessage("- Generating routes using ArcGIS Online")
# Set up input parameters for route request
service_params = {}
service_params["travelMode"] = AGOLRouteHelper.travel_mode
service_params["returnRoutes"] = True
service_params["outputLines"] = "esriNAOutputLineTrueShapeWithMeasure"
service_params["returnDirections"] = False
service_params["outSR"] = WGSCoords_WKID
# Create the output feature class
arcpy.management.CreateFeatureclass(outGDB, outRoutesfcName, "POLYLINE", '', '', '', WGSCoords)
arcpy.management.AddField(outRoutesfc, "Name", "TEXT")
# Set up insertCursors for output shapes polylines and stop sequences
# Have to open an edit session to have two simultaneous InsertCursors.
edit = arcpy.da.Editor(outGDB)
ucursor = arcpy.da.InsertCursor(outRoutesfc, ["SHAPE@", "Name"])
cur = arcpy.da.InsertCursor(outSequencePoints, ["SHAPE@X", "SHAPE@Y", "shape_id", "sequence", "stop_id"])
edit.startEditing()
# Generate routes with AGOL for sequences we want to make street-based shapes for.
sequences_Streets = []
num_shapes = len(sequence_shape_dict)
next_threshold = 10
progress = 0.0
num_routes_calculated = 0
for sequence in sequence_shape_dict:
# Print some progress indicators
progress += 1
percdone = (progress / num_shapes) * 100
if percdone > next_threshold:
last_threshold = percdone - percdone%10
arcpy.AddMessage("%s%% finished" % str(int(last_threshold)))
next_threshold = last_threshold + 10
shape_id = sequence_shape_dict[sequence]
route_id = sequence[0]
route_type = RouteDict[route_id][4]
if route_type not in route_types_Street:
continue
if len(sequence[1]) > AGOLRouteHelper.route_stop_limit:
# There are too many stops in this route to solve with the online services.
Too_Many_Stops.append(shape_id)
continue
stopstring = ""
sequence_num = 1
pt = arcpy.Point()
for stop in sequence[1]:
try:
stop_lat = stoplatlon_dict[stop][0]
stop_lon = stoplatlon_dict[stop][1]
except KeyError:
badStops.append(stop)
sequence_num += 1
continue
# Add stop sequences to points fc for user to look at.
pt.X = float(stop_lon)
pt.Y = float(stop_lat)
cur.insertRow((float(stop_lon), float(stop_lat), shape_id, sequence_num, stop))
sequence_num = sequence_num + 1
# Prepare string of stops to pass to AGOL
stopstring += str(stop_lon) + ", " + str(stop_lat) + "; "
service_params["stops"] = stopstring[:-2]
routeshapes, errors = AGOLRouteHelper.generate_routes_from_AGOL_as_polylines(AGOLRouteHelper.token, service_params)
if errors:
if "User does not have permissions to access" in errors:
arcpy.AddError("ArcGIS Online route generation failed. Please ensure that your ArcGIS Online account \
has routing privileges and sufficient credits for this analysis.")
raise CustomError
arcpy.AddWarning("ArcGIS Online route generation for shape_id %s failed. A straight-line shape will be generated for this shape_id instead. %s" % (shape_id, errors))
NoRouteGenerated.append(shape_id)
continue
for route in routeshapes: # actually, only one shape should be returned here, but loop just in case
ucursor.insertRow((route, shape_id))
num_routes_calculated += 1
del ucursor
del cur
edit.stopEditing(True)
arcpy.AddMessage("Done generating route shapes with ArcGIS Online. Number of ArcGIS Online routes calculated: %s" % str(num_routes_calculated))
if Too_Many_Stops:
arcpy.AddWarning("On-street route shapes for the following shape_ids could \
not be generated because the number of stops in the route exceeds the ArcGIS Online \
service limit of %s stops. Straight-line route shapes will be generated for these \
shape_ids instead:" % str(AGOLRouteHelper.route_stop_limit))
arcpy.AddWarning(sorted(Too_Many_Stops))
NoRouteGenerated.append(shape for shape in Too_Many_Stops)
def RunStep1():
'''Run Step 1 - Generate feature class of shapes for input to Step 2, which
generates the actual GTFS shapes.txt file.'''
try:
# It's okay to overwrite stuff.
orig_overwrite = arcpy.env.overwriteOutput
arcpy.env.overwriteOutput = True
# Check version
if ArcVersion == "10.0":
arcpy.AddError("You must have ArcGIS 10.2.1 or higher (or ArcGIS Pro) to run this \
tool. You have ArcGIS version %s." % ArcVersion)
raise CustomError
if ArcVersion in ["10.1", "10.2"]:
arcpy.AddWarning("Warning! You can run Step 1 of this tool in \
ArcGIS 10.1 or 10.2, but you will not be able to run Step 2 without ArcGIS \
10.2.1 or higher (or ArcGIS Pro). You have ArcGIS version %s." % ArcVersion)
if ProductName == "ArcGISPro" and ArcVersion in ["1.0", "1.1", "1.1.1"]:
arcpy.AddError("You must have ArcGIS Pro 1.2 or higher to run this \
tool. You have ArcGIS Pro version %s." % ArcVersion)
raise CustomError
if useAGOL and ArcVersion in ["10.2.1", "10.2.2"]:
arcpy.AddError("You must have ArcGIS 10.3 (or ArcGIS Pro) to run the ArcGIS Online \
version of this tool. You have ArcGIS version %s." % ArcVersion)
raise CustomError
# Check out the Network Analyst extension license
if useNA:
if arcpy.CheckExtension("Network") == "Available":
arcpy.CheckOutExtension("Network")
else:
arcpy.AddError("The Network Analyst license is unavailable.")
raise CustomError
if useAGOL:
# Get the user's ArcGIS Online token. They must already be signed in to use this tool.
# That way we don't need to collect a username and password.
# But, you can't run this script in standalone python.
AGOLRouteHelper.get_token()
if AGOLRouteHelper.token == None:
arcpy.AddError("Unable to retrieve token for ArcGIS Online. To use this tool, \
you must be signed in to ArcGIS Online with an account that has routing privileges and credits. \
Talk to your organization's ArcGIS Online administrator for assistance.")
raise CustomError
arcpy.AddMessage("Successfully retrieved ArcGIS Online token.")
# ----- Set up the run, fix some inputs -----
# Input format is a string separated by a ; ("0 - Tram, Streetcar, Light rail;3 - Bus;5 - Cable car")
global route_type_Straight_textlist, route_type_Street_textlist, route_types_Straight, route_types_Street
if in_route_type_Street:
route_type_Street_textlist = in_route_type_Street.split(";")
else:
route_type_Street_textlist = []
if in_route_type_Straight:
route_type_Straight_textlist = in_route_type_Straight.split(";")
else:
route_type_Straight_textlist = []
route_types_Street = []
route_types_Straight = []
for rtype in route_type_Street_textlist:
route_types_Street.append(int(rtype.split(" - ")[0].strip('\'')))
for rtype in route_type_Straight_textlist:
route_types_Straight.append(int(rtype.split(" - ")[0].strip('\'')))
# Set curb approach based on side of road vehicles drive on
global CurbApproach
driveSide = "Right"
if driveSide == "Right":
CurbApproach = 1 #"Right side of vehicle"
else:
CurbApproach = 2 #"Left side of vehcle"
# Uturn policy is explained here: http://resources.arcgis.com/en/help/main/10.1/index.html#//00480000000n000000
global UTurns
if UTurn_input == "Allowed anywhere":
UTurns = "ALLOW_UTURNS"
elif UTurn_input == "Allowed only at intersections and dead ends":
UTurns = "ALLOW_DEAD_ENDS_AND_INTERSECTIONS_ONLY"
elif UTurn_input == "Allowed only at dead ends":
UTurns = "ALLOW_DEAD_ENDS_ONLY"
elif UTurn_input == "Not allowed anywhere":
UTurns = "NO_UTURNS"
# Sometimes, when locating stops, they snap to the closest street, which is
# actually a side street instead of the main road where the stop is really
# located. The Route results consequently have a lot of little loops or
# spikes sticking out the side. Sometimes we can improve results by
# locating stops on network junctions instead of streets. Sometimes this
# messes up the results, however, but we allow the users to try.
global search_criteria
if useJunctions:
search_criteria = []
NAdesc = arcpy.Describe(inNetworkDataset)
for source in NAdesc.sources:
if source.sourceType in ["JunctionFeature", "SystemJunction"]:
search_criteria.append([source.name, "SHAPE"])
else:
search_criteria.append([source.name, "NONE"])
else:
search_criteria = "#"
# Initialize a list for shapes that couldn't be generated from the route solver
global NoRouteGenerated
NoRouteGenerated = []
# Set up the outputs
global outGDB, outSequencePoints, outRoutesfc, outRoutesfcName, SQLDbase, outGDBName
if not outGDBName.lower().endswith(".gdb"):
outGDBName += ".gdb"
outGDB = os.path.join(outDir, outGDBName)
outSequencePointsName = "Stops_wShapeIDs"
outSequencePoints = os.path.join(outGDB, outSequencePointsName)
outRoutesfcName = "Shapes"
outRoutesfc = os.path.join(outGDB, outRoutesfcName)
SQLDbase = os.path.join(outGDB, "SQLDbase.sql")
# Create output geodatabase
arcpy.management.CreateFileGDB(outDir, outGDBName)
# ----- Connect to the SQL database -----
global c, conn
conn = sqlite3.connect(SQLDbase)
c = conn.cursor()
# ----- SQLize the GTFS data -----
try:
SQLize_GTFS(files_to_sqlize)
except:
arcpy.AddError("Error SQLizing the GTFS data.")
raise
# ----- Get lat/long for all stops and add to dictionary. Calculate location fields if necessary. -----
arcpy.AddMessage("Collecting and processing GTFS stop information...")
# Find all stops with lat/lon
global stoplatlon_dict
stoplatlon_dict = {}
stoplatlonfetch = '''
SELECT stop_id, stop_lat, stop_lon FROM stops
;'''
c.execute(stoplatlonfetch)
stoplatlons = c.fetchall()
for stop in stoplatlons:
# Add stop lat/lon to dictionary
stoplatlon_dict[stop[0]] = [stop[1], stop[2]]
# Calculate location fields for the stops and save them to a dictionary.
if useNA:
# Temporary feature class of stops for calculating location fields
arcpy.management.CreateFeatureclass(outGDB, "TempStopswLocationFields", "POINT", "", "", "", WGSCoords)
LocFieldStops = os.path.join(outGDB, "TempStopswLocationFields")
arcpy.management.AddField(LocFieldStops, "stop_id", "TEXT")
with arcpy.da.InsertCursor(LocFieldStops, ["SHAPE@X", "SHAPE@Y", "stop_id"]) as cur:
for stop in stoplatlons:
# Insert stop into fc for location field calculation
cur.insertRow((float(stop[2]), float(stop[1]), stop[0]))
# It would be easier to use CalculateLocations, but then we can't
# exclude restricted network elements.
# Instead, create a dummy Route layer and Add Locations
RLayer = arcpy.na.MakeRouteLayer(inNetworkDataset, "DummyLayer", impedanceAttribute,
restriction_attribute_name=restrictions).getOutput(0)
naSubLayerNames = arcpy.na.GetNAClassNames(RLayer)
stopsSubLayer = naSubLayerNames["Stops"]
fieldMappings = arcpy.na.NAClassFieldMappings(RLayer, stopsSubLayer)
fieldMappings["Name"].mappedFieldName = "stop_id"
arcpy.na.AddLocations(RLayer, stopsSubLayer, LocFieldStops, fieldMappings,
search_criteria=search_criteria,
snap_to_position_along_network="NO_SNAP",
exclude_restricted_elements="EXCLUDE")
if ProductName == "ArcGISPro":
StopsLayer = RLayer.listLayers(stopsSubLayer)[0]
else:
StopsLayer = arcpy.mapping.ListLayers(RLayer, stopsSubLayer)[0]
# Iterate over the located stops and create a dictionary of location fields
global stoplocfielddict
stoplocfielddict = {}
with arcpy.da.SearchCursor(StopsLayer, ["Name", "SourceID", "SourceOID", "PosAlong", "SideOfEdge"]) as cur:
for stop in cur:
locfields = [stop[1], stop[2], stop[3], stop[4]]
stoplocfielddict[stop[0]] = locfields
arcpy.management.Delete(StopsLayer)
arcpy.management.Delete(LocFieldStops)
# ----- Make dictionary of route info -----
arcpy.AddMessage("Collecting GTFS route information...")
# GTFS route_type information
#0 - Tram, Streetcar, Light rail. Any light rail or street level system within a metropolitan area.
#1 - Subway, Metro. Any underground rail system within a metropolitan area.
#2 - Rail. Used for intercity or long-distance travel.
#3 - Bus. Used for short- and long-distance bus routes.
#4 - Ferry. Used for short- and long-distance boat service.
#5 - Cable car. Used for street-level cable cars where the cable runs beneath the car.
#6 - Gondola, Suspended cable car. Typically used for aerial cable cars where the car is suspended from the cable.
#7 - Funicular. Any rail system designed for steep inclines.
route_type_dict = {0: "Tram, Streetcar, Light rail",
1: "Subway, Metro",
2: "Rail",
3: "Bus",
4: "Ferry",
5: "Cable car",
6: "Gondola, Suspended cable car",
7: "Funicular"}
# Find all routes and associated info.
global RouteDict
RouteDict = {}
routesfetch = '''
SELECT route_id, agency_id, route_short_name, route_long_name,
route_desc, route_type, route_url, route_color, route_text_color
FROM routes
;'''
c.execute(routesfetch)
routelist = c.fetchall()
for route in routelist:
# {route_id: [all route.txt fields + route_type_text]}
try:
route_type_text = route_type_dict[int(route[5])]
except:
route_type_text = "Other / Type not specified"
route[5] = '100'
RouteDict[route[0]] = [route[1], route[2], route[3], route[4], route[5],
route[6], route[7], route[8],
route_type_text]
# ----- Match trip_ids with route_ids -----
arcpy.AddMessage("Collecting GTFS trip information...")
global trip_route_dict
trip_route_dict = {}
triproutefetch = '''
SELECT trip_id, route_id FROM trips
;'''
c.execute(triproutefetch)
triproutelist = c.fetchall()
for triproute in triproutelist:
# {trip_id: route_id}
trip_route_dict[triproute[0]] = triproute[1]
# Find all trip_ids.
triplist = []
tripsfetch = '''
SELECT DISTINCT trip_id FROM stop_times
;'''
c.execute(tripsfetch)
alltrips = c.fetchall()
# ----- Create ordered stop sequences -----
arcpy.AddMessage("Calculating unique sequences of stops...")
# Select stops in that trip
global sequence_shape_dict, shape_trip_dict
sequence_shape_dict = {}
shape_trip_dict = {}
shape_id = 1
for trip in alltrips:
stopfetch = "SELECT stop_id, stop_sequence FROM stop_times WHERE trip_id='%s'" % trip[0]
c.execute(stopfetch)
selectedstops = c.fetchall()
# Sort the stop list by sequence.
selectedstops.sort(key=operator.itemgetter(1))
stop_sequence = ()
for stop in selectedstops:
stop_sequence += (stop[0],)
route_id = trip_route_dict[trip[0]]
sequence_shape_dict_key = (route_id, stop_sequence)
try:
sh = sequence_shape_dict[sequence_shape_dict_key]
shape_trip_dict.setdefault(sh, []).append(trip[0])
except KeyError:
sequence_shape_dict[sequence_shape_dict_key] = shape_id
shape_trip_dict.setdefault(shape_id, []).append(trip[0])
shape_id += 1
numshapes = shape_id - 1
arcpy.AddMessage("Your GTFS data contains %s unique shapes." % str(numshapes))
# ----- Figure out which routes go with which shapes and update trips table -----
global shape_route_dict
shape_route_dict = {}
for shape in shape_trip_dict:
shaperoutes = []
for trip in shape_trip_dict[shape]:
shaperoutes.append(trip_route_dict[trip])
# Update the trips table with the shape assigned to the trip
updatetripstablestmt = "UPDATE trips SET shape_id='%s' WHERE trip_id='%s'" % (shape, trip)
c.execute(updatetripstablestmt)
conn.commit()
shaperoutesset = set(shaperoutes)
for route in shaperoutesset:
shape_route_dict.setdefault(shape, []).append(route)
conn.close()
# ----- Generate street and straight routes -----
# Create a points feature class for the stops to input for Routes
# We'll save this so users can see the stop sequences with the shape_ids.
arcpy.management.CreateFeatureclass(outGDB, outSequencePointsName, "POINT", "", "", "", WGSCoords)
arcpy.management.AddField(outSequencePoints, "stop_id", "TEXT")
arcpy.management.AddField(outSequencePoints, "shape_id", "LONG")
arcpy.management.AddField(outSequencePoints, "sequence", "LONG")
arcpy.management.AddField(outSequencePoints, "CurbApproach", "SHORT")
if useNA:
arcpy.management.AddField(outSequencePoints, "SourceID", "LONG")
arcpy.management.AddField(outSequencePoints, "SourceOID", "LONG")
arcpy.management.AddField(outSequencePoints, "PosAlong", "DOUBLE")
arcpy.management.AddField(outSequencePoints, "SideOfEdge", "LONG")
# Flag for whether we created the output fc in from Routes or if we need
# to create it in the straight-line part
Created_Street_Output = False
# Generate shapes following the streets
if route_types_Street:
if useNA:
Generate_Shapes_Street()
Created_Street_Output = True
elif useAGOL:
Generate_Shapes_AGOL()
Created_Street_Output = True
# Generate routes as straight lines between stops
if route_types_Straight or NoRouteGenerated:
Generate_Shapes_Straight(Created_Street_Output)
global badStops
if badStops:
badStops = sorted(list(set(badStops)))
messageText = "Your stop_times.txt lists times for the following stops which are not included in your stops.txt file. These stops have been ignored. "
if ProductName == "ArcGISPro":
messageText += str(badStops)
else:
messageText += unicode(badStops)
arcpy.AddWarning(messageText)
# ----- Add route information to output feature class -----
arcpy.AddMessage("Adding GTFS route information to output shapes feature class")
# Explicitly set max allowed length for route_desc. Some agencies are wordy.
max_route_desc_length = 250
arcpy.management.AddField(outRoutesfc, "shape_id", "LONG")
arcpy.management.AddField(outRoutesfc, "route_id", "TEXT")
arcpy.management.AddField(outRoutesfc, "route_short_name", "TEXT")
arcpy.management.AddField(outRoutesfc, "route_long_name", "TEXT")
arcpy.management.AddField(outRoutesfc, "route_desc", "TEXT", "", "", max_route_desc_length)
arcpy.management.AddField(outRoutesfc, "route_type", "SHORT")
arcpy.management.AddField(outRoutesfc, "route_type_text", "TEXT")
with arcpy.da.UpdateCursor(outRoutesfc, ["Name", "shape_id", "route_id",
"route_short_name", "route_long_name", "route_desc",
"route_type", "route_type_text"]) as ucursor:
for row in ucursor:
shape_id = row[0]
route_id = shape_route_dict[int(shape_id)][0]
route_short_name = RouteDict[route_id][1]
route_long_name = RouteDict[route_id][2]
route_desc = RouteDict[route_id][3]
route_type = RouteDict[route_id][4]
route_type_text = RouteDict[route_id][8]
row[0] = row[0]
row[1] = shape_id
row[2] = route_id
row[3] = route_short_name
row[4] = route_long_name
row[5] = route_desc[0:max_route_desc_length] if route_desc else route_desc #logic handles the case where it's empty
row[6] = route_type
row[7] = route_type_text
ucursor.updateRow(row)
# ----- Finish things up -----
# Add output to map.
if useNA:
arcpy.SetParameterAsText(11, outRoutesfc)
arcpy.SetParameterAsText(12, outSequencePoints)
elif useAGOL:
arcpy.SetParameterAsText(5, outRoutesfc)
arcpy.SetParameterAsText(6, outSequencePoints)
else:
arcpy.SetParameterAsText(6, outRoutesfc)
arcpy.SetParameterAsText(7, outSequencePoints)
arcpy.AddMessage("Done!")
arcpy.AddMessage("Output generated in " + outGDB + ":")
arcpy.AddMessage("- Shapes")
arcpy.AddMessage("- Stops_wShapeIDs")
except CustomError:
arcpy.AddError("Error generating shapes feature class from GTFS data.")
pass
except:
raise
finally:
arcpy.env.overwriteOutput = orig_overwrite
