def readBusRecords(shapePath, vistaGraph, gtfsShapes, unusedShapeIDs,
                   restrictService):
    # Read in the routes information:
    print("INFO: Read GTFS routesfile...", file=sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, RoutesEntry>"

    # Read in the stops information:
    print("INFO: Read GTFS stopsfile...", file=sys.stderr)
    gtfsStops = gtfs.fillStops(shapePath, vistaGraph.gps)
    "@type gtfsStops: dict<int, StopsEntry>"

    # Read in the trips information:
    print("INFO: Read GTFS tripsfile...", file=sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes,
                                                gtfsRoutes, unusedShapeIDs,
                                                restrictService)
    "@type gtfsTrips: dict<int, TripsEntry>"
    "@type unusedTripIDs: set<int>"

    # Read stop times information:
    print("INFO: Read GTFS stop times...", file=sys.stderr)
    gtfsStopTimes = gtfs.fillStopTimes(shapePath, gtfsTrips, gtfsStops,
                                       unusedTripIDs)
    "@type gtfsStopTimes: dict<TripsEntry, list<StopTimesEntry>>"

    return gtfsRoutes, gtfsStops, gtfsTrips, gtfsStopTimes
def readBusRecords(shapePath, vistaGraph, gtfsShapes, unusedShapeIDs, restrictService):
    # Read in the routes information:
    print("INFO: Read GTFS routesfile...", file=sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, RoutesEntry>"
    
    # Read in the stops information:
    print("INFO: Read GTFS stopsfile...", file=sys.stderr)
    gtfsStops = gtfs.fillStops(shapePath, vistaGraph.gps)
    "@type gtfsStops: dict<int, StopsEntry>"
    
    # Read in the trips information:
    print("INFO: Read GTFS tripsfile...", file=sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes, gtfsRoutes, unusedShapeIDs, restrictService)
    "@type gtfsTrips: dict<int, TripsEntry>"
    "@type unusedTripIDs: set<int>"
        
    # Read stop times information:
    print("INFO: Read GTFS stop times...", file=sys.stderr)
    gtfsStopTimes = gtfs.fillStopTimes(shapePath, gtfsTrips, gtfsStops, unusedTripIDs)
    "@type gtfsStopTimes: dict<TripsEntry, list<StopTimesEntry>>"

    return gtfsRoutes, gtfsStops, gtfsTrips, gtfsStopTimes
def filterRoutes(gtfsNodes, shapePath, gtfsShapes, routeRestrictFilename, inclusiveFlag = False):
    """
    filterRoutes removes all GTFS nodes whose routes are not listed in the given file.
    @type gtfsNodes: dict<int, list<path_engine.PathEnd>>
    @type shapePath: str
    @type gtfsShapes: gtfs.ShapesEntry
    @type routeRestrictFilename: str
    @rtype dict<int, list<path_engine.PathEnd>>
    """
    # TODO: This type of filtering capability would probably best go in another centralized
    # location where multiple methods can all act upon the filter list and save processing time.

    if (routeRestrictFilename is None) or (len(routeRestrictFilename) == 0):
        return gtfsNodes

    print("INFO: Read GTFS routesfile...", file = sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, gtfs.RoutesEntry>"
    
    print("INFO: Read GTFS tripsfile...", file = sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes, gtfsRoutes)
    "@type gtfsTrips: dict<int, gtfs.TripsEntry>"
    
    routeIDSet = set()
    "@type routeIDSet: set<int>"
    with open(routeRestrictFilename, 'r') as inFile:
        # Go through the lines of the file:
        for fileLine in inFile:
            if len(fileLine) > 0:                
                routeIDSet.add(int(fileLine))
    
    shapeIDSet = set()
    "@type shapeIDSet: set<int>"

    # Resolve shapeIDs from the routeIDs by getting all of the tripIDs that use the routes.
    if inclusiveFlag:
        matchingTrips = [gtfsTrip for gtfsTrip in gtfsTrips.values() if gtfsTrip.route.routeID not in routeIDSet]
    else:
        matchingTrips = [gtfsTrip for gtfsTrip in gtfsTrips.values() if gtfsTrip.route.routeID in routeIDSet]
    for gtfsTrip in matchingTrips:
        "@type gtfsTrip: gtfs.TripsEntry"
        shapeIDSet.add(gtfsTrip.shapeEntries[0].shapeID)
        
    # Report shapes:
    shapeStr = ""
    shapeIDReport = list(shapeIDSet)
    shapeIDReport.sort()
    ctr = 0
    for shapeID in shapeIDReport:
        shapeStr += str(shapeID)
        ctr += 1
        if ctr < len(shapeIDReport):
            shapeStr += ", "
    print("INFO: Route restrictions select Shape IDs %s." % shapeStr, file = sys.stderr)

    # Create a new GTFS list with the shapeIDs that had been selected:
    ret = {}
    "@type ret: dict<int, list<path_engine.PathEnd>>"
    shapeIDs = list(shapeIDSet)
    shapeIDs.sort()
    for shapeID in shapeIDs:
        if shapeID not in gtfsNodes:
            print("WARNING: Restriction shape ID %s does not exist in the GTFS set." % str(shapeID), file = sys.stderr)
        else:
            ret[shapeID] = gtfsNodes[shapeID]
    return ret
def fillHints(filename, shapePath, gtfsShapes, GPS, unusedShapeIDs):
    """
    fillHints retrieves hints that guide where paths should be drawn. 
    @type filePath: str
    @type shapePath: str
    @type gtfsShapes: gtfs.ShapesEntry
    @type GPS: GPS.GPS
    @return A map of stop_id to a hint_entry.
    @rtype dict<int, path_engine.ShapesEntry>
    """
    ret = {}
    "@type ret: dict<int, list<path_engine.ShapesEntry>>"
    
    if (filename is None) or (len(filename) == 0):
        return ret
    
    # Retrieve trips file information that will get us the routes-to-shape mapping that we want. 
    print("INFO: Read GTFS routesfile...", file = sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, gtfs.RoutesEntry>"
    
    print("INFO: Read GTFS tripsfile...", file = sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes, gtfsRoutes, unusedShapeIDs)
    "@type gtfsTrips: dict<int, gtfs.TripsEntry>"
        
    with open(filename, 'r') as inFile:
        # Sanity check:
        fileLine = inFile.readline()
        if not fileLine.startswith("route_id,hint_seq,lat,lon"):
            print("ERROR: The hints file '%s' doesn't have the expected header." % filename, file = sys.stderr)
            return None
        
        # Go through the lines of the file:
        for fileLine in inFile:
            if len(fileLine) > 0:
                lineElems = fileLine.split(',')
                
                routeID = int(lineElems[0])
                hintSeq = int(lineElems[1])
                gpsLat = float(lineElems[2])
                gpsLng = float(lineElems[3])
                
                # Resolve shapeIDs from the routeIDs by getting all of the tripIDs that use the route.
                matchingTrips = [gtfsTrip for gtfsTrip in gtfsTrips.values() if gtfsTrip.route.routeID == routeID]
                shapeIDSet = set()
                for gtfsTrip in matchingTrips:
                    "@type gtfsTrip: gtfs.TripsEntry"
                    shapeID = gtfsTrip.shapeEntries[0].shapeID
                    if shapeID not in shapeIDSet: # Only write in a hint once per shape.
                        newEntry = gtfs.ShapesEntry(shapeID, hintSeq, gpsLat, gpsLng, True)
                        (newEntry.pointX, newEntry.pointY) = GPS.gps2feet(gpsLat, gpsLng)
                
                        if shapeID not in ret:
                            ret[shapeID] = list()
                        ret[shapeID].append(newEntry)
                        shapeIDSet.add(shapeID)

    # Sort the hints so that they will be intercepted in the correct order:                    
    for shapeID in ret:
        ret[shapeID].sort(key = operator.attrgetter('shapeSeq'))
    
    # Return the hints file contents:
    return ret
def main(argv):
    global problemReport
    excludeUpstream = False
    
    # Initialize from command-line parameters:
    if len(argv) < 7:
        syntax(1)
    dbServer = argv[1]
    networkName = argv[2]
    userName = argv[3]
    password = argv[4]
    shapePath = argv[5]
    pathMatchFilename = argv[6]
    endTimeInt = 86400
    refTime = None
    widenBegin = False
    widenEnd = False
    excludeBegin = False
    excludeEnd = False
    
    restrictService = set()
    "@type restrictService: set<string>"

    if len(argv) > 6:
        i = 7
        while i < len(argv):
            if argv[i] == "-t" and i < len(argv) - 1:
                refTime = datetime.strptime(argv[i + 1], '%H:%M:%S')
                i += 1
            elif argv[i] == "-e" and i < len(argv) - 1:
                endTimeInt = int(argv[i + 1])
                i += 1
            elif argv[i] == "-c" and i < len(argv) - 1:
                restrictService.add(argv[i + 1])
                i += 1
            elif argv[i] == "-u":
                excludeUpstream = True
            elif argv[i] == "-w":
                widenBegin = True
                widenEnd = True
            elif argv[i] == "-wb":
                widenBegin = True
            elif argv[i] == "-we":
                widenEnd = True
            elif argv[i] == "-x":
                excludeBegin = True
                excludeEnd = True
            elif argv[i] == "-xb":
                excludeBegin = True
            elif argv[i] == "-xe":
                excludeEnd = True
            elif argv[i] == "-p":
                problemReport = True
            i += 1
    
    if refTime is None:
        print("ERROR: No reference time is specified. You must use the -t parameter.", file = sys.stderr)
        syntax(1)
    endTime = refTime + timedelta(seconds = endTimeInt)
    
    if widenBegin and excludeBegin:
        print("ERROR: Widening (-w or -wb) and exclusion (-x or -xb) cannot be used together.")
        syntax(1)    
    if widenEnd and excludeEnd:
        print("ERROR: Widening (-w or -we) and exclusion (-x or -xe) cannot be used together.")
        syntax(1)
    
    # Default parameters:
    stopSearchRadius = 800
    
    # Restore the stuff that was built with path_match:
    (vistaGraph, gtfsShapes, gtfsNodes, unusedShapeIDs) = restorePathMatch(dbServer, networkName, userName,
        password, shapePath, pathMatchFilename)
    
    # Read in the routes information:
    print("INFO: Read GTFS routesfile...", file = sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, RoutesEntry>"
    
    # Read in the stops information:
    print("INFO: Read GTFS stopsfile...", file = sys.stderr)
    gtfsStops = gtfs.fillStops(shapePath, vistaGraph.gps)
    "@type gtfsStops: dict<int, StopsEntry>"
    
    # Read in the trips information:
    print("INFO: Read GTFS tripsfile...", file = sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes, gtfsRoutes, unusedShapeIDs, restrictService)
    "@type gtfsTrips: dict<int, TripsEntry>"
    "@type unusedTripIDs: set<int>"
        
    # Read stop times information:
    print("INFO: Read GTFS stop times...", file = sys.stderr)
    gtfsStopTimes = gtfs.fillStopTimes(shapePath, gtfsTrips, gtfsStops, unusedTripIDs)
    "@type gtfsStopTimes: dict<TripsEntry, list<StopTimesEntry>>"
        
    # Output the routes_link file:
    print("INFO: Dumping public.bus_route_link.csv...", file = sys.stderr)
    with open("public.bus_route_link.csv", 'w') as outFile:
        (stopLinkMap, newStartTime, newEndTime) = dumpBusRouteLinks(gtfsTrips, gtfsStopTimes, gtfsNodes, vistaGraph,
            stopSearchRadius, excludeUpstream, userName, networkName, refTime, endTime, widenBegin, widenEnd,
            excludeBegin, excludeEnd, outFile)
        "@type stopLinkMap: dict<int, graph.PointOnLink>"
    
    # Filter only to bus stops and stop times that are used in the routes_link output:
    gtfsStopsFilterList = [gtfsStopID for gtfsStopID in gtfsStops if gtfsStopID not in stopLinkMap]
    for gtfsStopID in gtfsStopsFilterList:
        del gtfsStops[gtfsStopID]
    del gtfsStopsFilterList
    
    # Then, output the output the stop file:
    print("INFO: Dumping public.bus_stop.csv...", file = sys.stderr)
    with open("public.bus_stop.csv", 'w') as outFile:
        dumpBusStops(gtfsStops, stopLinkMap, userName, networkName, outFile)
        
    print("INFO: Dumping public.bus_frequency.csv...", file = sys.stderr)
    validTrips = {}
    "@type validTrips: dict<int, gtfs.TripsEntry>"
    with open("public.bus_frequency.csv", 'w') as outFile:
        _outHeader("public.bus_frequency", userName, networkName, outFile)
        print("\"route\",\"period\",\"frequency\",\"offsettime\",\"preemption\"", file = outFile)
        
        # Okay, here we iterate through stops until we get to the first defined one. That will
        # then affect the offsettime. (This is needed because of the idea that we want to start
        # a bus in the simulation wrt the topology that supports it, skipping those stops that
        # may fall outside the topology.)
        totalCycle = int((newEndTime - newStartTime).total_seconds()) 
        tripIDs = gtfsTrips.keys()
        tripIDs.sort()
        for tripID in tripIDs:
            stopsEntries = gtfsStopTimes[gtfsTrips[tripID]]
            for gtfsStopTime in stopsEntries:
                if gtfsStopTime.stop.stopID in gtfsStops:
                    # Here is a first valid entry! Use this offset value.
                    
                    # TODO: This could be inaccurate because the offset is that of the first
                    # valid stop time encountered in the underlying topology, not approximated
                    # to the first valid link encountered. While this isn't a big deal for an
                    # area with a high stop density, it could be a problem for limited-stop
                    # service where there happens to be a low density around where the bus
                    # first appears in the underlying topology.
                    stopTime = gtfsStopTime.arrivalTime
                    
                    # Adjust for cases where we need to add a day.
                    if stopTime < newStartTime: # Assume that we're working just within a day.
                        stopTime += timedelta(days = int((newStartTime - stopTime).total_seconds()) / 86400 + 1)
                    print("%d,1,%d,%d,0" % (tripID, totalCycle, int((stopTime - newStartTime).total_seconds())),
                        file = outFile)
                    validTrips[tripID] = gtfsTrips[tripID] # Record as valid.
                    break
                    
                # A byproduct of this scheme is that no bus_frequency entry will appear for
                # routes that don't have stops in the underlying topology.

    # Output the routes file:
    print("INFO: Dumping public.bus_route.csv...", file = sys.stderr)
    with open("public.bus_route.csv", 'w') as outFile:
        dumpBusRoutes(validTrips, userName, networkName, outFile)

    # Finally, define one period that spans the whole working time, which all of the individually
    # defined routes (again, one route per trip) will operate in.
    print("INFO: Dumping public.bus_period.csv...", file = sys.stderr)
    with open("public.bus_period.csv", 'w') as outFile:
        _outHeader("public.bus_period", userName, networkName, outFile)
        print("\"id\",\"starttime\",\"endtime\"", file = outFile)
        # The start time printed here is relative to the reference time.
        print("1,0,%d" % endTimeInt, file = outFile)
        
    if widenBegin or widenEnd:
        # Report the implicit adjustment in times because of warmup or cooldown:
        startTimeDiff = refTime - newStartTime
        endTimeDiff = newEndTime - endTime
        print("INFO: Widening requires start %d sec. earlier and duration %d sec. longer." % (startTimeDiff.total_seconds(),
            endTimeDiff.total_seconds() + startTimeDiff.total_seconds()), file = sys.stderr)
        totalTimeDiff = newEndTime - newStartTime
        print("INFO: New time reference is %s, duration %d sec." % (newStartTime.strftime("%H:%M:%S"), totalTimeDiff.total_seconds()),
            file = sys.stderr)

    print("INFO: Done.", file = sys.stderr)
def filterRoutes(gtfsNodes,
                 shapePath,
                 gtfsShapes,
                 routeRestrictFilename,
                 inclusiveFlag=False):
    """
    filterRoutes removes all GTFS nodes whose routes are not listed in the given file.
    @type gtfsNodes: dict<int, list<path_engine.PathEnd>>
    @type shapePath: str
    @type gtfsShapes: gtfs.ShapesEntry
    @type routeRestrictFilename: str
    @rtype dict<int, list<path_engine.PathEnd>>
    """
    # TODO: This type of filtering capability would probably best go in another centralized
    # location where multiple methods can all act upon the filter list and save processing time.

    if (routeRestrictFilename is None) or (len(routeRestrictFilename) == 0):
        return gtfsNodes

    print("INFO: Read GTFS routesfile...", file=sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, gtfs.RoutesEntry>"

    print("INFO: Read GTFS tripsfile...", file=sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes,
                                                gtfsRoutes)
    "@type gtfsTrips: dict<int, gtfs.TripsEntry>"

    routeIDSet = set()
    "@type routeIDSet: set<int>"
    with open(routeRestrictFilename, 'r') as inFile:
        # Go through the lines of the file:
        for fileLine in inFile:
            if len(fileLine) > 0:
                routeIDSet.add(int(fileLine))

    shapeIDSet = set()
    "@type shapeIDSet: set<int>"

    # Resolve shapeIDs from the routeIDs by getting all of the tripIDs that use the routes.
    if inclusiveFlag:
        matchingTrips = [
            gtfsTrip for gtfsTrip in gtfsTrips.values()
            if gtfsTrip.route.routeID not in routeIDSet
        ]
    else:
        matchingTrips = [
            gtfsTrip for gtfsTrip in gtfsTrips.values()
            if gtfsTrip.route.routeID in routeIDSet
        ]
    for gtfsTrip in matchingTrips:
        "@type gtfsTrip: gtfs.TripsEntry"
        shapeIDSet.add(gtfsTrip.shapeEntries[0].shapeID)

    # Report shapes:
    shapeStr = ""
    shapeIDReport = list(shapeIDSet)
    shapeIDReport.sort()
    ctr = 0
    for shapeID in shapeIDReport:
        shapeStr += str(shapeID)
        ctr += 1
        if ctr < len(shapeIDReport):
            shapeStr += ", "
    print("INFO: Route restrictions select Shape IDs %s." % shapeStr,
          file=sys.stderr)

    # Create a new GTFS list with the shapeIDs that had been selected:
    ret = {}
    "@type ret: dict<int, list<path_engine.PathEnd>>"
    shapeIDs = list(shapeIDSet)
    shapeIDs.sort()
    for shapeID in shapeIDs:
        if shapeID not in gtfsNodes:
            print(
                "WARNING: Restriction shape ID %s does not exist in the GTFS set."
                % str(shapeID),
                file=sys.stderr)
        else:
            ret[shapeID] = gtfsNodes[shapeID]
    return ret
def fillHints(filename, shapePath, gtfsShapes, GPS, unusedShapeIDs):
    """
    fillHints retrieves hints that guide where paths should be drawn. 
    @type filePath: str
    @type shapePath: str
    @type gtfsShapes: gtfs.ShapesEntry
    @type GPS: GPS.GPS
    @return A map of stop_id to a hint_entry.
    @rtype dict<int, path_engine.ShapesEntry>
    """
    ret = {}
    "@type ret: dict<int, list<path_engine.ShapesEntry>>"

    if (filename is None) or (len(filename) == 0):
        return ret

    # Retrieve trips file information that will get us the routes-to-shape mapping that we want.
    print("INFO: Read GTFS routesfile...", file=sys.stderr)
    gtfsRoutes = gtfs.fillRoutes(shapePath)
    "@type gtfsRoutes: dict<int, gtfs.RoutesEntry>"

    print("INFO: Read GTFS tripsfile...", file=sys.stderr)
    (gtfsTrips, unusedTripIDs) = gtfs.fillTrips(shapePath, gtfsShapes,
                                                gtfsRoutes, unusedShapeIDs)
    "@type gtfsTrips: dict<int, gtfs.TripsEntry>"

    with open(filename, 'r') as inFile:
        # Sanity check:
        fileLine = inFile.readline()
        if not fileLine.startswith("route_id,hint_seq,lat,lon"):
            print(
                "ERROR: The hints file '%s' doesn't have the expected header."
                % filename,
                file=sys.stderr)
            return None

        # Go through the lines of the file:
        for fileLine in inFile:
            if len(fileLine) > 0:
                lineElems = fileLine.split(',')

                routeID = int(lineElems[0])
                hintSeq = int(lineElems[1])
                gpsLat = float(lineElems[2])
                gpsLng = float(lineElems[3])

                # Resolve shapeIDs from the routeIDs by getting all of the tripIDs that use the route.
                matchingTrips = [
                    gtfsTrip for gtfsTrip in gtfsTrips.values()
                    if gtfsTrip.route.routeID == routeID
                ]
                shapeIDSet = set()
                for gtfsTrip in matchingTrips:
                    "@type gtfsTrip: gtfs.TripsEntry"
                    shapeID = gtfsTrip.shapeEntries[0].shapeID
                    if shapeID not in shapeIDSet:  # Only write in a hint once per shape.
                        newEntry = gtfs.ShapesEntry(shapeID, hintSeq, gpsLat,
                                                    gpsLng, True)
                        (newEntry.pointX,
                         newEntry.pointY) = GPS.gps2feet(gpsLat, gpsLng)

                        if shapeID not in ret:
                            ret[shapeID] = list()
                        ret[shapeID].append(newEntry)
                        shapeIDSet.add(shapeID)

    # Sort the hints so that they will be intercepted in the correct order:
    for shapeID in ret:
        ret[shapeID].sort(key=operator.attrgetter('shapeSeq'))

    # Return the hints file contents:
    return ret