コード例 #1
0
    def readAggregateDbfs(self, asgnFileName, aggregateFileName=None):
        """
        This is essentially the reverse of writeDbfs() below.
        """
        self.initializeFields()  # this may be unnecessary

        self.trnAsgnTable = dbfTableReader(asgnFileName)
        self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")

        # a little bit of cleanup
        headerTuples = []
        for headerFieldType in self.trnAsgnTable.header:
            headerTuples.append(headerFieldType.toTuple())

        # rstrip spaces off the end of the text fields
        for row in self.trnAsgnTable:
            for headerTuple in headerTuples:
                if headerTuple[1] == 'C':
                    row[headerTuple[0]] = string.rstrip(row[headerTuple[0]])

        # this is the index!
        self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")

        # the link-level aggregate table
        if not aggregateFileName:
            self.aggregateAll = False
            self.aggregateTable = False
            return

        self.aggregateAll = True
        self.aggregateTable = dbfTableReader(aggregateFileName)

        # cleanup again
        headerTuples = []
        for headerFieldType in self.aggregateTable.header:
            headerTuples.append(headerFieldType.toTuple())

        # rstrip spaces off the end of the text fields
        for row in self.aggregateTable:
            for headerTuple in headerTuples:
                if headerTuple[1] == 'C':
                    row[headerTuple[0]] = string.rstrip(row[headerTuple[0]])
コード例 #2
0
    def readAggregateDbfs(self, asgnFileName, aggregateFileName=None):
        """
        This is essentially the reverse of writeDbfs() below.
        """
        self.initializeFields()  # this may be unnecessary

        self.trnAsgnTable = dbfTableReader(asgnFileName)
        self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")

        # a little bit of cleanup
        headerTuples = []
        for headerFieldType in self.trnAsgnTable.header:
            headerTuples.append(headerFieldType.toTuple())

        # rstrip spaces off the end of the text fields
        for row in self.trnAsgnTable:
            for headerTuple in headerTuples:
                if headerTuple[1] == "C":
                    row[headerTuple[0]] = string.rstrip(row[headerTuple[0]])

        # this is the index!
        self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")

        # the link-level aggregate table
        if not aggregateFileName:
            self.aggregateAll = False
            self.aggregateTable = False
            return

        self.aggregateAll = True
        self.aggregateTable = dbfTableReader(aggregateFileName)

        # cleanup again
        headerTuples = []
        for headerFieldType in self.aggregateTable.header:
            headerTuples.append(headerFieldType.toTuple())

        # rstrip spaces off the end of the text fields
        for row in self.aggregateTable:
            for headerTuple in headerTuples:
                if headerTuple[1] == "C":
                    row[headerTuple[0]] = string.rstrip(row[headerTuple[0]])
コード例 #3
0
    def readTransitAssignmentCsvs(self):
        """
        Read the transit assignment dbfs, the direct output of Cube's transit assignment.
        """
        self.trnAsgnTable = False
        self.aggregateTable = False
        warnline = {}
        ABNameSeq_List = []  # (A,B,NAME,SEQ) from the dbf/csvs

        # open the input assignment files
        for mode in self.MODES:
            if mode == "WMWVIS":
                filename = os.path.join(self.assigndir,
                                        "VISWMW" + self.timeperiod + ".csv")
            elif mode[1] == "T":
                filename = os.path.join(self.assigndir,
                                        "NS" + mode + self.timeperiod + ".csv")
            else:
                filename = os.path.join(self.assigndir,
                                        "SF" + mode + self.timeperiod + ".csv")

            # Read the DBF file into datatable
            WranglerLogger.info("Reading " + filename)

            # Create our table data structure once
            if mode == self.MODES[0]:
                # figure out how many records
                numrecs = 0
                totalrows = 0

                filereader = csv.reader(open(filename, 'rb'),
                                        delimiter=',',
                                        quoting=csv.QUOTE_NONE)
                for row in filereader:

                    # header row?
                    if row[0] == "A":
                        self.initializeFields(row)
                        continue
                    elif totalrows == 0 and not self.csvColnames:
                        self.initializeFields()

                    totalrows += 1
                    if self.profileNode and \
                       (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and
                        int(row[self.colnameToCsvIndex["B"]]) != self.profileNode):
                        continue

                    if int(row[self.colnameToCsvIndex["MODE"]]
                           ) in self.ignoreModes:
                        continue

                    linename = row[self.colnameToCsvIndex["NAME"]].strip()

                    # exclude this system?
                    (system,
                     vehicletype) = self.capacity.getSystemAndVehicleType(
                         linename, self.timeperiod)

                    if len(self.system) > 0 and system not in self.system:
                        continue

                    numrecs += 1
                WranglerLogger.info("Keeping %d records out of %d" %
                                    (numrecs, totalrows))

                self.trnAsgnTable = DataTable(
                    numRecords=numrecs,
                    fieldNames=self.trnAsgnFields.keys(),
                    numpyFieldTypes=self.trnAsgnFields.values())
                ABNameSeqSet = set()

            # Go through the records
            newrownum = 0  # row number in the trnAsgnTable,ABNameSeq_List -- rows we're keeping
            oldrownum = 0  # row number in the csv,dbf -- all input rows

            filereader = csv.reader(open(filename, 'rb'),
                                    delimiter=',',
                                    quoting=csv.QUOTE_NONE)

            # for the first csv only, also read the dbf for the freq and seq fields
            if mode == self.MODES[0]:
                indbf = dbfTableReader(
                    os.path.join(self.assigndir,
                                 "SFWBW" + self.timeperiod + ".dbf"))
            else:
                indbf = None

            for row in filereader:
                # header row?
                if row[0] == "A": continue

                if self.profileNode:
                    if (int(row[self.colnameToCsvIndex["A"]]) !=
                            self.profileNode
                            and int(row[self.colnameToCsvIndex["B"]]) !=
                            self.profileNode):
                        continue
                    elif int(row[self.colnameToCsvIndex["AB_VOL"]]) > 0:
                        WranglerLogger.info(
                            "Link %s %s for mode %s has AB_VOL %s" %
                            (row[self.colnameToCsvIndex["A"]],
                             row[self.colnameToCsvIndex["B"]], mode,
                             row[self.colnameToCsvIndex["AB_VOL"]]))

                if int(row[
                        self.colnameToCsvIndex["MODE"]]) in self.ignoreModes:
                    oldrownum += 1
                    continue

                linename = row[self.colnameToCsvIndex["NAME"]].strip()

                # exclude this system?
                (system, vehicletype) = self.capacity.getSystemAndVehicleType(
                    linename, self.timeperiod)
                if len(self.system) > 0 and system not in self.system: continue

                # Initial table fill: Special stuff for the first time through
                if mode == self.MODES[0]:

                    # ------------ these fields just get used directly
                    for field in self.trnAsgnCopyFields:

                        try:

                            # integer fields
                            if self.trnAsgnFields[field][0] in ['u', 'b']:
                                if row[self.colnameToCsvIndex[field]] == "":
                                    self.trnAsgnTable[newrownum][field] = 0
                                elif field in ['TIME', 'DIST']:
                                    # backwards compatibility - dbfs were 100ths of a mile/min
                                    self.trnAsgnTable[newrownum][
                                        field] = float(
                                            row[self.colnameToCsvIndex[field]]
                                        ) * 100.0
                                else:
                                    self.trnAsgnTable[newrownum][field] = int(
                                        row[self.colnameToCsvIndex[field]])
                            # float fields
                            elif self.trnAsgnFields[field][0] == 'f':
                                if row[self.colnameToCsvIndex[field]] == "":
                                    self.trnAsgnTable[newrownum][field] = 0.0
                                else:
                                    self.trnAsgnTable[newrownum][
                                        field] = float(
                                            row[self.colnameToCsvIndex[field]])
                            # text fields
                            else:
                                self.trnAsgnTable[newrownum][field] = row[
                                    self.colnameToCsvIndex[field]]

                        except:
                            WranglerLogger.fatal(
                                "Error intepreting field %s: [%s]" %
                                (field, str(self.colnameToCsvIndex[field])))
                            WranglerLogger.fatal("row=%s" % str(row))
                            WranglerLogger.fatal(sys.exc_info()[0])
                            WranglerLogger.fatal(sys.exc_info()[1])
                            sys.exit(2)
                    # ------------ these fields come from the dbf because they're missing in the csv (sigh)
                    dbfRow = indbf.__getitem__(oldrownum)
                    if int(row[self.colnameToCsvIndex["A"]]) < 100000:
                        if dbfRow["A"] != int(
                                row[self.colnameToCsvIndex["A"]]):
                            raise NetworkException(
                                "Assertion error for A on row %d: %s != %s" %
                                (oldrownum, str(dbfRow["A"]),
                                 str(row[self.colnameToCsvIndex["A"]])))
                    if int(row[self.colnameToCsvIndex["B"]]) < 100000:
                        if dbfRow["B"] != int(
                                row[self.colnameToCsvIndex["B"]]):
                            raise NetworkException(
                                "Assertion error for B on row %d: %s != %s" %
                                (oldrownum, str(dbfRow["B"]),
                                 str(row[self.colnameToCsvIndex["B"]])))
                    self.trnAsgnTable[newrownum]["FREQ"] = dbfRow["FREQ"]
                    self.trnAsgnTable[newrownum]["SEQ"] = dbfRow["SEQ"]

                    trySeq = dbfRow["SEQ"]
                    # ------------ special one-time computed fields

                    # ABNameSeq is more complicated because we want it to be unique
                    AB = row[self.colnameToCsvIndex["A"]] + " " + row[
                        self.colnameToCsvIndex["B"]]
                    self.trnAsgnTable[newrownum]["AB"] = AB

                    ABNameSeq = AB + " " + linename
                    if trySeq > 0:
                        tryABNameSeq = ABNameSeq + " " + str(trySeq)

                        # This line seems to be a problem... A/B/NAME/SEQ are not unique
                        if tryABNameSeq in ABNameSeqSet:
                            WranglerLogger.warn("Non-Unique A/B/Name/Seq: " +
                                                tryABNameSeq + "; faking SEQ!")
                        # Find one that works
                        while tryABNameSeq in ABNameSeqSet:
                            trySeq += 1
                            tryABNameSeq = ABNameSeq + " " + str(trySeq)
                        ABNameSeq = tryABNameSeq
                    self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq
                    ABNameSeqSet.add(ABNameSeq)

                    ABNameSeq_List.append(
                        (int(row[self.colnameToCsvIndex["A"]]),
                         int(row[self.colnameToCsvIndex["B"]]),
                         row[self.colnameToCsvIndex["NAME"]], trySeq))

                    # ------------ straight lookup FULLNAME, VEHTYPE, VEHCAP; easy calc for PERIODCAP
                    self.trnAsgnTable[newrownum]["SYSTEM"] = system
                    self.trnAsgnTable[newrownum]["VEHTYPE"] = vehicletype

                    self.trnAsgnTable[newrownum][
                        "FULLNAME"] = self.capacity.getFullname(
                            linename, self.timeperiod)

                    try:
                        (vtype,
                         vehcap) = self.capacity.getVehicleTypeAndCapacity(
                             linename, self.timeperiod)

                        self.trnAsgnTable[newrownum]["VEHCAP"] = vehcap
                        self.trnAsgnTable[newrownum][
                            "PERIODCAP"] = TransitLine.HOURS_PER_TIMEPERIOD[
                                self.
                                timeperiod] * 60.0 * vehcap / self.trnAsgnTable[
                                    newrownum]["FREQ"]
                    except:
                        self.trnAsgnTable[newrownum]["VEHCAP"] = 0
                        self.trnAsgnTable[newrownum]["PERIODCAP"] = 0

                    # if we still don't have a system, warn
                    if self.trnAsgnTable[newrownum][
                            "SYSTEM"] == "" and not warnline.has_key(linename):
                        WranglerLogger.warning("No default system: " +
                                               linename)
                        warnline[linename] = 1

                    #---------add in any grouping that may want to use
                    if self.lineToGroup.has_key(linename):
                        self.trnAsgnTable[newrownum][
                            "GROUP"] = self.lineToGroup[linename]
                    else:
                        self.trnAsgnTable[newrownum]["GROUP"] = ""

                    # initialize additive fields
                    for field in self.trnAsgnAdditiveFields:
                        if row[self.colnameToCsvIndex[field]] == "":
                            self.trnAsgnTable[newrownum][field] = 0.0
                        else:
                            self.trnAsgnTable[newrownum][field] = float(
                                row[self.colnameToCsvIndex[field]])

                # end initial table fill

                # Add in the subsequent assignment files
                else:

                    # print oldrownum, newrownum, ABNameSeq_List[newrownum]
                    # print row[self.colnameToCsvIndex["NAME"]], ABNameSeq_List[oldrownum][2]

                    assert (int(row[self.colnameToCsvIndex["A"]]) ==
                            ABNameSeq_List[newrownum][0])
                    assert (int(row[self.colnameToCsvIndex["B"]]) ==
                            ABNameSeq_List[newrownum][1])
                    # these don't nec match, can be *32 in ferry skim rather than the bart vehicle name, for example
                    # assert(  row[self.colnameToCsvIndex["NAME"]] == ABNameSeq_List[newrownum][2])

                    ABNameSeq = row[self.colnameToCsvIndex["A"]] + " " + \
                                row[self.colnameToCsvIndex["B"]] + " " + \
                                row[self.colnameToCsvIndex["NAME"]].rstrip()
                    if ABNameSeq_List[newrownum][3] > 0:
                        ABNameSeq += " " + str(ABNameSeq_List[newrownum][3])
                    for field in self.trnAsgnAdditiveFields:
                        if row[self.colnameToCsvIndex[field]] != "":
                            self.trnAsgnTable[ABNameSeq][field] += float(
                                row[self.colnameToCsvIndex[field]])

                newrownum += 1
                oldrownum += 1

            # we're done with this; free it up
            del filereader
            if indbf:
                del indbf

            # Table is created and filled -- set the index
            if mode == self.MODES[0]:
                try:
                    self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")
                except:
                    # failure - try to figure out why
                    ABNameSeqList = []
                    for row in self.trnAsgnTable:
                        ABNameSeqList.append(row["ABNAMESEQ"])
                    ABNameSeqList.sort()
                    for idx in range(len(ABNameSeqList) - 1):
                        if ABNameSeqList[idx] == ABNameSeqList[idx + 1]:
                            WranglerLogger.warn(
                                "Duplicate ABNAMESEQ at idx %d : [%s]" %
                                (idx, ABNameSeqList[idx]))
                    exit(1)

        # ok the table is all filled in -- fill in the LOAD
        for row in self.trnAsgnTable:
            if row["VEHCAP"] == 0: continue
            tpfactor = self.TIMEPERIOD_FACTOR[self.timeperiod]

            # mode-specific peaking factor will over-ride
            if row["MODE"] in self.TIMEPERIOD_FACTOR:
                tpfactor = self.TIMEPERIOD_FACTOR[row["MODE"]][self.timeperiod]

            row["LOAD"] = row["AB_VOL"] * tpfactor * row["FREQ"] / (
                60.0 * row["VEHCAP"])

        # build the aggregate table for key="A B"
        if self.aggregateAll:
            self.buildAggregateTable()
コード例 #4
0
    def readTransitAssignmentCsvs(self):
        """
        Read the transit assignment dbfs, the direct output of Cube's transit assignment.
        """
        self.trnAsgnTable   = False
        self.aggregateTable = False
        warnline = {}
                
        # open the input assignment files
        for mode in self.MODES:
            if mode == "VWMW":
                filename = os.path.join(self.assigndir, "VISWMW" + self.timeperiod + ".csv")
            elif mode[1]=="T":
                filename = os.path.join(self.assigndir, "NS" + mode + self.timeperiod + ".csv")
            else:
                filename = os.path.join(self.assigndir, "SF" + mode + self.timeperiod + ".csv")
                
            # Read the DBF file into datatable
            WranglerLogger.info("Reading "+filename)

            # Create our table data structure once
            if mode == self.MODES[0]:
                # figure out how many records
                numrecs = 0
                totalrows = 0
                                
                filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE)
                for row in filereader:
                    
                    # header row?
                    if row[0]=="A": 
                        self.initializeFields(row)
                        continue
                    elif totalrows==0 and not self.csvColnames:
                        self.initializeFields()

                    
                    totalrows += 1
                    if self.profileNode and \
                       (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and
                        int(row[self.colnameToCsvIndex["B"]]) != self.profileNode): continue
                    
                    if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: continue
                    
                    linename = row[self.colnameToCsvIndex["NAME"]].strip()

                    # exclude this system?
                    (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod)

                    if len(self.system)>0 and system not in self.system: continue

                    numrecs += 1
                WranglerLogger.info("Keeping %d records out of %d" % (numrecs, totalrows))
                
                self.trnAsgnTable = DataTable(numRecords=numrecs,
                                              fieldNames=self.trnAsgnFields.keys(),
                                              numpyFieldTypes=self.trnAsgnFields.values())
                ABNameSeqSet = set()
                ABSet = set()
            
            
            # Go through the records
            newrownum = 0
            oldrownum = 0
            
            filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE)
            indbf = dbfTableReader(os.path.join(self.assigndir, "SFWBW" + self.timeperiod + ".dbf"))
            
            for row in filereader:
                # header row?
                if row[0]=="A": continue
                                       
                oldrownum += 1
                
                if self.profileNode:
                    if (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and
                        int(row[self.colnameToCsvIndex["B"]]) != self.profileNode): continue
                    elif int(row[self.colnameToCsvIndex["AB_VOL"]]) > 0:
                        WranglerLogger.info("Link %s %s for mode %s has AB_VOL %s" % 
                                     (row[self.colnameToCsvIndex["A"]], 
                                      row[self.colnameToCsvIndex["B"]], mode, 
                                      row[self.colnameToCsvIndex["AB_VOL"]]))
            
                if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: continue
                     
                linename = row[self.colnameToCsvIndex["NAME"]].strip()
                
                # exclude this system?
                (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod)
                if len(self.system)>0 and system not in self.system: continue
            
                # Initial table fill: Special stuff for the first time through
                if mode == self.MODES[0]:

                    # ------------ these fields just get used directly
                    for field in self.trnAsgnCopyFields:
                        
                        try:
                            
                            # integer fields
                            if self.trnAsgnFields[field][0] in ['u','b']:
                                if row[self.colnameToCsvIndex[field]]=="":
                                    self.trnAsgnTable[newrownum][field] = 0
                                elif field in ['TIME','DIST']:
                                    # backwards compatibility - dbfs were 100ths of a mile/min
                                    self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]])*100.0
                                else:
                                    self.trnAsgnTable[newrownum][field] = int(row[self.colnameToCsvIndex[field]])                               
                            # float fields
                            elif self.trnAsgnFields[field][0] == 'f':
                                if row[self.colnameToCsvIndex[field]]=="":
                                    self.trnAsgnTable[newrownum][field] = 0.0
                                else:
                                    self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]])
                            # text fields
                            else:
                                self.trnAsgnTable[newrownum][field] = row[self.colnameToCsvIndex[field]]
                        
                        except:
                            WranglerLogger.fatal("Error intepreting field %s: [%s]" % (field, str(self.colnameToCsvIndex[field])))
                            WranglerLogger.fatal("row=%s" % str(row))
                            WranglerLogger.fatal(sys.exc_info()[0])
                            WranglerLogger.fatal(sys.exc_info()[1])                        
                            sys.exit(2)  
                    # ------------ these fields come from the dbf because they're missing in the csv (sigh)
                    dbfRow = indbf.__getitem__(oldrownum-1)
                    if int(row[self.colnameToCsvIndex["A"]])<100000:
                        if dbfRow["A"]!=int(row[self.colnameToCsvIndex["A"]]):
                            raise NetworkException("Assertion error for A on row %d: %s != %s" % (oldrownum-1, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]])))
                    if int(row[self.colnameToCsvIndex["B"]])<100000:
                        if dbfRow["B"]!=int(row[self.colnameToCsvIndex["B"]]):
                            raise NetworkException("Assertion error for B on row %d: %s != %s" % (oldrownum-1, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]])))
                    self.trnAsgnTable[newrownum]["FREQ"] = dbfRow["FREQ"]
                    self.trnAsgnTable[newrownum]["SEQ"]  = dbfRow["SEQ"]

                    trySeq = dbfRow["SEQ"]
                    # ------------ special one-time computed fields
                    
                    # ABNameSeq is more complicated because we want it to be unique 
                    AB = row[self.colnameToCsvIndex["A"]] + " " + row[self.colnameToCsvIndex["B"]]
                    self.trnAsgnTable[newrownum]["AB"] = AB
                    
                    ABNameSeq = AB + " " + linename
                    if trySeq>0:
                        tryABNameSeq = ABNameSeq + " " + str(trySeq)
                    
                        # This line seems to be a problem... A/B/NAME/SEQ are not unique
                        if tryABNameSeq in ABNameSeqSet:
                            WranglerLogger.warn("Non-Unique A/B/Name/Seq: " + tryABNameSeq + "; faking SEQ!")
                        # Find one that works
                        while tryABNameSeq in ABNameSeqSet:
                            trySeq += 1
                            tryABNameSeq = ABNameSeq + " " + str(trySeq)
                        ABNameSeq = tryABNameSeq
                    self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq
                    ABNameSeqSet.add(ABNameSeq)
                    ABSet.add(AB)
                    
                    # ------------ straight lookup FULLNAME, VEHTYPE, VEHCAP; easy calc for PERIODCAP
                    self.trnAsgnTable[newrownum]["SYSTEM"]  = system
                    self.trnAsgnTable[newrownum]["VEHTYPE"] = vehicletype

                    self.trnAsgnTable[newrownum]["FULLNAME"] = self.capacity.getFullname(linename, self.timeperiod)
                        
                    try:
                       (vtype, vehcap) = self.capacity.getVehicleTypeAndCapacity(linename, self.timeperiod)

                       self.trnAsgnTable[newrownum]["VEHCAP"] = vehcap
                       self.trnAsgnTable[newrownum]["PERIODCAP"] = TransitLine.HOURS_PER_TIMEPERIOD[self.timeperiod] * 60.0 * vehcap/self.trnAsgnTable[newrownum]["FREQ"]
                    except:
                       self.trnAsgnTable[newrownum]["VEHCAP"] = 0
                       self.trnAsgnTable[newrownum]["PERIODCAP"] = 0

                    # if we still don't have a system, warn
                    if self.trnAsgnTable[newrownum]["SYSTEM"] == "" and not warnline.has_key(linename):
                        WranglerLogger.warning("No default system: " + linename)
                        warnline[linename] =1
                    
                    #---------add in any grouping that may want to use
                    if self.lineToGroup.has_key(linename):
                        self.trnAsgnTable[newrownum]["GROUP"] = self.lineToGroup[linename]
                    else:
                        self.trnAsgnTable[newrownum]["GROUP"] = ""
                    
                    # initialize additive fields
                    for field in self.trnAsgnAdditiveFields:
                        if row[self.colnameToCsvIndex[field]]=="":
                            self.trnAsgnTable[newrownum][field] = 0.0
                        else:
                            self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]])
                        
                # end initial table fill
                
                # Add in the subsequent assignment files
                else:
                    dbfRow = indbf.__getitem__(oldrownum-1)
                    
                    while (((int(row[self.colnameToCsvIndex["A"]])<100000) and
                            (dbfRow["A"]!=int(row[self.colnameToCsvIndex["A"]]))) or
                           ((int(row[self.colnameToCsvIndex["B"]])<100000) and
                            (dbfRow["B"]!=int(row[self.colnameToCsvIndex["B"]])))):
                        oldrownum += 1
                        dbfRow = indbf.__getitem__(oldrownum-1)
                    
                    if int(row[self.colnameToCsvIndex["A"]])<100000:
                        if dbfRow["A"]!=int(row[self.colnameToCsvIndex["A"]]):
                            raise NetworkException("Assertion error for A on row %d: %s != %s" % (oldrownum-1, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]])))
                    if int(row[self.colnameToCsvIndex["B"]])<100000:
                        if dbfRow["B"]!=int(row[self.colnameToCsvIndex["B"]]):
                            raise NetworkException("Assertion error for B on row %d: %s != %s" % (oldrownum-1, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]])))
                    ABNameSeq = row[self.colnameToCsvIndex["A"]] + " " + \
                                row[self.colnameToCsvIndex["B"]] + " " + \
                                row[self.colnameToCsvIndex["NAME"]].rstrip()
                    if dbfRow["SEQ"]>0:
                        ABNameSeq += " " + str(dbfRow["SEQ"])                    
                    for field in self.trnAsgnAdditiveFields:
                        if row[self.colnameToCsvIndex[field]] !="":
                            self.trnAsgnTable[ABNameSeq][field] += float(row[self.colnameToCsvIndex[field]])
                        
                newrownum += 1   

            # we're done with this; free it up
            del filereader
            del indbf
            
            # Table is created and filled -- set the index
            if mode == self.MODES[0]: 
                try:
                    self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")
                except:
                    # failure - try to figure out why
                    ABNameSeqList = []
                    for row in self.trnAsgnTable:
                        ABNameSeqList.append(row["ABNAMESEQ"])
                    ABNameSeqList.sort()
                    for idx in range(len(ABNameSeqList)-1):
                        if ABNameSeqList[idx]==ABNameSeqList[idx+1]:
                            WranglerLogger.warn("Duplicate ABNAMESEQ at idx %d : [%s]" % (idx,ABNameSeqList[idx]))
                    exit(1)

        # ok the table is all filled in -- fill in the LOAD
        for row in self.trnAsgnTable:
            if row["VEHCAP"] == 0: continue
            row["LOAD"] = row["AB_VOL"] * self.TIMEPERIOD_FACTOR[self.timeperiod] * row["FREQ"] / (60.0 * row["VEHCAP"])

        # build the aggregate table for key="A B"
        if self.aggregateAll:

            self.aggregateTable = DataTable(numRecords=len(ABSet),
                                            fieldNames=self.aggregateFields.keys(),
                                            numpyFieldTypes=self.aggregateFields.values())
            ABtoRowIndex = {}
            rowsUsed = 0
            for row in self.trnAsgnTable:
                if row["AB"] not in ABtoRowIndex:
                    rowIndex = rowsUsed
                    self.aggregateTable[rowIndex]["AB"]     = row["AB"]
                    self.aggregateTable[rowIndex]["A"]      = row["A"]
                    self.aggregateTable[rowIndex]["B"]      = row["B"]
                    self.aggregateTable[rowIndex]["DIST"]   = row["DIST"]
                    self.aggregateTable[rowIndex]["FREQ"]       = 0.0
                    self.aggregateTable[rowIndex]["PERIODCAP"]  = 0.0
                    self.aggregateTable[rowIndex]["LOAD"]       = 0.0
                    self.aggregateTable[rowIndex]["MAXLOAD"]    = 0.0
                    for field in self.trnAsgnAdditiveFields:    # sum
                        self.aggregateTable[rowIndex][field] = 0.0
                    ABtoRowIndex[row["AB"]] = rowsUsed
                    rowsUsed += 1
                else:
                    rowIndex = ABtoRowIndex[row["AB"]]
                
                for field in self.trnAsgnAdditiveFields:    # sum
                    self.aggregateTable[rowIndex][field] += row[field]
                self.aggregateTable[rowIndex]["AB_VOL"] += row["AB_VOL"]
                self.aggregateTable[rowIndex]["BA_VOL"] += row["BA_VOL"]
                self.aggregateTable[rowIndex]["PERIODCAP"] += row["PERIODCAP"]
                self.aggregateTable[rowIndex]["MAXLOAD"] = max(row["LOAD"], self.aggregateTable[rowIndex]["MAXLOAD"])
                self.aggregateTable[rowIndex]["FREQ"] += 1/row["FREQ"]  # combining -- will take reciprocal later
            
            self.aggregateTable.setIndex(fieldName="AB")

            count=0
            for row in self.aggregateTable:
                count += 1
                if row["FREQ"]>0: 
                    row["FREQ"] = 1/row["FREQ"]
                if row["PERIODCAP"]>0:
                    row["LOAD"] = float(row["AB_VOL"]) / row["PERIODCAP"]
                    # print row["LOAD"]
            WranglerLogger.debug("count "+str(count)+" lines in aggregate table")
コード例 #5
0
    def readTransitAssignmentCsvs(self):
        """
        Read the transit assignment dbfs, the direct output of Cube's transit assignment.
        """
        self.trnAsgnTable = False
        self.aggregateTable = False
        warnline = {}
        ABNameSeq_List = []  # (A,B,NAME,SEQ) from the dbf/csvs

        # open the input assignment files
        for mode in self.MODES:
            if mode == "WMWVIS":
                filename = os.path.join(self.assigndir, "VISWMW" + self.timeperiod + ".csv")
            elif mode[1] == "T":
                filename = os.path.join(self.assigndir, "NS" + mode + self.timeperiod + ".csv")
            else:
                filename = os.path.join(self.assigndir, "SF" + mode + self.timeperiod + ".csv")

            # Read the DBF file into datatable
            WranglerLogger.info("Reading " + filename)

            # Create our table data structure once
            if mode == self.MODES[0]:
                # figure out how many records
                numrecs = 0
                totalrows = 0

                filereader = csv.reader(open(filename, "rb"), delimiter=",", quoting=csv.QUOTE_NONE)
                for row in filereader:

                    # header row?
                    if row[0] == "A":
                        self.initializeFields(row)
                        continue
                    elif totalrows == 0 and not self.csvColnames:
                        self.initializeFields()

                    totalrows += 1
                    if self.profileNode and (
                        int(row[self.colnameToCsvIndex["A"]]) != self.profileNode
                        and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode
                    ):
                        continue

                    if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes:
                        continue

                    linename = row[self.colnameToCsvIndex["NAME"]].strip()

                    # exclude this system?
                    (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod)

                    if len(self.system) > 0 and system not in self.system:
                        continue

                    numrecs += 1
                WranglerLogger.info("Keeping %d records out of %d" % (numrecs, totalrows))

                self.trnAsgnTable = DataTable(
                    numRecords=numrecs,
                    fieldNames=self.trnAsgnFields.keys(),
                    numpyFieldTypes=self.trnAsgnFields.values(),
                )
                ABNameSeqSet = set()

            # Go through the records
            newrownum = 0  # row number in the trnAsgnTable,ABNameSeq_List -- rows we're keeping
            oldrownum = 0  # row number in the csv,dbf -- all input rows

            filereader = csv.reader(open(filename, "rb"), delimiter=",", quoting=csv.QUOTE_NONE)

            # for the first csv only, also read the dbf for the freq and seq fields
            if mode == self.MODES[0]:
                indbf = dbfTableReader(os.path.join(self.assigndir, "SFWBW" + self.timeperiod + ".dbf"))
            else:
                indbf = None

            for row in filereader:
                # header row?
                if row[0] == "A":
                    continue

                if self.profileNode:
                    if (
                        int(row[self.colnameToCsvIndex["A"]]) != self.profileNode
                        and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode
                    ):
                        continue
                    elif int(row[self.colnameToCsvIndex["AB_VOL"]]) > 0:
                        WranglerLogger.info(
                            "Link %s %s for mode %s has AB_VOL %s"
                            % (
                                row[self.colnameToCsvIndex["A"]],
                                row[self.colnameToCsvIndex["B"]],
                                mode,
                                row[self.colnameToCsvIndex["AB_VOL"]],
                            )
                        )

                if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes:
                    oldrownum += 1
                    continue

                linename = row[self.colnameToCsvIndex["NAME"]].strip()

                # exclude this system?
                (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod)
                if len(self.system) > 0 and system not in self.system:
                    continue

                # Initial table fill: Special stuff for the first time through
                if mode == self.MODES[0]:

                    # ------------ these fields just get used directly
                    for field in self.trnAsgnCopyFields:

                        try:

                            # integer fields
                            if self.trnAsgnFields[field][0] in ["u", "b"]:
                                if row[self.colnameToCsvIndex[field]] == "":
                                    self.trnAsgnTable[newrownum][field] = 0
                                elif field in ["TIME", "DIST"]:
                                    # backwards compatibility - dbfs were 100ths of a mile/min
                                    self.trnAsgnTable[newrownum][field] = (
                                        float(row[self.colnameToCsvIndex[field]]) * 100.0
                                    )
                                else:
                                    self.trnAsgnTable[newrownum][field] = int(row[self.colnameToCsvIndex[field]])
                            # float fields
                            elif self.trnAsgnFields[field][0] == "f":
                                if row[self.colnameToCsvIndex[field]] == "":
                                    self.trnAsgnTable[newrownum][field] = 0.0
                                else:
                                    self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]])
                            # text fields
                            else:
                                self.trnAsgnTable[newrownum][field] = row[self.colnameToCsvIndex[field]]

                        except:
                            WranglerLogger.fatal(
                                "Error intepreting field %s: [%s]" % (field, str(self.colnameToCsvIndex[field]))
                            )
                            WranglerLogger.fatal("row=%s" % str(row))
                            WranglerLogger.fatal(sys.exc_info()[0])
                            WranglerLogger.fatal(sys.exc_info()[1])
                            sys.exit(2)
                    # ------------ these fields come from the dbf because they're missing in the csv (sigh)
                    dbfRow = indbf.__getitem__(oldrownum)
                    if int(row[self.colnameToCsvIndex["A"]]) < 100000:
                        if dbfRow["A"] != int(row[self.colnameToCsvIndex["A"]]):
                            raise NetworkException(
                                "Assertion error for A on row %d: %s != %s"
                                % (oldrownum, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]]))
                            )
                    if int(row[self.colnameToCsvIndex["B"]]) < 100000:
                        if dbfRow["B"] != int(row[self.colnameToCsvIndex["B"]]):
                            raise NetworkException(
                                "Assertion error for B on row %d: %s != %s"
                                % (oldrownum, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]]))
                            )
                    self.trnAsgnTable[newrownum]["FREQ"] = dbfRow["FREQ"]
                    self.trnAsgnTable[newrownum]["SEQ"] = dbfRow["SEQ"]

                    trySeq = dbfRow["SEQ"]
                    # ------------ special one-time computed fields

                    # ABNameSeq is more complicated because we want it to be unique
                    AB = row[self.colnameToCsvIndex["A"]] + " " + row[self.colnameToCsvIndex["B"]]
                    self.trnAsgnTable[newrownum]["AB"] = AB

                    ABNameSeq = AB + " " + linename
                    if trySeq > 0:
                        tryABNameSeq = ABNameSeq + " " + str(trySeq)

                        # This line seems to be a problem... A/B/NAME/SEQ are not unique
                        if tryABNameSeq in ABNameSeqSet:
                            WranglerLogger.warn("Non-Unique A/B/Name/Seq: " + tryABNameSeq + "; faking SEQ!")
                        # Find one that works
                        while tryABNameSeq in ABNameSeqSet:
                            trySeq += 1
                            tryABNameSeq = ABNameSeq + " " + str(trySeq)
                        ABNameSeq = tryABNameSeq
                    self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq
                    ABNameSeqSet.add(ABNameSeq)

                    ABNameSeq_List.append(
                        (
                            int(row[self.colnameToCsvIndex["A"]]),
                            int(row[self.colnameToCsvIndex["B"]]),
                            row[self.colnameToCsvIndex["NAME"]],
                            trySeq,
                        )
                    )

                    # ------------ straight lookup FULLNAME, VEHTYPE, VEHCAP; easy calc for PERIODCAP
                    self.trnAsgnTable[newrownum]["SYSTEM"] = system
                    self.trnAsgnTable[newrownum]["VEHTYPE"] = vehicletype

                    self.trnAsgnTable[newrownum]["FULLNAME"] = self.capacity.getFullname(linename, self.timeperiod)

                    try:
                        (vtype, vehcap) = self.capacity.getVehicleTypeAndCapacity(linename, self.timeperiod)

                        self.trnAsgnTable[newrownum]["VEHCAP"] = vehcap
                        self.trnAsgnTable[newrownum]["PERIODCAP"] = (
                            TransitLine.HOURS_PER_TIMEPERIOD[self.timeperiod]
                            * 60.0
                            * vehcap
                            / self.trnAsgnTable[newrownum]["FREQ"]
                        )
                    except:
                        self.trnAsgnTable[newrownum]["VEHCAP"] = 0
                        self.trnAsgnTable[newrownum]["PERIODCAP"] = 0

                    # if we still don't have a system, warn
                    if self.trnAsgnTable[newrownum]["SYSTEM"] == "" and not warnline.has_key(linename):
                        WranglerLogger.warning("No default system: " + linename)
                        warnline[linename] = 1

                    # ---------add in any grouping that may want to use
                    if self.lineToGroup.has_key(linename):
                        self.trnAsgnTable[newrownum]["GROUP"] = self.lineToGroup[linename]
                    else:
                        self.trnAsgnTable[newrownum]["GROUP"] = ""

                    # initialize additive fields
                    for field in self.trnAsgnAdditiveFields:
                        if row[self.colnameToCsvIndex[field]] == "":
                            self.trnAsgnTable[newrownum][field] = 0.0
                        else:
                            self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]])

                # end initial table fill

                # Add in the subsequent assignment files
                else:

                    # print oldrownum, newrownum, ABNameSeq_List[newrownum]
                    # print row[self.colnameToCsvIndex["NAME"]], ABNameSeq_List[oldrownum][2]

                    assert int(row[self.colnameToCsvIndex["A"]]) == ABNameSeq_List[newrownum][0]
                    assert int(row[self.colnameToCsvIndex["B"]]) == ABNameSeq_List[newrownum][1]
                    # these don't nec match, can be *32 in ferry skim rather than the bart vehicle name, for example
                    # assert(  row[self.colnameToCsvIndex["NAME"]] == ABNameSeq_List[newrownum][2])

                    ABNameSeq = (
                        row[self.colnameToCsvIndex["A"]]
                        + " "
                        + row[self.colnameToCsvIndex["B"]]
                        + " "
                        + row[self.colnameToCsvIndex["NAME"]].rstrip()
                    )
                    if ABNameSeq_List[newrownum][3] > 0:
                        ABNameSeq += " " + str(ABNameSeq_List[newrownum][3])
                    for field in self.trnAsgnAdditiveFields:
                        if row[self.colnameToCsvIndex[field]] != "":
                            self.trnAsgnTable[ABNameSeq][field] += float(row[self.colnameToCsvIndex[field]])

                newrownum += 1
                oldrownum += 1

            # we're done with this; free it up
            del filereader
            if indbf:
                del indbf

            # Table is created and filled -- set the index
            if mode == self.MODES[0]:
                try:
                    self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ")
                except:
                    # failure - try to figure out why
                    ABNameSeqList = []
                    for row in self.trnAsgnTable:
                        ABNameSeqList.append(row["ABNAMESEQ"])
                    ABNameSeqList.sort()
                    for idx in range(len(ABNameSeqList) - 1):
                        if ABNameSeqList[idx] == ABNameSeqList[idx + 1]:
                            WranglerLogger.warn("Duplicate ABNAMESEQ at idx %d : [%s]" % (idx, ABNameSeqList[idx]))
                    exit(1)

        # ok the table is all filled in -- fill in the LOAD
        for row in self.trnAsgnTable:
            if row["VEHCAP"] == 0:
                continue
            tpfactor = self.TIMEPERIOD_FACTOR[self.timeperiod]

            # mode-specific peaking factor will over-ride
            if row["MODE"] in self.TIMEPERIOD_FACTOR:
                tpfactor = self.TIMEPERIOD_FACTOR[row["MODE"]][self.timeperiod]

            row["LOAD"] = row["AB_VOL"] * tpfactor * row["FREQ"] / (60.0 * row["VEHCAP"])

        # build the aggregate table for key="A B"
        if self.aggregateAll:
            self.buildAggregateTable()