def readAggregateDbfs(self, asgnFileName, aggregateFileName=None): """ This is essentially the reverse of writeDbfs() below. """ self.initializeFields() # this may be unnecessary self.trnAsgnTable = dbfTableReader(asgnFileName) self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") # a little bit of cleanup headerTuples = [] for headerFieldType in self.trnAsgnTable.header: headerTuples.append(headerFieldType.toTuple()) # rstrip spaces off the end of the text fields for row in self.trnAsgnTable: for headerTuple in headerTuples: if headerTuple[1] == 'C': row[headerTuple[0]] = string.rstrip(row[headerTuple[0]]) # this is the index! self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") # the link-level aggregate table if not aggregateFileName: self.aggregateAll = False self.aggregateTable = False return self.aggregateAll = True self.aggregateTable = dbfTableReader(aggregateFileName) # cleanup again headerTuples = [] for headerFieldType in self.aggregateTable.header: headerTuples.append(headerFieldType.toTuple()) # rstrip spaces off the end of the text fields for row in self.aggregateTable: for headerTuple in headerTuples: if headerTuple[1] == 'C': row[headerTuple[0]] = string.rstrip(row[headerTuple[0]])
def readAggregateDbfs(self, asgnFileName, aggregateFileName=None): """ This is essentially the reverse of writeDbfs() below. """ self.initializeFields() # this may be unnecessary self.trnAsgnTable = dbfTableReader(asgnFileName) self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") # a little bit of cleanup headerTuples = [] for headerFieldType in self.trnAsgnTable.header: headerTuples.append(headerFieldType.toTuple()) # rstrip spaces off the end of the text fields for row in self.trnAsgnTable: for headerTuple in headerTuples: if headerTuple[1] == "C": row[headerTuple[0]] = string.rstrip(row[headerTuple[0]]) # this is the index! self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") # the link-level aggregate table if not aggregateFileName: self.aggregateAll = False self.aggregateTable = False return self.aggregateAll = True self.aggregateTable = dbfTableReader(aggregateFileName) # cleanup again headerTuples = [] for headerFieldType in self.aggregateTable.header: headerTuples.append(headerFieldType.toTuple()) # rstrip spaces off the end of the text fields for row in self.aggregateTable: for headerTuple in headerTuples: if headerTuple[1] == "C": row[headerTuple[0]] = string.rstrip(row[headerTuple[0]])
def readTransitAssignmentCsvs(self): """ Read the transit assignment dbfs, the direct output of Cube's transit assignment. """ self.trnAsgnTable = False self.aggregateTable = False warnline = {} ABNameSeq_List = [] # (A,B,NAME,SEQ) from the dbf/csvs # open the input assignment files for mode in self.MODES: if mode == "WMWVIS": filename = os.path.join(self.assigndir, "VISWMW" + self.timeperiod + ".csv") elif mode[1] == "T": filename = os.path.join(self.assigndir, "NS" + mode + self.timeperiod + ".csv") else: filename = os.path.join(self.assigndir, "SF" + mode + self.timeperiod + ".csv") # Read the DBF file into datatable WranglerLogger.info("Reading " + filename) # Create our table data structure once if mode == self.MODES[0]: # figure out how many records numrecs = 0 totalrows = 0 filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE) for row in filereader: # header row? if row[0] == "A": self.initializeFields(row) continue elif totalrows == 0 and not self.csvColnames: self.initializeFields() totalrows += 1 if self.profileNode and \ (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode): continue if int(row[self.colnameToCsvIndex["MODE"]] ) in self.ignoreModes: continue linename = row[self.colnameToCsvIndex["NAME"]].strip() # exclude this system? (system, vehicletype) = self.capacity.getSystemAndVehicleType( linename, self.timeperiod) if len(self.system) > 0 and system not in self.system: continue numrecs += 1 WranglerLogger.info("Keeping %d records out of %d" % (numrecs, totalrows)) self.trnAsgnTable = DataTable( numRecords=numrecs, fieldNames=self.trnAsgnFields.keys(), numpyFieldTypes=self.trnAsgnFields.values()) ABNameSeqSet = set() # Go through the records newrownum = 0 # row number in the trnAsgnTable,ABNameSeq_List -- rows we're keeping oldrownum = 0 # row number in the csv,dbf -- all input rows filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE) # for the first csv only, also read the dbf for the freq and seq fields if mode == self.MODES[0]: indbf = dbfTableReader( os.path.join(self.assigndir, "SFWBW" + self.timeperiod + ".dbf")) else: indbf = None for row in filereader: # header row? if row[0] == "A": continue if self.profileNode: if (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode): continue elif int(row[self.colnameToCsvIndex["AB_VOL"]]) > 0: WranglerLogger.info( "Link %s %s for mode %s has AB_VOL %s" % (row[self.colnameToCsvIndex["A"]], row[self.colnameToCsvIndex["B"]], mode, row[self.colnameToCsvIndex["AB_VOL"]])) if int(row[ self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: oldrownum += 1 continue linename = row[self.colnameToCsvIndex["NAME"]].strip() # exclude this system? (system, vehicletype) = self.capacity.getSystemAndVehicleType( linename, self.timeperiod) if len(self.system) > 0 and system not in self.system: continue # Initial table fill: Special stuff for the first time through if mode == self.MODES[0]: # ------------ these fields just get used directly for field in self.trnAsgnCopyFields: try: # integer fields if self.trnAsgnFields[field][0] in ['u', 'b']: if row[self.colnameToCsvIndex[field]] == "": self.trnAsgnTable[newrownum][field] = 0 elif field in ['TIME', 'DIST']: # backwards compatibility - dbfs were 100ths of a mile/min self.trnAsgnTable[newrownum][ field] = float( row[self.colnameToCsvIndex[field]] ) * 100.0 else: self.trnAsgnTable[newrownum][field] = int( row[self.colnameToCsvIndex[field]]) # float fields elif self.trnAsgnFields[field][0] == 'f': if row[self.colnameToCsvIndex[field]] == "": self.trnAsgnTable[newrownum][field] = 0.0 else: self.trnAsgnTable[newrownum][ field] = float( row[self.colnameToCsvIndex[field]]) # text fields else: self.trnAsgnTable[newrownum][field] = row[ self.colnameToCsvIndex[field]] except: WranglerLogger.fatal( "Error intepreting field %s: [%s]" % (field, str(self.colnameToCsvIndex[field]))) WranglerLogger.fatal("row=%s" % str(row)) WranglerLogger.fatal(sys.exc_info()[0]) WranglerLogger.fatal(sys.exc_info()[1]) sys.exit(2) # ------------ these fields come from the dbf because they're missing in the csv (sigh) dbfRow = indbf.__getitem__(oldrownum) if int(row[self.colnameToCsvIndex["A"]]) < 100000: if dbfRow["A"] != int( row[self.colnameToCsvIndex["A"]]): raise NetworkException( "Assertion error for A on row %d: %s != %s" % (oldrownum, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]]))) if int(row[self.colnameToCsvIndex["B"]]) < 100000: if dbfRow["B"] != int( row[self.colnameToCsvIndex["B"]]): raise NetworkException( "Assertion error for B on row %d: %s != %s" % (oldrownum, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]]))) self.trnAsgnTable[newrownum]["FREQ"] = dbfRow["FREQ"] self.trnAsgnTable[newrownum]["SEQ"] = dbfRow["SEQ"] trySeq = dbfRow["SEQ"] # ------------ special one-time computed fields # ABNameSeq is more complicated because we want it to be unique AB = row[self.colnameToCsvIndex["A"]] + " " + row[ self.colnameToCsvIndex["B"]] self.trnAsgnTable[newrownum]["AB"] = AB ABNameSeq = AB + " " + linename if trySeq > 0: tryABNameSeq = ABNameSeq + " " + str(trySeq) # This line seems to be a problem... A/B/NAME/SEQ are not unique if tryABNameSeq in ABNameSeqSet: WranglerLogger.warn("Non-Unique A/B/Name/Seq: " + tryABNameSeq + "; faking SEQ!") # Find one that works while tryABNameSeq in ABNameSeqSet: trySeq += 1 tryABNameSeq = ABNameSeq + " " + str(trySeq) ABNameSeq = tryABNameSeq self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq ABNameSeqSet.add(ABNameSeq) ABNameSeq_List.append( (int(row[self.colnameToCsvIndex["A"]]), int(row[self.colnameToCsvIndex["B"]]), row[self.colnameToCsvIndex["NAME"]], trySeq)) # ------------ straight lookup FULLNAME, VEHTYPE, VEHCAP; easy calc for PERIODCAP self.trnAsgnTable[newrownum]["SYSTEM"] = system self.trnAsgnTable[newrownum]["VEHTYPE"] = vehicletype self.trnAsgnTable[newrownum][ "FULLNAME"] = self.capacity.getFullname( linename, self.timeperiod) try: (vtype, vehcap) = self.capacity.getVehicleTypeAndCapacity( linename, self.timeperiod) self.trnAsgnTable[newrownum]["VEHCAP"] = vehcap self.trnAsgnTable[newrownum][ "PERIODCAP"] = TransitLine.HOURS_PER_TIMEPERIOD[ self. timeperiod] * 60.0 * vehcap / self.trnAsgnTable[ newrownum]["FREQ"] except: self.trnAsgnTable[newrownum]["VEHCAP"] = 0 self.trnAsgnTable[newrownum]["PERIODCAP"] = 0 # if we still don't have a system, warn if self.trnAsgnTable[newrownum][ "SYSTEM"] == "" and not warnline.has_key(linename): WranglerLogger.warning("No default system: " + linename) warnline[linename] = 1 #---------add in any grouping that may want to use if self.lineToGroup.has_key(linename): self.trnAsgnTable[newrownum][ "GROUP"] = self.lineToGroup[linename] else: self.trnAsgnTable[newrownum]["GROUP"] = "" # initialize additive fields for field in self.trnAsgnAdditiveFields: if row[self.colnameToCsvIndex[field]] == "": self.trnAsgnTable[newrownum][field] = 0.0 else: self.trnAsgnTable[newrownum][field] = float( row[self.colnameToCsvIndex[field]]) # end initial table fill # Add in the subsequent assignment files else: # print oldrownum, newrownum, ABNameSeq_List[newrownum] # print row[self.colnameToCsvIndex["NAME"]], ABNameSeq_List[oldrownum][2] assert (int(row[self.colnameToCsvIndex["A"]]) == ABNameSeq_List[newrownum][0]) assert (int(row[self.colnameToCsvIndex["B"]]) == ABNameSeq_List[newrownum][1]) # these don't nec match, can be *32 in ferry skim rather than the bart vehicle name, for example # assert( row[self.colnameToCsvIndex["NAME"]] == ABNameSeq_List[newrownum][2]) ABNameSeq = row[self.colnameToCsvIndex["A"]] + " " + \ row[self.colnameToCsvIndex["B"]] + " " + \ row[self.colnameToCsvIndex["NAME"]].rstrip() if ABNameSeq_List[newrownum][3] > 0: ABNameSeq += " " + str(ABNameSeq_List[newrownum][3]) for field in self.trnAsgnAdditiveFields: if row[self.colnameToCsvIndex[field]] != "": self.trnAsgnTable[ABNameSeq][field] += float( row[self.colnameToCsvIndex[field]]) newrownum += 1 oldrownum += 1 # we're done with this; free it up del filereader if indbf: del indbf # Table is created and filled -- set the index if mode == self.MODES[0]: try: self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") except: # failure - try to figure out why ABNameSeqList = [] for row in self.trnAsgnTable: ABNameSeqList.append(row["ABNAMESEQ"]) ABNameSeqList.sort() for idx in range(len(ABNameSeqList) - 1): if ABNameSeqList[idx] == ABNameSeqList[idx + 1]: WranglerLogger.warn( "Duplicate ABNAMESEQ at idx %d : [%s]" % (idx, ABNameSeqList[idx])) exit(1) # ok the table is all filled in -- fill in the LOAD for row in self.trnAsgnTable: if row["VEHCAP"] == 0: continue tpfactor = self.TIMEPERIOD_FACTOR[self.timeperiod] # mode-specific peaking factor will over-ride if row["MODE"] in self.TIMEPERIOD_FACTOR: tpfactor = self.TIMEPERIOD_FACTOR[row["MODE"]][self.timeperiod] row["LOAD"] = row["AB_VOL"] * tpfactor * row["FREQ"] / ( 60.0 * row["VEHCAP"]) # build the aggregate table for key="A B" if self.aggregateAll: self.buildAggregateTable()
def readTransitAssignmentCsvs(self): """ Read the transit assignment dbfs, the direct output of Cube's transit assignment. """ self.trnAsgnTable = False self.aggregateTable = False warnline = {} # open the input assignment files for mode in self.MODES: if mode == "VWMW": filename = os.path.join(self.assigndir, "VISWMW" + self.timeperiod + ".csv") elif mode[1]=="T": filename = os.path.join(self.assigndir, "NS" + mode + self.timeperiod + ".csv") else: filename = os.path.join(self.assigndir, "SF" + mode + self.timeperiod + ".csv") # Read the DBF file into datatable WranglerLogger.info("Reading "+filename) # Create our table data structure once if mode == self.MODES[0]: # figure out how many records numrecs = 0 totalrows = 0 filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE) for row in filereader: # header row? if row[0]=="A": self.initializeFields(row) continue elif totalrows==0 and not self.csvColnames: self.initializeFields() totalrows += 1 if self.profileNode and \ (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode): continue if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: continue linename = row[self.colnameToCsvIndex["NAME"]].strip() # exclude this system? (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod) if len(self.system)>0 and system not in self.system: continue numrecs += 1 WranglerLogger.info("Keeping %d records out of %d" % (numrecs, totalrows)) self.trnAsgnTable = DataTable(numRecords=numrecs, fieldNames=self.trnAsgnFields.keys(), numpyFieldTypes=self.trnAsgnFields.values()) ABNameSeqSet = set() ABSet = set() # Go through the records newrownum = 0 oldrownum = 0 filereader = csv.reader(open(filename, 'rb'), delimiter=',', quoting=csv.QUOTE_NONE) indbf = dbfTableReader(os.path.join(self.assigndir, "SFWBW" + self.timeperiod + ".dbf")) for row in filereader: # header row? if row[0]=="A": continue oldrownum += 1 if self.profileNode: if (int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode): continue elif int(row[self.colnameToCsvIndex["AB_VOL"]]) > 0: WranglerLogger.info("Link %s %s for mode %s has AB_VOL %s" % (row[self.colnameToCsvIndex["A"]], row[self.colnameToCsvIndex["B"]], mode, row[self.colnameToCsvIndex["AB_VOL"]])) if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: continue linename = row[self.colnameToCsvIndex["NAME"]].strip() # exclude this system? (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod) if len(self.system)>0 and system not in self.system: continue # Initial table fill: Special stuff for the first time through if mode == self.MODES[0]: # ------------ these fields just get used directly for field in self.trnAsgnCopyFields: try: # integer fields if self.trnAsgnFields[field][0] in ['u','b']: if row[self.colnameToCsvIndex[field]]=="": self.trnAsgnTable[newrownum][field] = 0 elif field in ['TIME','DIST']: # backwards compatibility - dbfs were 100ths of a mile/min self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]])*100.0 else: self.trnAsgnTable[newrownum][field] = int(row[self.colnameToCsvIndex[field]]) # float fields elif self.trnAsgnFields[field][0] == 'f': if row[self.colnameToCsvIndex[field]]=="": self.trnAsgnTable[newrownum][field] = 0.0 else: self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]]) # text fields else: self.trnAsgnTable[newrownum][field] = row[self.colnameToCsvIndex[field]] except: WranglerLogger.fatal("Error intepreting field %s: [%s]" % (field, str(self.colnameToCsvIndex[field]))) WranglerLogger.fatal("row=%s" % str(row)) WranglerLogger.fatal(sys.exc_info()[0]) WranglerLogger.fatal(sys.exc_info()[1]) sys.exit(2) # ------------ these fields come from the dbf because they're missing in the csv (sigh) dbfRow = indbf.__getitem__(oldrownum-1) if int(row[self.colnameToCsvIndex["A"]])<100000: if dbfRow["A"]!=int(row[self.colnameToCsvIndex["A"]]): raise NetworkException("Assertion error for A on row %d: %s != %s" % (oldrownum-1, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]]))) if int(row[self.colnameToCsvIndex["B"]])<100000: if dbfRow["B"]!=int(row[self.colnameToCsvIndex["B"]]): raise NetworkException("Assertion error for B on row %d: %s != %s" % (oldrownum-1, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]]))) self.trnAsgnTable[newrownum]["FREQ"] = dbfRow["FREQ"] self.trnAsgnTable[newrownum]["SEQ"] = dbfRow["SEQ"] trySeq = dbfRow["SEQ"] # ------------ special one-time computed fields # ABNameSeq is more complicated because we want it to be unique AB = row[self.colnameToCsvIndex["A"]] + " " + row[self.colnameToCsvIndex["B"]] self.trnAsgnTable[newrownum]["AB"] = AB ABNameSeq = AB + " " + linename if trySeq>0: tryABNameSeq = ABNameSeq + " " + str(trySeq) # This line seems to be a problem... A/B/NAME/SEQ are not unique if tryABNameSeq in ABNameSeqSet: WranglerLogger.warn("Non-Unique A/B/Name/Seq: " + tryABNameSeq + "; faking SEQ!") # Find one that works while tryABNameSeq in ABNameSeqSet: trySeq += 1 tryABNameSeq = ABNameSeq + " " + str(trySeq) ABNameSeq = tryABNameSeq self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq ABNameSeqSet.add(ABNameSeq) ABSet.add(AB) # ------------ straight lookup FULLNAME, VEHTYPE, VEHCAP; easy calc for PERIODCAP self.trnAsgnTable[newrownum]["SYSTEM"] = system self.trnAsgnTable[newrownum]["VEHTYPE"] = vehicletype self.trnAsgnTable[newrownum]["FULLNAME"] = self.capacity.getFullname(linename, self.timeperiod) try: (vtype, vehcap) = self.capacity.getVehicleTypeAndCapacity(linename, self.timeperiod) self.trnAsgnTable[newrownum]["VEHCAP"] = vehcap self.trnAsgnTable[newrownum]["PERIODCAP"] = TransitLine.HOURS_PER_TIMEPERIOD[self.timeperiod] * 60.0 * vehcap/self.trnAsgnTable[newrownum]["FREQ"] except: self.trnAsgnTable[newrownum]["VEHCAP"] = 0 self.trnAsgnTable[newrownum]["PERIODCAP"] = 0 # if we still don't have a system, warn if self.trnAsgnTable[newrownum]["SYSTEM"] == "" and not warnline.has_key(linename): WranglerLogger.warning("No default system: " + linename) warnline[linename] =1 #---------add in any grouping that may want to use if self.lineToGroup.has_key(linename): self.trnAsgnTable[newrownum]["GROUP"] = self.lineToGroup[linename] else: self.trnAsgnTable[newrownum]["GROUP"] = "" # initialize additive fields for field in self.trnAsgnAdditiveFields: if row[self.colnameToCsvIndex[field]]=="": self.trnAsgnTable[newrownum][field] = 0.0 else: self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]]) # end initial table fill # Add in the subsequent assignment files else: dbfRow = indbf.__getitem__(oldrownum-1) while (((int(row[self.colnameToCsvIndex["A"]])<100000) and (dbfRow["A"]!=int(row[self.colnameToCsvIndex["A"]]))) or ((int(row[self.colnameToCsvIndex["B"]])<100000) and (dbfRow["B"]!=int(row[self.colnameToCsvIndex["B"]])))): oldrownum += 1 dbfRow = indbf.__getitem__(oldrownum-1) if int(row[self.colnameToCsvIndex["A"]])<100000: if dbfRow["A"]!=int(row[self.colnameToCsvIndex["A"]]): raise NetworkException("Assertion error for A on row %d: %s != %s" % (oldrownum-1, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]]))) if int(row[self.colnameToCsvIndex["B"]])<100000: if dbfRow["B"]!=int(row[self.colnameToCsvIndex["B"]]): raise NetworkException("Assertion error for B on row %d: %s != %s" % (oldrownum-1, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]]))) ABNameSeq = row[self.colnameToCsvIndex["A"]] + " " + \ row[self.colnameToCsvIndex["B"]] + " " + \ row[self.colnameToCsvIndex["NAME"]].rstrip() if dbfRow["SEQ"]>0: ABNameSeq += " " + str(dbfRow["SEQ"]) for field in self.trnAsgnAdditiveFields: if row[self.colnameToCsvIndex[field]] !="": self.trnAsgnTable[ABNameSeq][field] += float(row[self.colnameToCsvIndex[field]]) newrownum += 1 # we're done with this; free it up del filereader del indbf # Table is created and filled -- set the index if mode == self.MODES[0]: try: self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") except: # failure - try to figure out why ABNameSeqList = [] for row in self.trnAsgnTable: ABNameSeqList.append(row["ABNAMESEQ"]) ABNameSeqList.sort() for idx in range(len(ABNameSeqList)-1): if ABNameSeqList[idx]==ABNameSeqList[idx+1]: WranglerLogger.warn("Duplicate ABNAMESEQ at idx %d : [%s]" % (idx,ABNameSeqList[idx])) exit(1) # ok the table is all filled in -- fill in the LOAD for row in self.trnAsgnTable: if row["VEHCAP"] == 0: continue row["LOAD"] = row["AB_VOL"] * self.TIMEPERIOD_FACTOR[self.timeperiod] * row["FREQ"] / (60.0 * row["VEHCAP"]) # build the aggregate table for key="A B" if self.aggregateAll: self.aggregateTable = DataTable(numRecords=len(ABSet), fieldNames=self.aggregateFields.keys(), numpyFieldTypes=self.aggregateFields.values()) ABtoRowIndex = {} rowsUsed = 0 for row in self.trnAsgnTable: if row["AB"] not in ABtoRowIndex: rowIndex = rowsUsed self.aggregateTable[rowIndex]["AB"] = row["AB"] self.aggregateTable[rowIndex]["A"] = row["A"] self.aggregateTable[rowIndex]["B"] = row["B"] self.aggregateTable[rowIndex]["DIST"] = row["DIST"] self.aggregateTable[rowIndex]["FREQ"] = 0.0 self.aggregateTable[rowIndex]["PERIODCAP"] = 0.0 self.aggregateTable[rowIndex]["LOAD"] = 0.0 self.aggregateTable[rowIndex]["MAXLOAD"] = 0.0 for field in self.trnAsgnAdditiveFields: # sum self.aggregateTable[rowIndex][field] = 0.0 ABtoRowIndex[row["AB"]] = rowsUsed rowsUsed += 1 else: rowIndex = ABtoRowIndex[row["AB"]] for field in self.trnAsgnAdditiveFields: # sum self.aggregateTable[rowIndex][field] += row[field] self.aggregateTable[rowIndex]["AB_VOL"] += row["AB_VOL"] self.aggregateTable[rowIndex]["BA_VOL"] += row["BA_VOL"] self.aggregateTable[rowIndex]["PERIODCAP"] += row["PERIODCAP"] self.aggregateTable[rowIndex]["MAXLOAD"] = max(row["LOAD"], self.aggregateTable[rowIndex]["MAXLOAD"]) self.aggregateTable[rowIndex]["FREQ"] += 1/row["FREQ"] # combining -- will take reciprocal later self.aggregateTable.setIndex(fieldName="AB") count=0 for row in self.aggregateTable: count += 1 if row["FREQ"]>0: row["FREQ"] = 1/row["FREQ"] if row["PERIODCAP"]>0: row["LOAD"] = float(row["AB_VOL"]) / row["PERIODCAP"] # print row["LOAD"] WranglerLogger.debug("count "+str(count)+" lines in aggregate table")
def readTransitAssignmentCsvs(self): """ Read the transit assignment dbfs, the direct output of Cube's transit assignment. """ self.trnAsgnTable = False self.aggregateTable = False warnline = {} ABNameSeq_List = [] # (A,B,NAME,SEQ) from the dbf/csvs # open the input assignment files for mode in self.MODES: if mode == "WMWVIS": filename = os.path.join(self.assigndir, "VISWMW" + self.timeperiod + ".csv") elif mode[1] == "T": filename = os.path.join(self.assigndir, "NS" + mode + self.timeperiod + ".csv") else: filename = os.path.join(self.assigndir, "SF" + mode + self.timeperiod + ".csv") # Read the DBF file into datatable WranglerLogger.info("Reading " + filename) # Create our table data structure once if mode == self.MODES[0]: # figure out how many records numrecs = 0 totalrows = 0 filereader = csv.reader(open(filename, "rb"), delimiter=",", quoting=csv.QUOTE_NONE) for row in filereader: # header row? if row[0] == "A": self.initializeFields(row) continue elif totalrows == 0 and not self.csvColnames: self.initializeFields() totalrows += 1 if self.profileNode and ( int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode ): continue if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: continue linename = row[self.colnameToCsvIndex["NAME"]].strip() # exclude this system? (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod) if len(self.system) > 0 and system not in self.system: continue numrecs += 1 WranglerLogger.info("Keeping %d records out of %d" % (numrecs, totalrows)) self.trnAsgnTable = DataTable( numRecords=numrecs, fieldNames=self.trnAsgnFields.keys(), numpyFieldTypes=self.trnAsgnFields.values(), ) ABNameSeqSet = set() # Go through the records newrownum = 0 # row number in the trnAsgnTable,ABNameSeq_List -- rows we're keeping oldrownum = 0 # row number in the csv,dbf -- all input rows filereader = csv.reader(open(filename, "rb"), delimiter=",", quoting=csv.QUOTE_NONE) # for the first csv only, also read the dbf for the freq and seq fields if mode == self.MODES[0]: indbf = dbfTableReader(os.path.join(self.assigndir, "SFWBW" + self.timeperiod + ".dbf")) else: indbf = None for row in filereader: # header row? if row[0] == "A": continue if self.profileNode: if ( int(row[self.colnameToCsvIndex["A"]]) != self.profileNode and int(row[self.colnameToCsvIndex["B"]]) != self.profileNode ): continue elif int(row[self.colnameToCsvIndex["AB_VOL"]]) > 0: WranglerLogger.info( "Link %s %s for mode %s has AB_VOL %s" % ( row[self.colnameToCsvIndex["A"]], row[self.colnameToCsvIndex["B"]], mode, row[self.colnameToCsvIndex["AB_VOL"]], ) ) if int(row[self.colnameToCsvIndex["MODE"]]) in self.ignoreModes: oldrownum += 1 continue linename = row[self.colnameToCsvIndex["NAME"]].strip() # exclude this system? (system, vehicletype) = self.capacity.getSystemAndVehicleType(linename, self.timeperiod) if len(self.system) > 0 and system not in self.system: continue # Initial table fill: Special stuff for the first time through if mode == self.MODES[0]: # ------------ these fields just get used directly for field in self.trnAsgnCopyFields: try: # integer fields if self.trnAsgnFields[field][0] in ["u", "b"]: if row[self.colnameToCsvIndex[field]] == "": self.trnAsgnTable[newrownum][field] = 0 elif field in ["TIME", "DIST"]: # backwards compatibility - dbfs were 100ths of a mile/min self.trnAsgnTable[newrownum][field] = ( float(row[self.colnameToCsvIndex[field]]) * 100.0 ) else: self.trnAsgnTable[newrownum][field] = int(row[self.colnameToCsvIndex[field]]) # float fields elif self.trnAsgnFields[field][0] == "f": if row[self.colnameToCsvIndex[field]] == "": self.trnAsgnTable[newrownum][field] = 0.0 else: self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]]) # text fields else: self.trnAsgnTable[newrownum][field] = row[self.colnameToCsvIndex[field]] except: WranglerLogger.fatal( "Error intepreting field %s: [%s]" % (field, str(self.colnameToCsvIndex[field])) ) WranglerLogger.fatal("row=%s" % str(row)) WranglerLogger.fatal(sys.exc_info()[0]) WranglerLogger.fatal(sys.exc_info()[1]) sys.exit(2) # ------------ these fields come from the dbf because they're missing in the csv (sigh) dbfRow = indbf.__getitem__(oldrownum) if int(row[self.colnameToCsvIndex["A"]]) < 100000: if dbfRow["A"] != int(row[self.colnameToCsvIndex["A"]]): raise NetworkException( "Assertion error for A on row %d: %s != %s" % (oldrownum, str(dbfRow["A"]), str(row[self.colnameToCsvIndex["A"]])) ) if int(row[self.colnameToCsvIndex["B"]]) < 100000: if dbfRow["B"] != int(row[self.colnameToCsvIndex["B"]]): raise NetworkException( "Assertion error for B on row %d: %s != %s" % (oldrownum, str(dbfRow["B"]), str(row[self.colnameToCsvIndex["B"]])) ) self.trnAsgnTable[newrownum]["FREQ"] = dbfRow["FREQ"] self.trnAsgnTable[newrownum]["SEQ"] = dbfRow["SEQ"] trySeq = dbfRow["SEQ"] # ------------ special one-time computed fields # ABNameSeq is more complicated because we want it to be unique AB = row[self.colnameToCsvIndex["A"]] + " " + row[self.colnameToCsvIndex["B"]] self.trnAsgnTable[newrownum]["AB"] = AB ABNameSeq = AB + " " + linename if trySeq > 0: tryABNameSeq = ABNameSeq + " " + str(trySeq) # This line seems to be a problem... A/B/NAME/SEQ are not unique if tryABNameSeq in ABNameSeqSet: WranglerLogger.warn("Non-Unique A/B/Name/Seq: " + tryABNameSeq + "; faking SEQ!") # Find one that works while tryABNameSeq in ABNameSeqSet: trySeq += 1 tryABNameSeq = ABNameSeq + " " + str(trySeq) ABNameSeq = tryABNameSeq self.trnAsgnTable[newrownum]["ABNAMESEQ"] = ABNameSeq ABNameSeqSet.add(ABNameSeq) ABNameSeq_List.append( ( int(row[self.colnameToCsvIndex["A"]]), int(row[self.colnameToCsvIndex["B"]]), row[self.colnameToCsvIndex["NAME"]], trySeq, ) ) # ------------ straight lookup FULLNAME, VEHTYPE, VEHCAP; easy calc for PERIODCAP self.trnAsgnTable[newrownum]["SYSTEM"] = system self.trnAsgnTable[newrownum]["VEHTYPE"] = vehicletype self.trnAsgnTable[newrownum]["FULLNAME"] = self.capacity.getFullname(linename, self.timeperiod) try: (vtype, vehcap) = self.capacity.getVehicleTypeAndCapacity(linename, self.timeperiod) self.trnAsgnTable[newrownum]["VEHCAP"] = vehcap self.trnAsgnTable[newrownum]["PERIODCAP"] = ( TransitLine.HOURS_PER_TIMEPERIOD[self.timeperiod] * 60.0 * vehcap / self.trnAsgnTable[newrownum]["FREQ"] ) except: self.trnAsgnTable[newrownum]["VEHCAP"] = 0 self.trnAsgnTable[newrownum]["PERIODCAP"] = 0 # if we still don't have a system, warn if self.trnAsgnTable[newrownum]["SYSTEM"] == "" and not warnline.has_key(linename): WranglerLogger.warning("No default system: " + linename) warnline[linename] = 1 # ---------add in any grouping that may want to use if self.lineToGroup.has_key(linename): self.trnAsgnTable[newrownum]["GROUP"] = self.lineToGroup[linename] else: self.trnAsgnTable[newrownum]["GROUP"] = "" # initialize additive fields for field in self.trnAsgnAdditiveFields: if row[self.colnameToCsvIndex[field]] == "": self.trnAsgnTable[newrownum][field] = 0.0 else: self.trnAsgnTable[newrownum][field] = float(row[self.colnameToCsvIndex[field]]) # end initial table fill # Add in the subsequent assignment files else: # print oldrownum, newrownum, ABNameSeq_List[newrownum] # print row[self.colnameToCsvIndex["NAME"]], ABNameSeq_List[oldrownum][2] assert int(row[self.colnameToCsvIndex["A"]]) == ABNameSeq_List[newrownum][0] assert int(row[self.colnameToCsvIndex["B"]]) == ABNameSeq_List[newrownum][1] # these don't nec match, can be *32 in ferry skim rather than the bart vehicle name, for example # assert( row[self.colnameToCsvIndex["NAME"]] == ABNameSeq_List[newrownum][2]) ABNameSeq = ( row[self.colnameToCsvIndex["A"]] + " " + row[self.colnameToCsvIndex["B"]] + " " + row[self.colnameToCsvIndex["NAME"]].rstrip() ) if ABNameSeq_List[newrownum][3] > 0: ABNameSeq += " " + str(ABNameSeq_List[newrownum][3]) for field in self.trnAsgnAdditiveFields: if row[self.colnameToCsvIndex[field]] != "": self.trnAsgnTable[ABNameSeq][field] += float(row[self.colnameToCsvIndex[field]]) newrownum += 1 oldrownum += 1 # we're done with this; free it up del filereader if indbf: del indbf # Table is created and filled -- set the index if mode == self.MODES[0]: try: self.trnAsgnTable.setIndex(fieldName="ABNAMESEQ") except: # failure - try to figure out why ABNameSeqList = [] for row in self.trnAsgnTable: ABNameSeqList.append(row["ABNAMESEQ"]) ABNameSeqList.sort() for idx in range(len(ABNameSeqList) - 1): if ABNameSeqList[idx] == ABNameSeqList[idx + 1]: WranglerLogger.warn("Duplicate ABNAMESEQ at idx %d : [%s]" % (idx, ABNameSeqList[idx])) exit(1) # ok the table is all filled in -- fill in the LOAD for row in self.trnAsgnTable: if row["VEHCAP"] == 0: continue tpfactor = self.TIMEPERIOD_FACTOR[self.timeperiod] # mode-specific peaking factor will over-ride if row["MODE"] in self.TIMEPERIOD_FACTOR: tpfactor = self.TIMEPERIOD_FACTOR[row["MODE"]][self.timeperiod] row["LOAD"] = row["AB_VOL"] * tpfactor * row["FREQ"] / (60.0 * row["VEHCAP"]) # build the aggregate table for key="A B" if self.aggregateAll: self.buildAggregateTable()