def _Execute(self):
with _m.logbook_trace(name="{classname} v{version}".format(classname=(self.__class__.__name__), version=self.version),
attributes=self._GetAtts()):
with self._zipFileMANAGER() as zf:
reader = zf.open('run_order.txt', 'rU')
try:
scripts = [line.strip() for line in reader]
finally:
reader.close()
self.TRACKER.reset(len(scripts))
nl = set(zf.namelist())
for script in scripts:
if script.endswith(".mac"):
if not script in nl:
raise IOError("NUP file formatting error: Script %s does not exist" %script)
with _m.logbook_trace("Running MACRO %s" %script):
self._RunMacro(zf, script)
elif script.endswith(".py"):
if not script in nl:
raise IOError("NUP file formatting error: Script %s does not exist" %script)
with _m.logbook_trace("Running PY_SCRIPT %s" %script):
self._RunPythonScript(zf, script)
else:
_m.logbook_write("Script type of '%'s not supported" %script)
self.TRACKER.completeTask()
def _WriteReport(self, lines):
with open(self.ExportFile, 'w') as writer:
writer.write("Countpost,Link,Auto Volume,Additional Volume,Auto Time")
for line in lines:
line = [str(c) for c in line]
writer.write("\n" + ','.join(line))
_m.logbook_write("Wrote report to %s" %self.ExportFile)
def _exectue(self):
with _m.logbook_trace("Renumber nodes from DMG2001 to NCS11"):
report = ""
for Scenario in self.Scenarios:
network = Scenario.get_network()
count = 0
for i in range(30000, 80000):
# All of the regional nodes from 30,000 to 79,999
# must be shifted down by 10,000.
# Cycling through them from bottom-up, to ensure no id conflicts occur.
n = network.node(i)
if n == None:
continue
newNumber = n.number - 10000
nn = network.node(newNumber)
if nn != None:
raise Exception("Renumbering of node " + str(n.number) + " failed! New number " + str(newNumber) +" already exists!")
n.number -= 10000
count += 1
for i in range(0,1000):
n = network.node(91999 - i)
# GO Rail nodes need to be shifted up by 7,000
# Cycling through them top to bottom to prevent id conflicts.
if n == None:
continue
newNumber = n.number + 7000
nn = network.node(newNumber)
if nn != None:
raise Exception("Renumbering of node " + str(n.number) + " failed! New number " + str(newNumber) +" already exists!")
n.number += 7000
count += 1
for i in range(0,1000):
n = network.node(90999 - i)
# Subway nodes need to be shifted up by 7,000
# Cycling through them top to bottom to prevent id conflicts.
if n == None:
continue
newNumber = n.number + 7000
nn = network.node(newNumber)
if nn != None:
raise Exception("Renumbering of node " + str(n.number) + " failed! New number " + str(newNumber) +" already exists!")
n.number += 7000
count += 1
Scenario.publish_network(network)
s = "{0} nodes were changed in scenario {1}".format(count, Scenario.id)
_m.logbook_write(s)
report += "<br>%s" %s
self.tool_run_msg = _m.PageBuilder.format_info(report)
def CheckToolExists(self):
ns = self.ReadString()
ret = ns in self.Modeller.tool_namespaces()
if ret == False:
_m.logbook_write("Unable to find a tool named " + ns)
self.SendReturnSuccess(ret)
return
def _PatternMatching(self, patternList, pattId):
tool = _MODELLER.tool('inro.emme.network_calculation.network_calculator')
def getSpec(value, selection):
return {
"result": pattId,
"expression": str(value),
"aggregation": None,
"selections": {
"transit_line": selection
},
"type": "NETWORK_CALCULATION"
}
for i in range(len(patternList)):
spec = getSpec(i, patternList[i])
try:
tool(spec, scenario= self.BaseScenario)
except ModuleError:
msg = "Emme runtime error processing line group '%s'." %(patternList[i])
_m.logbook_write(msg)
print msg
raise
msg = "Loaded group %s" %(patternList[i])
print msg
_m.logbook_write(msg)
def GetToolParameterTypes(self, tool):
# get the names of the parameters
parameterNames = inspect.getargspec(tool.__call__)[0][1:]
ret = []
for param in parameterNames:
try:
paramVar = eval("tool.__class__." + str(param))
except:
_m.logbook_write("A parameter with the name '" + param + "' does not exist in the executing EMME tool! Make sure that the EMME tool defines this attribute as a class variable.")
self.SendParameterError("A parameter with the name '" + param + "' does not exist in the executing EMME tool! Make sure that the EMME tool defines this attribute as a class variable.")
return None
typeOfParam = paramVar.type
if typeOfParam == _m.Attribute(float).type:
ret.append("float")
elif typeOfParam == _m.Attribute(int).type:
ret.append("int")
elif typeOfParam == _m.Attribute(str).type:
ret.append("string")
elif typeOfParam == _m.Attribute(bool).type:
ret.append("bool")
else:
_m.logbook_write(param + " uses a type unsupported by the ModellerBridge '" + str(typeOfParam) + "'!")
self.SendParameterError(param + " uses a type unsupported by the ModellerBridge '" + str(typeOfParam) + "'!")
return None
return ret
def _LoadCheckGtfsRoutesFile(self):
routesPath = self.GtfsFolder + "/routes.csv"
if not _path.exists(routesPath):
routesPath = self.GtfsFolder + "/routes.txt"
if not _path.exists(routesPath):
raise IOError("Folder does not contain a routes file")
with _util.CSVReader(routesPath) as reader:
for label in ['emme_id', 'emme_vehicle', 'route_id', 'route_long_name']:
if label not in reader.header:
raise IOError("Routes file does not define column '%s'" %label)
useLineNames = False
if 'emme_descr' in reader.header:
useLineNames = True
emIdSet = set()
routes = {}
for record in reader.readlines():
emmeId = record['emme_id'][:5]
if emmeId in emIdSet:
raise IOError("Route file contains duplicate id '%s'" %emmeId)
emIdSet.add(emmeId)
if useLineNames:
descr = record['emme_descr']
route = Route(record, description= descr[:17])
else:
route = Route(record)
routes[route.route_id] = route
msg = "%s routes loaded from transit feed" %len(routes)
print msg
_m.logbook_write(msg)
return routes
def _LoadNodes(self, folderName, primaryNetwork, secondaryNetwork):
with open(folderName + "/nodes.txt") as reader:
primaryNetwork.create_attribute('NODE', "twin_node", default_value=None)
secondaryNetwork.create_attribute('NODE', "twin_node", default_value=None)
reader.readline() # Toss the header
twins = 0
for line in reader.readlines():
cells = line.strip().split(',')
pNode = primaryNetwork.node(cells[0])
if pNode == None:
raise IOError("Could not find node in primary network with id='%s'" %cells[0])
if cells[1] == "null":
continue
sNode = secondaryNetwork.node(cells[1])
if sNode == None:
raise IOError("Could not find node in secondary network with id='%s'" %cells[1])
pNode['twin_node'] = sNode
sNode['twin_node'] = pNode
twins += 1
_m.logbook_write("%s nodes twinned." %twins)
self.TRACKER.completeTask()
def _WriteReport(self, log):
pb = _m.PageBuilder(title="Error log")
doc = "<br>".join(log)
pb.wrap_html(body=doc)
_m.logbook_write("Error report", value=pb.render())
def _GetNodeSet(self, network):
subwayStations = set()
goStations = set()
subStopCount = 0
goStopCount = 0
self.TRACKER.startProcess(network.element_totals['regular_nodes'])
for node in network.regular_nodes():
if node.isStop == 1:
subwayStations.add(node)
subStopCount += 1
elif node.isStop == 2 and not self.GoStationSelectorExpression:
goStations.add(node)
goStopCount += 1
self.TRACKER.completeSubtask()
self.TRACKER.completeTask()
if self.GoStationSelectorExpression:
self.TRACKER.startProcess(network.element_totals['centroids'])
for centroid in network.centroids():
if centroid['@xflag'] == 1:
goStations.add(centroid)
goStopCount += 1
self.TRACKER.completeSubtask()
self.TRACKER.completeTask()
_m.logbook_write("Found %s subway stations and %s GO train stations." %(subStopCount, goStopCount))
return (subwayStations, goStations)
def _ProcessResults(self, linkResults, alternateLinkResults):
lines = []
posts = 0
self.TRACKER.startProcess(len(linkResults))
for linkIdTuple, attributes in linkResults.iteritems():
linkId = "%s-%s" %linkIdTuple
post1 = attributes[self.CountpostAttributeId]
post2 = 0
if linkIdTuple in alternateLinkResults:
post2 = alternateLinkResults[linkIdTuple][self.AlternateCountpostAttributeId]
volau = attributes['auto_volume']
volad = attributes['additional_volume']
timau = attributes['auto_time']
data = [linkId, volau, volad, timau]
if post1:
lines.append((post1, linkId, volau, volad, timau))
posts += 1
if post2:
lines.append((post2, linkId, volau, volad, timau))
posts += 1
self.TRACKER.completeSubtask()
_m.logbook_write("Found %s countposts in network" %posts)
lines.sort()
return lines
def __call__(self):
report = ""
with _m.logbook_trace(name="Check connector speeds",
attributes={
"Scenario" : self.Scenario.id,
"self" : self.__MODELLER_NAMESPACE__}):
_m.logbook_write(
name="Checking connector speeds for scenario %s" %self.Scenario.id,
attributes={
"version":"1.00.00"})
linksChecked = 0
problemLinks = 0
for link in self.Scenario.get_network().links():
if link.i_node.is_centroid or link.j_node.is_centroid:
#link is a centroid connector
if link.data2 != 40:
report += "<br>Centroid connector link <b>" + link.id + "</b> speed should be 40, instead is " + str(link.data2)
problemLinks += 1
linksChecked += 1
_m.logbook_write(name="Report",
value=report)
self.tool_run_msg = "Tool complete. " + str(linksChecked) + " links were checked, " + str(problemLinks) + " were flagged as problems. See logbook for details."
def _AssignAggType(self, groups, aggTypeId):
tool = _MODELLER.tool('inro.emme.network_calculation.network_calculator')
def getSpec(value, selection):
return {
"result": aggTypeId,
"expression": str(value),
"aggregation": None,
"selections": {
"transit_line": selection
},
"type": "NETWORK_CALCULATION"
}
i = 0
for items in groups:
selector = groups[i][0]
groupAgg = groups[i][1]
id = groups[i][2]
i += 1
spec = getSpec(groupAgg, selector)
try:
tool(spec, scenario= self.BaseScenario)
except ModuleError:
msg = "Emme runtime error processing line group '%s'." %id
_m.logbook_write(msg)
print msg
raise
msg = "Loaded group %s: %s" %(groupAgg, id)
print msg
_m.logbook_write(msg)
def _execute(self):
with _m.logbook_trace(name="Extract transit feasibility matrix v%s" %self.version,
attributes={
"Scenario" : self.scenario.id,
"Walk Time Cutoff" : self.WalkTimeCutoff,
"Wait Time Cutoff": self.WaitTimeCutoff,
"Total Time Cutoff": self.TotalTimeCutoff,
"ModeString": self.ModeString,
"Is running from XTMF?" : str(self.isRunningFromXTMF),
"self": self.__MODELLER_NAMESPACE__}):
self._assignmentCheck()
#---1 Initialize temporary matrices for storing walk, wait, and in-vehicle times
with nested(_util.tempMatrixMANAGER(description='Temp walk time matrix'),
_util.tempMatrixMANAGER(description='Temp wait time matrix'),
_util.tempMatrixMANAGER(description='Temp ivtt matrix'))\
as (self.walkMatrix, self.waitMatrix, self.ivttMatrix):
#---2 Compute the temporary matrices
_m.logbook_write("Computing temporary matrices")
self.matrixResultTool(self._getStrategyAnalysisSpec())
#---3 Compute the final results matrix
_m.logbook_write("Computing feasibility matrix")
if EMME_VERSION >= (4,2,1):
self.matrixCalcTool(self._getMatrixCalcSpec(), self.scenario,
num_processors=self.NumberOfProcessors)
else:
self.matrixCalcTool(self._getMatrixCalcSpec(), self.scenario)
def _addWalkOnlyReverseLinksToNetwork(self):
network = self.scenario.get_network()
modes = ['w']
report = "<h2>Reverse Link Creation Report</h2>"
counter = 0
for link in network.links():
if link[self._walkableLinkFlag.id] == 0:
continue #skip links which shouldn't be one-way roads
if link.reverse_link == None:
# Reverse link doesn't exist, therefore it is a one way road
rLink = network.create_link(link.j_node.id, link.i_node.id, modes)
rLink.length = link.length
rLink.type = link.type
rLink.num_lanes = 0.0
rLink.volume_delay_func = 90
rLink.data2 = 40
rLink.data3 = 9999
report += "<br>Created link %s." %rLink
counter += 1
self.scenario.publish_network(network)
_m.logbook_write("Created %s new links in the network. Click for report." %counter, value=report)
def _demandMatrixMANAGER(self):
#Code here is executed upon entry
with _m.logbook_trace("Initializing temporary demand matrix"):
id=None
if self.demandMatrix == None:
self.demandMatrix = _util.initializeMatrix(id,
matrix_type='SCALAR',
name='trscal',
description='Scalar matrix to get transit times')
self._tracker.completeTask()
else:
cachedMatrix = self.demandMatrix
self.demandMatrix = _util.initializeMatrix(matrix_type='FULL', description="Constrained full matrix for station-to-station assignment")
_m.logbook_write("Created temporary constrained full demand matrix '%s'" %id)
try:
matrixCalcTool = _m.Modeller().tool("inro.emme.standard.matrix_calculation.matrix_calculator")
except Exception, e:
matrixCalcTool = _m.Modeller().tool("inro.emme.matrix_calculation.matrix_calculator")
self._tracker.runTool(matrixCalcTool,
self._getConstraintSpec(cachedMatrix, self.demandMatrix),
scenario=self.scenario) #TASK 1
def _WriteReport(self, issues, errCount):
print "Writing report to logbook"
pb = _m.PageBuilder(title="Network Integrity Report for Scenario %s" %self.Scenario.number,
description="Tool found %s errors in the network for scenario %s. These \
errors are likely to cause infinite loops or division-by-zero errors \
when running road or transit assignments, so it is highly recommended \
that they are fixed before running any assignments." %(errCount, self.Scenario.number))
doc = '''
<table align="center">
<tr>
<th>Element Type</th>
<th>ID</th>
<th>Problem</th>
</tr>
'''
self.TRACKER.startProcess(errCount)
for type, id, list in issues:
for problem in list:
doc += '''
<tr>
<td>{typ}</td>
<td>{id}</td>
<td>{prob}</td>
</tr>
'''.format(typ=type, id=id, prob=problem)
self.TRACKER.completeSubtask()
doc += '</table>'
pb.wrap_html(body=doc)
_m.logbook_write("%s problems found" %errCount, value= pb.render())
print "Done."
def _loadBoundaryFile(self, filename):
with _g.Shapely2ESRI(filename) as reader:
minx = float('inf')
miny = float('inf')
maxx = float('-inf')
maxy = float('-inf')
boundaries = []
for boundary in reader.readThrough():
bminx, bminy, bmaxx, bmaxy = boundary.bounds
if bminx < minx: minx = bminx
if bminy < miny: miny = bminy
if bmaxx > maxx: maxx = bmaxx
if bmaxy > maxy: maxy = bmaxy
boundaries.append(boundary)
extents = minx - 1.0, miny - 1.0, maxx + 1.0, maxy + 1.0
spatialIndex = _spindex.GridIndex(extents, 200, 200)
for ls in boundaries:
spatialIndex.insertLineString(ls)
self._Boundaries = spatialIndex
print "Loaded and indexed boundaries."
_m.logbook_write("Boundary file loaded: '%s'" %filename)
def _WriteReport(self, dataTuples):
pb = _m.PageBuilder("Network Connectivity Report")
for type, modes, fountains, sinks, orphans in dataTuples:
self._AddReportSection(pb, type, modes, fountains, sinks, orphans)
_m.logbook_write("Network connectivity report", value= pb.render())
def _LoadAltFile(self, fileNames):
altData = {}
for fileName in fileNames:
with open(fileName) as reader:
header = reader.readline()
cells = header.strip().split(self.COMMA)
emmeIdCol = cells.index('emme_id')
headwayTitle = "{:0>4.0f}".format(self.TimePeriodStart) + '_hdw'
speedTitle = "{:0>4.0f}".format(self.TimePeriodStart) + '_spd'
try:
headwayCol = cells.index(headwayTitle)
except Exception, e:
msg = "Error. No headway match for specified time period start: '%s'." %self._ParseIntTime(self.TimePeriodStart)
_m.logbook_write(msg)
print msg
try:
speedCol = cells.index(speedTitle)
except Exception, e:
msg = "Error. No speed match for specified time period start: '%s'." %self._ParseIntTime(self.TimePeriodStart)
_m.logbook_write(msg)
print msg
localAltData = {}
for num, line in enumerate(reader):
cells = line.strip().split(self.COMMA)
id = cells[emmeIdCol]
hdw = cells[headwayCol]
spd = cells[speedCol]
if id not in localAltData:
localAltData[id] = (float(hdw),float(spd))
else:
raise ValueError('Line %s has multiple entries. Please revise your alt file.' %id)
def _Execute(self):
with _m.logbook_trace(name="{classname} v{version}".format(classname=(self.__class__.__name__), version=self.version),
attributes=self._GetAtts()):
#Creating a link extra attribute to hold TTS Cordon Counts
if self.Scenario.extra_attribute('@cord') is not None:
_m.logbook_write("Deleting Previous Extra Attributes.")
self.Scenario.delete_extra_attribute('@cord')
print "Cleared previously created Cordon attribute"
_m.logbook_write("Created extra attribute for cordon counts.")
self.Scenario.create_extra_attribute('LINK', '@cord', default_value=0)
print "Created extra attribute for cordon counts"
net = self.Scenario.get_network()
with open(self.CordonTruthTable) as input:
next(input, None)
for row in input:
#process individual row
column = row.split(',')
countpostlinkid = column[1].split("-")
value = column[2].strip()
#apply the values
ourLink = net.link(countpostlinkid[0],countpostlinkid[1])
if ourLink != None:
print value
ourLink["@cord"] = int(value)
else:
print "%d - %d" %countpostlinki %value
self.Scenario.publish_network(net)
def _WriteErrorReport(self, errorTable):
h = HTML()
t = h.table()
tr = t.tr()
tr.th("Line ID")
tr.th("Error Message")
tr.th("Error Details")
for lineId, errorMsg, errorDetail in errorTable:
tr = t.tr()
tr.td(lineId)
tr.td(errorMsg)
tr.td(str(errorDetail))
pb = _m.PageBuilder(title= "Error Report")
headerText = "<b>Source Emmebank:</b> %s" %self.SourceEmmebankPath +\
"<br><b>Source Scenario:</b> %s" %self.SourceScenarioId +\
"<br><b>Target Scenario:</b> %s" %self.TargetScenario
pb.add_text_element(headerText)
pb.wrap_html(body= str(t))
_m.logbook_write("Error report", value= pb.render())
pass
def _execute(self):
with _m.logbook_trace("Load Boarding Penalties From File"):
_m.logbook_write("Opening file:'%s'" % self.BoardingsFile)
file = open(self.BoardingsFile, "r")
line = file.readline()
cells = line.split(";")
if cells.count("boarding_penalty") < 1:
raise Exception("Boardings file does not have a boarding_penalty header!")
if cells.count("filter_expression") < 1:
raise Exception("Boardings file does not have a filter_expression header!")
headers = {"penalty": cells.index("boarding_penalty"), "expression": cells.index("filter_expression")}
for line in file.readlines():
try:
cells = line.split(";")
val = float(cells[headers["penalty"]])
expr = cells[headers["expression"]]
specification = {
"result": "ut3",
"expression": "%s" % val,
"aggregation": None,
"selections": {"transit_line": expr},
"type": "NETWORK_CALCULATION",
}
self.netCalcTool(specification)
except ValueError:
continue
def run(self):
self.tool_run_msg = ""
try:
with _m.logbook_trace(name="Move Networks",
attributes=self._getAtts()):
calculator = None
try:
calculator = _m.Modeller().tool("inro.emme.network_calculation.network_calculator")
except Exception, e:
calculator = _m.Modeller().tool("inro.emme.standard.network_calculation.network_calculator")
spec = {
"result": "yi",
"expression": "yi + 4000000",
"aggregation": None,
"selections": {
"node": "all"
},
"type": "NETWORK_CALCULATION"
}
for scenario in self.Scenarios:
_m.logbook_write("Changing scenario %s" %scenario.id)
calculator(spec, scenario=scenario)
self.tool_run_msg = _m.PageBuilder.format_info("Tool complete.")
except Exception, e:
self.tool_run_msg = _m.PageBuilder.format_exception(
e, _traceback.format_exc(e))
raise
def _calculateUL1(self):
_m.logbook_write("Calculating UL1 for tuv links.")
networkCalculator = None #Use the standard tools; it is faster and more verbose.
try:
networkCalculator = _m.Modeller().tool("inro.emme.network_calculation.network_calculator")
except Exception, e:
networkCalculator = _m.Modeller().tool("inro.emme.standard.network_calculation.network_calculator")
def _WriteReport(self, lines):
with open(self.ExportFile, 'w') as writer:
writer.write(
"Countpost,Link,Auto Volume,Additional Volume,Auto Time")
for line in lines:
line = [str(c) for c in line]
writer.write("\n" + ','.join(line))
_m.logbook_write("Wrote report to %s" % self.ExportFile)
def _temp_file(self):
foldername = _tf.mkdtemp()
_m.logbook_write("Created temporary directory at '%s'" % foldername)
try:
yield foldername
finally:
_shutil.rmtree(foldername, True)
_m.logbook_write("Deleted temporary directory at '%s'" % foldername)
def _execute(self):
with _m.logbook_trace(name="{classname} v{version}".format(classname=(self.__class__.__name__), version=self.version),
attributes=self._getAtts()):
_m.logbook_write("Executing Python script from file '%s'" %self.filename)
with self._printRedirectMANAGER():
execfile(self.filename, locals())
def _TempDirectoryMANAGER(self):
foldername = _tf.mkdtemp()
_m.logbook_write("Created temporary directory at '%s'" %foldername)
try:
yield foldername
finally:
_shutil.rmtree(foldername, True)
_m.logbook_write("Deleted temporary directory at '%s'" %foldername)
def EnsureModellerToolExists(self, macroName):
for i in range(1, 10):
if macroName in self.Modeller.tool_namespaces():
return True
time.sleep(1)
_m.logbook_write("A tool with the following namespace could not be found: %s" % macroName)
self.SendToolDoesNotExistError(macroName)
return False
def __call__(self):
report = "<h3>Check Link Lanes</h3>\
<table><tr>\
<th>I Node</th><th>J Node</th><th>Link Class</th><th>Req'd Lanes</th><th>Actual Lanes</th>\
</tr>"
with _m.logbook_trace(name="Check Link lanes",
attributes={
"Scenario": self.Scenario.id,
"self": self.__MODELLER_NAMESPACE__
}):
_m.logbook_write(name="Checking link lanes for scenario %s" %
self.Scenario.id,
attributes={"version": "1.0.1"})
problemLinks = 0
for link in self.Scenario.get_network().links():
if link.i_node.is_centroid or link.j_node.is_centroid:
#link is a centroid connector
if link.num_lanes != 2.0:
report += "<tr><td>{inode}</td><td>{jnode}</td><td>Centroid connector\
</td><td>2.0</td><td>{lanes}</td></tr>".format(
inode=link.i_node.id,
jnode=link.j_node.id,
lanes=str(link.num_lanes))
#report += "<br>Centroid connector link <b>" + link.id + "</b> should have 2.0 lanes, instead has " + str(link.num_lanes)
problemLinks += 1
elif self.linkIsTransitOnly(link):
#link is exclusive ROW
if link.num_lanes != 0.0:
report += "<tr><td>{inode}</td><td>{jnode}</td><td>Exclusive transit link\
</td><td>0.0</td><td>{lanes}</td></tr>".format(
inode=link.i_node.id,
jnode=link.j_node.id,
lanes=str(link.num_lanes))
#report += "<br>Exclusive transit link <b>" + link.id + "</b> should have 0.0 lanes, instead has " + str(link.num_lanes)
problemLinks += 1
elif self.linkHasMode(link, "t") or self.linkHasMode(
link, "u"):
#transfer link
if link.num_lanes != 0.0:
report += "<tr><td>{inode}</td><td>{jnode}</td><td>Transfer link\
</td><td>0.0</td><td>{lanes}</td></tr>".format(
inode=link.i_node.id,
jnode=link.j_node.id,
lanes=str(link.num_lanes))
report += "<br>Transfer link <b>" + link.id + "</b> should have 0.0 lanes, instead has " + str(
link.num_lanes)
problemLinks += 1
report += "</table>"
_m.logbook_write(name="Report", value=report)
self.tool_run_msg = _m.PageBuilder.format_info(
"%s links were flagged as problems. See logbook for details." %
problemLinks)
def _ProcessTransitLines(self, network, start, end, altData):
bounds = _util.FloatRange(0.01, 1000.0)
toDelete = set()
if altData:
for k, v in altData.items(): #check if any headways or speeds are zero. Allow those lines to be deletable
if v[0] == 0 or v[1] == 0:
del altData[k]
#if v[0] == 9999: #prep an unused line for deletion
# toDelete.add(k)
doNotDelete = altData.keys()
else:
doNotDelete = None
self.TRACKER.startProcess(network.element_totals['transit_lines'])
for line in network.transit_lines():
#Pick aggregation type for given line
if line.aggtype == 'n':
aggregator = naiveAggregation
elif line.aggtype == 'a':
aggregator = averageAggregation
elif self.DefaultAgg == 'n':
aggregator = naiveAggregation
_m.logbook_write("Default aggregation was used for line %s" %(line.id))
else:
aggregator = averageAggregation
_m.logbook_write("Default aggregation was used for line %s" %(line.id))
if not line.trips: #Line trips list is empty or None
if doNotDelete:
if line.id not in doNotDelete: #don't delete lines whose headways we wish to manually set
toDelete.add(line.id)
elif line.id not in toDelete:
toDelete.add(line.id)
self.TRACKER.completeSubtask()
continue
#Calc line headway
departures = [dep for dep, arr in line.trips]
departures.sort()
headway = aggregator(departures, start, end) / 60.0 #Convert from seconds to minutes
if not headway in bounds: print "%s: %s" %(line.id, headway)
line.headway = headway
#Calc line speed
sumTimes = 0
for dep, arr in line.trips: sumTimes += arr - dep
avgTime = sumTimes / len(line.trips) / 3600.0 #Convert from seconds to hours
length = sum([seg.link.length for seg in line.segments()]) #Given in km
speed = length / avgTime #km/hr
if not speed in bounds: print "%s: %s" %(line.id, speed)
line.speed = speed
self.TRACKER.completeSubtask()
for id in toDelete:
network.delete_transit_line(id)
self.TRACKER.completeTask()
def _CalcNetworkSize(self, network):
baseSurfaceNodes = []
baseStationNodes = []
for node in network.regular_nodes():
if node.role == 1: baseSurfaceNodes.append(node)
elif node.role == 2: baseStationNodes.append(node)
baseNodes = network.element_totals['regular_nodes']
baseLinks = network.element_totals['links']
nVirtualSurfaceNodes, nBaseConnectorLinks = 0, 0
self.TRACKER.startProcess(len(baseSurfaceNodes) + len(baseStationNodes))
for node in baseSurfaceNodes:
addNodes, addLinks = self._CalcSurfaceNode(node)
nVirtualSurfaceNodes += addNodes
nBaseConnectorLinks += addLinks
self.TRACKER.completeSubtask()
print "%s virtual road nodes" %nVirtualSurfaceNodes
print "%s access links to virtual road nodes" %nBaseConnectorLinks
nVirtualStationNodes, nStationConnectorLinks = 0, 0
for node in baseStationNodes:
addNodes, addLinks = self._CalcStationNode(node)
nVirtualStationNodes += addNodes
nStationConnectorLinks += addLinks
self.TRACKER.completeSubtask()
self.TRACKER.completeTask()
print "%s virtual station nodes" %nVirtualStationNodes
print "%s access links to virtual station nodes" %nStationConnectorLinks
nConnectorLinks = 0
for node in baseStationNodes + baseSurfaceNodes:
nConnectorLinks += self._CalcStationToSurfaceConnectors(node)
print "%s road-to-transit connector links." %nConnectorLinks
inVehicleLinks = 0
network.create_attribute('LINK', 'copies', None)
for link in network.links():
link.copies = set()
for line in network.transit_lines():
group = line.group
for segment in line.segments():
segment.link.copies.add(group)
for link in network.links():
if link.role == 3: #Is existing hyper link
inVehicleLinks += max(len(link.copies) - 1, 0)
else:
inVehicleLinks += len(link.copies)
print "%s in-vehicle links" %inVehicleLinks
totalNodes = baseNodes + nVirtualStationNodes + nVirtualSurfaceNodes
totalLinks = baseLinks + nBaseConnectorLinks + nConnectorLinks + nStationConnectorLinks + inVehicleLinks
_m.logbook_write("The hyper network will contain exactly %s total nodes and approximately %s links" %(totalNodes, totalLinks))
return totalNodes, totalLinks
def _execute(self):
with _m.logbook_trace(name="{classname} v{version}".format(
classname=self.__class__.__name__, version=self.version),
attributes=self._get_logbook_attributes()):
if _bank.scenario(
self.scenario_Id
) is not None and not self.overwrite_scenario_flag:
raise IOError(
"Scenario %s exists and overwrite flag is set to false." %
self.scenario_Id)
self._components.reset(
) # Clear any held-over contents from previous run
with _zipfile.ZipFile(self.network_package_file
) as zf, self._temp_file() as temp_folder:
self._check_network_package(zf) # Check the file format.
if _bank.scenario(self.scenario_Id) is not None:
if not self.overwrite_scenario_flag:
raise IOError("Scenario %s already exists." %
self.scenario_Id)
sc = _bank.scenario(self.scenario_Id)
if sc.modify_protected or sc.delete_protected:
raise IOError(
"Scenario %s is protected against modifications" %
self.scenario_Id)
_bank.delete_scenario(self.scenario_Id)
scenario = _bank.create_scenario(self.scenario_Id)
scenario.title = self.scenario_description
_m.logbook_write("Created new scenario %s" % self.scenario_Id)
self.TRACKER.completeTask()
self._batchin_modes(scenario, temp_folder, zf)
self._batchin_vehicles(scenario, temp_folder, zf)
self._batchin_base(scenario, temp_folder, zf)
self._batchin_link_shapes(scenario, temp_folder, zf)
self._batchin_lines(scenario, temp_folder, zf)
self._batchin_turns(scenario, temp_folder, zf)
if self._components.traffic_results_files is not None:
self._batchin_traffic_results(scenario, temp_folder, zf)
if self._components.transit_results_files is not None:
self._batchin_transit_results(scenario, temp_folder, zf)
if self._components.attribute_header_file is not None:
self._batchin_extra_attributes(scenario, temp_folder, zf)
self.TRACKER.completeTask()
if self._components.functions_file is not None:
self._batchin_functions(temp_folder, zf)
self.TRACKER.completeTask()
def _calculateUL1(self):
_m.logbook_write("Calculating UL1 for tuv links.")
networkCalculator = None #Use the standard tools; it is faster and more verbose.
try:
networkCalculator = _m.Modeller().tool(
"inro.emme.network_calculation.network_calculator")
except Exception, e:
networkCalculator = _m.Modeller().tool(
"inro.emme.standard.network_calculation.network_calculator")
def EnsureModellerToolExists(self, macroName):
for i in range(1, 10):
if macroName in self.Modeller.tool_namespaces():
return True
time.sleep(1)
_m.logbook_write(
"A tool with the following namespace could not be found: %s" %
macroName)
self.SendToolDoesNotExistError(macroName)
return False
def _execute(self):
with _m.logbook_trace(name="{classname} v{version}".format(
classname=(self.__class__.__name__), version=self.version),
attributes=self._getAtts()):
_m.logbook_write("Executing Python script from file '%s'" %
self.filename)
with self._printRedirectMANAGER():
execfile(self.filename, locals())
def _tempDirectoryMANAGER(self):
databankDirectory = path.dirname(_MODELLER.emmebank.path)
tempDirectory = _tf.mkdtemp(dir=databankDirectory)
_m.logbook_write("Created temp directory at %s" % tempDirectory)
try:
yield tempDirectory
finally:
_shutil.rmtree(tempDirectory, True)
_m.logbook_write("Deleted temp directory at %s" % tempDirectory)
def _ChangeWalkSpeed4p1(self):
partialNetwork = self.Scenario.get_partial_network(['MODE'], True)
for mode in partialNetwork.modes():
if mode.type != 'AUX_TRANSIT':
continue
mode.speed = self.WalkSpeed
_m.logbook_write('Changed mode %s' % mode.id)
baton = partialNetwork.get_attribute_values('MODE', ['speed'])
self.Scenario.set_attribute_values('MODE', ['speed'], baton)
def _ConcatenateLines(self, network, lineSet):
try:
newId = self._GetNewId(lineSet[0], network)
_netedit.lineConcatenator(network, lineSet, newId)
_m.logbook_write("Line set %s concatenated" % (lineSet))
except Exception:
self.FailureFlag = True
_m.logbook_write("This line set is not valid: %s" % (lineSet))
return None
return newId
def _execute(self):
with _m.logbook_trace(name="Flag premium buses v%s" % self.version,
attributes={
"Scenario":
str(self.scenario.id),
"Flag GO Buses":
self.FlagGO,
"Flag TTC Premium Buses":
self.FlagPremTTC,
"Flag VIVA Buses":
self.FlagVIVA,
"Flag ZUM":
self.FlagZum,
"Is running from XTMF?":
str(self.isRunningFromXTMF),
"self":
self.__MODELLER_NAMESPACE__
}):
#---0. Set up all Emme tools and data structures
try:
self.netWorkCalculationTool = _m.Modeller().tool(
"inro.emme.standard.network_calculation.network_calculator"
)
_m.logbook_write(name="Emme 3.4.2 Tool Names used")
except Exception as e:
self.netWorkCalculationTool = _m.Modeller().tool(
"inro.emme.network_calculation.network_calculator")
_m.logbook_write(name="Emme 4.0.3 Tool Names used")
#---Initialize the flag attribute
self._initFlagAttribute()
self.counter = 0
#---Get network
network = self.scenario.get_network()
#---Set up the list of functions
functions = []
if self.FlagGO:
functions.append(self._flagGoBuses)
if self.FlagPremTTC:
functions.append(self._flagTTCPremiumBuses)
if self.FlagVIVA:
functions.append(self._flagVIVABuses)
if self.FlagZum:
functions.append(self._flagZumBuses)
#---Execute
for line in network.transit_lines():
for f in functions:
f(line)
#---Publish
self.scenario.publish_network(network)
def __call__(self):
report = ""
with _m.logbook_trace(name="Check Link Lengths",
attributes={
"Scenario" : self.Scenario.id,
"Tolerance" : self.Error,
"self" : self.__MODELLER_NAMESPACE__}):
_m.logbook_write(
name="Checking link lengths for scenario %s" %self.Scenario.id,
attributes={
"version":"1.00.00"})
linksChecked = 0
problemLinks = 0
for link in self.Scenario.get_network().links():
i = link.i_node
j = link.j_node
type = "link"
distance = 0
if self.linkHasMode(link, "t"):
distance = 0.10
type = "t-link"
elif self.linkHasMode(link, "u"):
distance = 0.10
type = "u-link"
elif (i.number > 900000 and j.number < 900000) or (j.number > 900000 and i < 900000):
# HOV ramp
distance = 0.0
type = "HOV ramp link"
else:
d = math.sqrt((i.x - j.x)**2 + (i.y - j.y)**2)
distance = (int(d /10) / 100.0) # Gives distance in km, truncated to 2 decimal places
length = (int(link.length * 100) / 100.0) #Eliminates rounding error in the link length; length is in km, truncated to 2 decimal places
diff = math.fabs(distance - length)
if diff > self.Error:
problemLinks += 1
# TODO: Have a better report, maybe in a table, which better organizes the results.
report += "<p><b>Distance error:</b><ul><li><em>link id</em>: " + link.id +"</li><li><em>type</em>: " \
+ type + "</li><li><em>distance</em>: " + str(distance) + "</li><li><em>length</em>: " + str(length) + "</li></ul></p>"
linksChecked += 1
_m.logbook_write(name="Report",
value=report)
#test = """<select><option value="option1">1</option><option value="option2">2</option><option value="option3">3</option></select>"""
#_m.logbook_write(name="Test Report", value=test)
self.tool_run_msg = "Tool complete. " + str(linksChecked) + " links were checked, " + str(problemLinks) + " were flagged as problems. See logbook for details."
def _WriteTableReport(self, groupings):
pb = _m.PageBuilder()
html = "<table><tr><th>Group<th>Code</tr>"
for value, selector, description in groupings:
html += "<tr><td>%s<td>%s</tr>" %(description, value)
html += "</table>"
pb.wrap_html(title="Grouping Table", body=html, note="Codes saved into transit line attribute %s" %self.AttributeId)
_m.logbook_write("Value Table", value=pb.render())
def _replaceVehicle(self, oldVehicle, newVehicle):
if not oldVehicle in self.vm:
return
for line in self.vm[oldVehicle]:
line.vehicle = newVehicle
_m.logbook_write("Changed {0} line(s) with vehicle \
{1} to use {2}.".format(len(self.vm[oldVehicle]),
oldVehicle.number,
newVehicle.number))
def _lineAttributeMANAGER(self):
# Code here is executed upon entry {
self.Scenario.create_extra_attribute('TRANSIT_LINE', '@tlf1')
_m.logbook_write("Created temporary attribute @tlf1")
# }
try:
yield '@tlf1'
finally:
self.Scenario.delete_extra_attribute('@tlf1')
_m.logbook_write("Deleted temporary attribute @tlf1")
def _Execute(self):
with _m.logbook_trace(name="{classname} v{version}".format(
classname=(self.__class__.__name__), version=self.version),
attributes=self._GetAtts()):
network = self.BaseScenario.get_network()
self.TRACKER.completeTask()
print("Loaded network")
start = self._ParseIntTime(self.TimePeriodStart)
end = self._ParseIntTime(self.TimePeriodEnd)
badIdSet = self._LoadServiceTable(network, start, end).union(
self._LoadAggTypeSelect(network))
self.TRACKER.completeTask()
print("Loaded service table")
if len(badIdSet) > 0:
print(
"%s transit line IDs were not found in the network and were skipped."
% len(badIdSet))
pb = _m.PageBuilder("Transit line IDs not in network")
pb.add_text_element(
"<b>The following line IDs were not found in the network:</b>"
)
for id in badIdSet:
pb.add_text_element(id)
_m.logbook_write(
"Some IDs were not found in the network. Click for details.",
value=pb.render())
if len(self.InputFiles) <= 0:
self._ProcessTransitLines(network, start, end, None)
else:
if self.AlternativeDataFile:
altData = self._LoadAltFile(self.InputFiles)
else:
altData = None
self._ProcessTransitLines(network, start, end, altData)
if altData:
self._ProcessAltLines(network, altData)
print("Done processing transit lines")
newScenario = _MODELLER.emmebank.copy_scenario(
self.BaseScenario.id, self.NewScenarioNumber)
newScenario.title = self.NewScenarioDescription
print("Publishing network")
network.delete_attribute('TRANSIT_LINE', 'trips')
network.delete_attribute('TRANSIT_LINE', 'aggtype')
newScenario.publish_network(network)
def update_data(self):
# dialog = self.dialog
acceptedChanges = self.dialog.getFunctionsToChange()
for fid, expression in acceptedChanges.iteritems():
func = _MODELLER.emmebank.function(fid)
oldExpression = func.expression
func.expression = expression
modifiedFunctions[fid] = oldExpression
with _m.logbook_trace("Modified function %s" % fid.upper()):
_m.logbook_write("Old expression: %s" % oldExpression)
_m.logbook_write("New expression: %s" % expression)
def _execute(self):
with _m.logbook_trace(name="Legacy Fare Based Transit Assignment v%s" %self.version,
attributes=self._getAtts()):
self._calculateFareFactor()
with nested(self._demandMatrixMANAGER(), self._walkLinksMANAGER(), self._transitFunctionsMANAGER()):
self._calculateUL1()
_m.logbook_write(name="Running extended transit assignment")
self.transitAssignmentTool(self._setUpAssignment(), # Specification
self.scenario, # Scenario
self.UseAdditiveDemand) # Use additional volumes
def run_proc(self, name, arguments, log_message, capture_output=False):
path = _join(self._path, "bin", name)
if not os.path.exists(path):
raise Exception("No command / batch file '%s'" % path)
command = path + " " + " ".join([str(x) for x in arguments])
attrs = {"command": command, "name": name, "arguments": arguments}
with _m.logbook_trace(log_message, attributes=attrs):
if capture_output and self._log_level != "NO_EXTERNAL_REPORTS":
report = _m.PageBuilder(title="Process run %s" % name)
report.add_html(
'Command:<br><br><div class="preformat">%s</div><br>' %
command)
# temporary file to capture output error messages generated by Java
err_file_ref, err_file_path = _tempfile.mkstemp(suffix='.log')
err_file = os.fdopen(err_file_ref, "w")
try:
output = _subprocess.check_output(command,
stderr=err_file,
cwd=self._path,
shell=True)
report.add_html(
'Output:<br><br><div class="preformat">%s</div>' %
output)
except _subprocess.CalledProcessError as error:
report.add_html(
'Output:<br><br><div class="preformat">%s</div>' %
error.output)
raise
finally:
err_file.close()
with open(err_file_path, 'r') as f:
error_msg = f.read()
os.remove(err_file_path)
if error_msg:
report.add_html(
'Error message(s):<br><br><div class="preformat">%s</div>'
% error_msg)
try:
# No raise on writing report error
# due to observed issue with runs generating reports which cause
# errors when logged
_m.logbook_write("Process run %s report" % name,
report.render())
except Exception as error:
print _time.strftime("%Y-%M-%d %H:%m:%S")
print "Error writing report '%s' to logbook" % name
print error
print _traceback.format_exc(error)
if self._log_level == "DISABLE_ON_ERROR":
_m.logbook_level(_m.LogbookLevel.NONE)
else:
_subprocess.check_call(command, cwd=self._path, shell=True)
def _WriteReport(self, lines):
with open(self.ExportFile, 'wb') as csv_file:
writer = csv.writer(csv_file)
#writer.writerow("Countpost,Link,Auto Volume,Additional Volume,Auto Time")
#for line in lines:
# line = [str(c) for c in line]
# writer.writerow(line)'''
writer.writerow(["Countpost", "Auto Volume"])
for post in sorted(lines.iterkeys()):
post = int(post)
volau = float(lines[post])
writer.writerow([post, volau])
_m.logbook_write("Wrote report to %s" % self.ExportFile)
def _walkableLinkFlagAttributeMANAGER(self):
#Code here is executed upon entry
_m.logbook_write("Creating temporary link flag attribute.")
self._walkableLinkFlag = self.scenario.create_extra_attribute(
'LINK', '@flg1w', default_value=0)
try:
yield
# Code here is executed upon clean exit
finally:
# Code here is executed in all cases.
_m.logbook_write("Deleting temporary link flag attribute.")
self.scenario.delete_extra_attribute(self._walkableLinkFlag)
def _changeVehicleProperties(self,
vehicle,
description="",
aeq=0.0,
scap=0,
tcap=0):
if description != "" and description != None:
vehicle.description = description
_m.logbook_write("Description = '%s'" % description)
if aeq != 0.0:
vehicle.auto_equivalent = aeq
_m.logbook_write("Auto equivalence = %s" % aeq)
if scap != 0:
vehicle.seated_capacity = scap
_m.logbook_write("Seated capacity = %s" % scap)
if tcap < scap:
tcap = scap
if tcap != 0:
vehicle.total_capacity = tcap
_m.logbook_write("Total capacity = %s" % tcap)
def _costAttributeMANAGER(self):
#Code here is executed upon entry
attributeCreated = False
costAttribute = self.Scenario.extra_attribute('@lkcst')
if costAttribute == None:
#@lkcst hasn't been defined
_m.logbook_write(
"Creating temporary link cost attribute '@lkcst'.")
costAttribute = self.Scenario.create_extra_attribute(
'LINK', '@lkcst', default_value=0)
attributeCreated = True
elif self.Scenario.extra_attribute('@lkcst').type != 'LINK':
#for some reason '@lkcst' exists, but is not a link attribute
_m.logbook_write(
"Creating temporary link cost attribute '@lcost'.")
costAttribute = self.Scenario.create_extra_attribute(
'LINK', '@lcst2', default_value=0)
attributeCreated = True
if not attributeCreated:
costAttribute.initialize()
_m.logbook_write("Initialized link cost attribute to 0.")
try:
yield costAttribute
# Code here is executed upon clean exit
finally:
# Code here is executed in all cases.
if attributeCreated:
_m.logbook_write("Deleting temporary link cost attribute.")
self.Scenario.delete_extra_attribute(costAttribute.id)
def _constraintMatrixMANAGER(self):
#Code here is executed upon entry
_m.logbook_write("Initializing temporary constraint matrix.")
self.constraintMatrix = _util.initializeMatrix(matrix_type='FULL')
try:
yield
# Code here is executed upon clean exit
finally:
# Code here is executed in all cases.
_m.logbook_write("Deleting temporary constraint matrix %s." %
self.constraintMatrix.id)
self.databank.delete_matrix(self.constraintMatrix.id)
def _vdfAttributeMANAGER(self):
#Code here is executed upon entry
att = None
att = self.scenario.extra_attribute("@vdf")
if att is None:
att = self.scenario.create_extra_attribute('LINK',
'@vdf',
default_value=0)
_m.logbook_write(
"Created temporary link '%s' attribute to store new VDFs." %
att.id)
else:
att.initialize()
_m.logbook_write("Initialized attribute '%s'." % att.id)
try:
yield att
finally:
# Code here is executed in all cases.
if self.makeChangesPermanent:
i = att.id
self.scenario.delete_extra_attribute(att)
_m.logbook_write("Deleted temporary link attribute '%s'" % i)
else:
_m.logbook_write(
"Temporary link attribute '%s' made permanent." % att.id)
def _WriteMainReport(self, reversedLines):
h = HTML()
t = h.table()
tr = t.tr()
tr.th('Original ID')
tr.th('Reversed ID')
for originalId, newId in reversedLines:
tr = t.tr()
tr.td(originalId)
tr.td(newId)
pb = _m.PageBuilder(title= "Reversed Lines Report")
pb.wrap_html(body= str(t))
_m.logbook_write("Reversed lines report", value= pb.render())
def _stnNodeFlagAttributeMANAGER(self):
#Code here is executed upon entry
_m.logbook_write("Creating temporary station centroid flag attribute.")
self._stnNodeFlag = self.scenario.create_extra_attribute(
'NODE', '@stn', default_value=0)
try:
yield
# Code here is executed upon clean exit
finally:
# Code here is executed in all cases.
_m.logbook_write(
"Deleting temporary station centroid flag attribute.")
self.scenario.delete_extra_attribute(self._stnNodeFlag)
def _WriteErrorReport(self, linesMissingInNetwork):
h = HTML()
t = h.table()
tr = t.tr()
tr.th("Line ID")
for id in linesMissingInNetwork:
tr = t.tr()
tr.td(str(id))
pb = _m.PageBuilder(title="Lines not in network report")
pb.wrap_html("Lines references in file but not in network",
body=str(t))
_m.logbook_write("Error report", value=pb.render())
def _LaunchGUI(self, conflicts, modifiedFunctions):
dialog = FunctionConflictDialog(conflicts)
result = dialog.exec_()
if result == dialog.Accepted:
acceptedChanges = dialog.getFunctionsToChange()
for fid, expression in acceptedChanges.iteritems():
func = _MODELLER.emmebank.function(fid)
oldExpression = func.expression
func.expression = expression
modifiedFunctions[fid] = oldExpression
with _m.logbook_trace("Modified function %s" % fid.upper()):
_m.logbook_write("Old expression: %s" % oldExpression)
_m.logbook_write("New expression: %s" % expression)
dialog.deleteLater()
def move_files(self, file_names, from_dir, to_dir):
with _m.logbook_trace("Move files %s" % ", ".join(file_names)):
if not os.path.exists(to_dir):
os.mkdir(to_dir)
for file_name in file_names:
all_files = _glob.glob(_join(from_dir, file_name))
for path in all_files:
try:
dst_file = _join(to_dir, os.path.basename(path))
if os.path.exists(dst_file):
os.remove(dst_file)
_shutil.move(path, to_dir)
except Exception as error:
_m.logbook_write(
"Error moving file %s" % path,
{"error": _traceback.format_exc(error)})
