def saveNodesSHP(self, targetPath, isFake=False):
"Save nodes to a shapefile"
geometry_store.save(
store.replaceFileExtension(targetPath, "shp"),
self.getProj4(),
[shapely.geometry.asShape(node) for node in self.cycleNodes(isFake)],
)
def saveSubnetsSHP(self, targetPath):
'Save subnets to a shapefile'
# If there are no subnets,
if not self.countSubnets():
return
# Save
geometry_store.save(store.replaceFileExtension(targetPath, 'shp'), self.getProj4(), [shapely.geometry.asShape(x) for x in self.cycleSubnets()])
def saveMetricsCSV(self, targetPath, metricModel, headerType=VS.HEADER_TYPE_SECTION_OPTION):
'Save node-level metrics in CSV format'
# Make sure that nodes exist
if not self.countNodes():
return
# Prepare column headers in order
# Use the 1st node's input to get the "pass-through" fields
node = self.cycleNodes().next()
nodeInput = node.input
headerPacks = [('', key) for key in sorted(nodeInput)]
# get the section/option values in order from the model
# and append to the headerPacks
# Note: metricModel.VariableStore.variableClasses should have all the
# the variableClasses associated with the model as long as
# metricModel.VariableStore() has been called.
baseVars = metricModel.VariableStore.variableClasses
baseVarHeaders = [(var.section, var.option) for var in
sorted(baseVars, key=lambda v: (v.section, v.option))]
headerPacks.extend(baseVarHeaders)
headerPacksToNames = VS.getFieldNamesForHeaderPacks(metricModel,
headerPacks, headerType)
csvWriter = csv.writer(open(store.replaceFileExtension(targetPath, 'csv'), 'wb'))
csvWriter.writerow(['PROJ.4 ' + self.getProj4()])
csvWriter.writerow([headerPacksToNames[(section, option)] for
section, option in headerPacks])
# csvWriter.writerow(['%s > %s' % (section.capitalize(), option.capitalize()) if section else option.capitalize() for section, option in headerPacks])
# For each node,
for node in self.cycleNodes():
# Write row
csvWriter.writerow([node.output.get(section, {}).get(option, '') if section else node.input.get(option, '') for section, option in headerPacks])
def saveMetricsCSV(self, targetPath, metricModel):
"Save node-level metrics in CSV format"
# Make sure that nodes exist
if not self.countNodes():
return
# Prepare column headers in order
node = self.cycleNodes().next()
nodeInput = node.input
nodeOutput = node.output
headerPacks = [("", key) for key in sorted(nodeInput)]
for section, valueByOption in sorted(nodeOutput.iteritems(), key=lambda x: metricModel.sections.index(x[0])):
for option in sorted(valueByOption):
headerPacks.append((section, option))
# Prepare
csvWriter = csv.writer(open(store.replaceFileExtension(targetPath, "csv"), "wb"))
csvWriter.writerow(["PROJ.4 " + self.getProj4()])
csvWriter.writerow(
[
"%s > %s" % (section.capitalize(), option.capitalize()) if section else option.capitalize()
for section, option in headerPacks
]
)
# For each node,
for node in self.cycleNodes():
# Write row
csvWriter.writerow(
[
node.output.get(section, {}).get(option, "") if section else node.input.get(option, "")
for section, option in headerPacks
]
)
def saveNodesSHP(self, targetPath, isFake=False):
'Save nodes to a shapefile'
geometry_store.save(store.replaceFileExtension(targetPath, 'shp'),
self.getProj4(), [
shapely.geometry.asShape(node)
for node in self.cycleNodes(isFake)
])
def saveSubnetsSHP(self, targetPath):
'Save subnets to a shapefile'
# If there are no subnets,
if not self.countSubnets():
return
# Save
geometry_store.save(
store.replaceFileExtension(targetPath, 'shp'), self.getProj4(),
[shapely.geometry.asShape(x) for x in self.cycleSubnets()])
def saveMetricsConfigurationCSV(targetPath, valueByOptionBySection):
'Save scenario-level INPUT metrics as a CSV file'
csvFile = open(store.replaceFileExtension(targetPath, 'csv'), 'wt')
csvWriter = csv.writer(csvFile)
# Note, this assumes section, option, value tuples returned by flatten function
for section, option, value in sorted(util.flatten_to_tuples(valueByOptionBySection)):
csvWriter.writerow([section, option, value])
csvFile.close()
def saveMetricsConfigurationCSV(targetPath, valueByOptionBySection):
'Save scenario-level INPUT metrics as a CSV file'
csvFile = open(store.replaceFileExtension(targetPath, 'csv'), 'wt')
csvWriter = csv.writer(csvFile)
# Note, this assumes section, option, value tuples returned by flatten function
for section, option, value in sorted(
util.flatten_to_tuples(valueByOptionBySection)):
csvWriter.writerow([section, option, value])
csvFile.close()
def saveSegmentsSHP(self, targetPath, is_existing=None):
# If there are no segments,
if not self.countSegments(is_existing):
return
# Save
return geometry_store.save(
store.replaceFileExtension(targetPath, 'shp'), self.getProj4(), [
shapely.geometry.asShape(x)
for x in self.cycleSegments(is_existing)
])
def saveSegmentsSHP(self, targetPath, is_existing=None):
# If there are no segments,
if not self.countSegments(is_existing):
return
# Save
return geometry_store.save(
store.replaceFileExtension(targetPath, "shp"),
self.getProj4(),
[shapely.geometry.asShape(x) for x in self.cycleSegments(is_existing)],
)
def saveMetricsCSV(targetPath, metricModel, valueByOptionBySection):
'Save scenario-level metrics as a CSV file'
# Initialize
csvFile = open(store.replaceFileExtension(targetPath, 'csv'), 'wt')
csvWriter = csv.writer(csvFile)
vs = metricModel.VariableStore
for variableClass in sorted(itertools.chain(vs.aggregateClasses,
vs.summaryClasses),
key=lambda x: (x.__module__, x.__name__)):
section = variableClass.section
option = variableClass.option
value = valueByOptionBySection[section][option]
csvWriter.writerow([section, option, value])
csvFile.close()
def saveMetricsCSV(targetPath, metricModel, valueByOptionBySection):
"Save scenario-level metrics as a CSV file"
# Initialize
csvFile = open(store.replaceFileExtension(targetPath, "csv"), "wt")
csvWriter = csv.writer(csvFile)
vs = metricModel.VariableStore
for variableClass in sorted(
itertools.chain(vs.aggregateClasses, vs.summaryClasses), key=lambda x: (x.__module__, x.__name__)
):
section = variableClass.section
option = variableClass.option
value = valueByOptionBySection[section][option]
csvWriter.writerow([section, option, value])
csvFile.close()
def saveNodesCSV(self, targetPath, isFake=False):
'Save nodes to a csv'
# Initialize
node = self.cycleNodes(isFake).next()
csvWriter = csv.writer(open(store.replaceFileExtension(targetPath, 'csv'), 'wb'))
# Write spatial reference
csvWriter.writerow(['PROJ.4 ' + self.getProj4()])
# Write column headers
customHeaders = sorted(set(node.input) - set(['name', 'x', 'y']))
csvWriter.writerow(['Name', 'X', 'Y'] + [x.capitalize() for x in customHeaders])
# For each node,
for node in self.cycleNodes(isFake):
# Write row
csvWriter.writerow([node.input.get('name', ''), node.getX(), node.getY()] + [node.input.get(x, '') for x in customHeaders])
def __init__(self, datasetPath, proj4=None):
# Connect
datasetPath = store.replaceFileExtension(datasetPath, "db")
engine = sa.create_engine("sqlite:///" + datasetPath, echo=self.debug)
metadata.create_all(engine)
# Set
self.session = orm.sessionmaker(bind=engine)()
self.datasetPath = datasetPath
# Set proj4
if proj4:
self.session.execute(spatial_references_table.delete())
self.session.add(SpatialReference(proj4))
self.session.commit()
self.proj4 = str(self.session.query(SpatialReference).first().proj4)
self.transform_point = geometry_store.get_transform_point(self.proj4)
def __init__(self, datasetPath, proj4=None):
# Connect
datasetPath = store.replaceFileExtension(datasetPath, 'db')
engine = sa.create_engine('sqlite:///' + datasetPath, echo=self.debug)
metadata.create_all(engine)
# Set
self.session = orm.sessionmaker(bind=engine)()
self.datasetPath = datasetPath
# Set proj4
if proj4:
self.session.execute(spatial_references_table.delete())
self.session.add(SpatialReference(proj4))
self.session.commit()
self.proj4 = str(self.session.query(SpatialReference).first().proj4)
self.transform_point = geometry_store.get_transform_point(self.proj4)
def create(targetPath, sourcePath):
"Import the sourcePath to create the dataset"
# Initialize
digestByExtension = {".csv": digestNodesFromCSV, ".zip": digestNodesFromZIP}
# Prepare
sourceExtension = os.path.splitext(sourcePath)[1].lower()
if sourceExtension not in digestByExtension:
raise DatasetError("Only the following formats are currently supported: " + " ".join(digestByExtension))
# Import
proj4, nodePacks = digestByExtension[sourceExtension](sourcePath)
# Save
store.removeSafely(store.replaceFileExtension(targetPath, "db"))
dataset = Store(targetPath, proj4)
dataset.addNodes(nodePacks)
# Return
return dataset
def saveNodesCSV(self, targetPath, isFake=False):
"Save nodes to a csv"
# Initialize
node = self.cycleNodes(isFake).next()
csvWriter = csv.writer(open(store.replaceFileExtension(targetPath, "csv"), "wb"))
# Write spatial reference
csvWriter.writerow(["PROJ.4 " + self.getProj4()])
# Write column headers
customHeaders = sorted(set(node.input) - set(["name", "x", "y"]))
csvWriter.writerow(["Name", "X", "Y"] + [x.capitalize() for x in customHeaders])
# For each node,
for node in self.cycleNodes(isFake):
# Write row
csvWriter.writerow(
[node.input.get("name", ""), node.getX(), node.getY()] + [node.input.get(x, "") for x in customHeaders]
)
def saveMetricsCSV(self,
targetPath,
metricModel,
headerType=VS.HEADER_TYPE_SECTION_OPTION):
'Save node-level metrics in CSV format'
# Make sure that nodes exist
if not self.countNodes():
return
# Prepare column headers in order
# Use the 1st node's input to get the "pass-through" fields
node = self.cycleNodes().next()
nodeInput = node.input
headerPacks = [('', key) for key in sorted(nodeInput)]
# get the section/option values in order from the model
# and append to the headerPacks
# Note: metricModel.VariableStore.variableClasses should have all the
# the variableClasses associated with the model as long as
# metricModel.VariableStore() has been called.
baseVars = metricModel.VariableStore.variableClasses
baseVarHeaders = [
(var.section, var.option)
for var in sorted(baseVars, key=lambda v: (v.section, v.option))
]
headerPacks.extend(baseVarHeaders)
headerPacksToNames = VS.getFieldNamesForHeaderPacks(
metricModel, headerPacks, headerType)
csvWriter = csv.writer(
open(store.replaceFileExtension(targetPath, 'csv'), 'wb'))
csvWriter.writerow(['PROJ.4 ' + self.getProj4()])
csvWriter.writerow([
headerPacksToNames[(section, option)]
for section, option in headerPacks
])
# csvWriter.writerow(['%s > %s' % (section.capitalize(), option.capitalize()) if section else option.capitalize() for section, option in headerPacks])
# For each node,
for node in self.cycleNodes():
# Write row
csvWriter.writerow([
node.output.get(section, {}).get(option, '')
if section else node.input.get(option, '')
for section, option in headerPacks
])
def saveMetricsConfigurationFullCSV(targetPath, metricModel, valueByOptionBySection):
'Save scenario-level INPUT metrics WITH DETAIL as a CSV file'
variableStore = metricModel.VariableStore(valueByOptionBySection)
csvFile = open(store.replaceFileExtension(targetPath, 'csv'), 'wt')
csvWriter = csv.writer(csvFile)
# Note, this assumes section, option, value tuples returned by flatten function
for section, option, value in sorted(util.flatten_to_tuples(valueByOptionBySection)):
find_var = lambda x, section=section, option=option: \
isinstance(x, type) and x.section == section and x.option == option
var = filter(find_var, variableStore.variableClasses)[0]
alias = ""
if var.aliases and len(var.aliases) > 0:
alias = var.aliases[0]
csvWriter.writerow([section, option, alias, value, var.units, var.__doc__])
csvFile.close()
def saveNodesCSV(self, targetPath, isFake=False):
'Save nodes to a csv'
# Initialize
node = self.cycleNodes(isFake).next()
csvWriter = csv.writer(
open(store.replaceFileExtension(targetPath, 'csv'), 'wb'))
# Write spatial reference
csvWriter.writerow(['PROJ.4 ' + self.getProj4()])
# Write column headers
customHeaders = sorted(set(node.input) - set(['name', 'x', 'y']))
csvWriter.writerow(['Name', 'X', 'Y'] +
[x.capitalize() for x in customHeaders])
# For each node,
for node in self.cycleNodes(isFake):
# Write row
csvWriter.writerow(
[node.input.get('name', ''),
node.getX(),
node.getY()] + [node.input.get(x, '') for x in customHeaders])
def saveMetricsConfigurationFullCSV(targetPath, metricModel,
valueByOptionBySection):
'Save scenario-level INPUT metrics WITH DETAIL as a CSV file'
variableStore = metricModel.VariableStore(valueByOptionBySection)
csvFile = open(store.replaceFileExtension(targetPath, 'csv'), 'wt')
csvWriter = csv.writer(csvFile)
# Note, this assumes section, option, value tuples returned by flatten function
for section, option, value in sorted(
util.flatten_to_tuples(valueByOptionBySection)):
find_var = lambda x, section=section, option=option: \
isinstance(x, type) and x.section == section and x.option == option
var = filter(find_var, variableStore.variableClasses)[0]
alias = ""
if var.aliases and len(var.aliases) > 0:
alias = var.aliases[0]
csvWriter.writerow(
[section, option, alias, value, var.units, var.__doc__])
csvFile.close()
def create(targetPath, sourcePath):
'Import the sourcePath to create the dataset'
# Initialize
digestByExtension = {
'.csv': digestNodesFromCSV,
'.zip': digestNodesFromZIP,
}
# Prepare
sourceExtension = os.path.splitext(sourcePath)[1].lower()
if sourceExtension not in digestByExtension:
raise DatasetError(
'Only the following formats are currently supported: ' +
' '.join(digestByExtension))
# Import
proj4, nodePacks = digestByExtension[sourceExtension](sourcePath)
# Save
store.removeSafely(store.replaceFileExtension(targetPath, 'db'))
dataset = Store(targetPath, proj4)
dataset.addNodes(nodePacks)
# Return
return dataset
# If the user is running the script from the command-line,
if __name__ == '__main__':
# Connect (get config and setup model from appropriate DB)
configuration = script_process.connect()
# get ids from stdin into a list
ids = []
for id in sys.stdin:
ids.append(int(id))
# required to know where scenarios dir is
config['storage_path'] = configuration.get('app:main', 'storage_path')
# Iterate through scenarios
scenarios = Session.query(model.Scenario).\
filter(and_(model.Scenario.id.in_(ids),
(model.Scenario.status == model.statusDone))).\
order_by(model.Scenario.id)
for scenario in scenarios:
scenarioFolder = scenario.getDatasetPath()
datasetPath = store.replaceFileExtension(scenarioFolder, 'db')
ds = dataset_store.load(datasetPath)
metricModel = metric.getModel(scenario.input['metric model name'])
vs = metricModel.VariableStore()
specialSaveMetricsCSV(ds, "metrics-local-%s.csv" % scenario.id, metricModel)