def save_job_input_file(scenario):
""" Save scenario leve inputs """
scenarioFolder = scenario.getFolder()
store.makeFolderSafely(scenarioFolder)
expandPath = lambda x: os.path.join(scenarioFolder, x)
metricConfiguration = scenario.input['metric configuration']
metric.saveMetricsConfigurationCSV(expandPath('metrics-job-input'), metricConfiguration)
store.zipFolder(scenarioFolder + '.zip', scenarioFolder)
def extractConfigurationByName(valueByName, scenarioFolder):
# Initialize
configuration = {}
# For each value,
for key, value in valueByName.iteritems():
# Parse key
keyTerms = variable_store.parseKey(key)
# If the key is compound,
if len(keyTerms) > 1:
# Extract
modelType, section, option = keyTerms
# If the value already exists, then it must have been overridden
if modelType in configuration:
if section in configuration[modelType]:
if option in configuration[modelType][section]:
continue
# If we have a hidden field,
if option.endswith('_h'):
# If the hidden field was overridden, skip this and wait until we find the real value
if int(value) == 0:
continue
# Remove suffix
option = option[:-2]
# Prepare
sourceScenario = Session.query(model.Scenario).get(value)
relativePath = sourceScenario.input['%s configuration' % modelType][section][option]
# If the old scenario did not specify a file here,
if not relativePath:
value = ''
else:
# Prepare
sourcePath = os.path.join(sourceScenario.getFolder(), relativePath)
# Copy source in case it is deleted
store.makeFolderSafely(os.path.join(scenarioFolder, os.path.dirname(relativePath)))
targetPath = os.path.join(scenarioFolder, relativePath)
shutil.copyfile(sourcePath, targetPath)
value = relativePath
# If the user wants to use a new file and the value is an upload,
elif hasattr(value, 'file'):
# Prepare
relativePath = os.path.join(modelType, section, option + os.path.splitext(value.filename)[1])
# Save it
store.makeFolderSafely(os.path.join(scenarioFolder, os.path.dirname(relativePath)))
targetPath = os.path.join(scenarioFolder, relativePath)
shutil.copyfileobj(value.file, open(targetPath, 'wb'))
value = relativePath
# Store
if modelType not in configuration:
configuration[modelType] = {}
if section not in configuration[modelType]:
configuration[modelType][section] = {}
configuration[modelType][section][option] = value
# Return
return configuration
def run(step, scriptPath, convertByName):
'Runs the script on a configuration file'
# Import custom modules
from np.lib import store, folder_store
# Parse options and arguments
optionParser = optparse.OptionParser(usage='%prog QUEUE-PATH')
optionParser.add_option('-t', '--test', dest='isTest',
default=False, action='store_true',
help='test your QUEUE-PATH')
optionParser.add_option('-d', '--directory', dest='directory',
default='.', metavar='DIRECTORY',
help='save results in DIRECTORY')
options, arguments = optionParser.parse_args()
if len(arguments) != 1:
return optionParser.print_help()
filePath = arguments[0]
# Go
try:
# Load information
parameterByNameByTask = store.loadQueue(filePath, convertByName)
# Initialize
if options.isTest:
options.directory = store.makeFolderSafely(os.path.join(options.directory, 'test'))
folderStore = folder_store.Store(options.directory)
# Step through each task
for taskName, parameterByName in parameterByNameByTask.iteritems():
step(taskName, parameterByName, folderStore, options.isTest)
except (store.StoreError, folder_store.FolderError, ScriptError), error:
print '%s %s\n%s' % (store.extractFileBaseName(scriptPath), filePath, error)
def regenerate_demographic_file(scenario, proj4=geometry_store.proj4LL):
""" In case demographic input file is lost, this regenerates it
NOTE: This uses the node dict stored in scenario.output.
The nodes there are all nodes from the original datasetStore
Which may contain some "FAKE" nodes if the scenario had an
input network.
"""
scenarioFolder = scenario.getFolder()
store.makeFolderSafely(scenarioFolder)
expandPath = lambda x: os.path.join(scenarioFolder, x)
# recreate datasetStore
targetPath = scenario.getDatasetPath()
targetPath = targetPath.replace(".", "_tmp.") #Don't overwrite existing store if it's there
datasetStore = dataset_store.Store(targetPath, proj4)
nodeDict = scenario.output['variables']['node']
datasetStore.addNodesFromNodeDict(nodeDict)
sourcePath = expandPath(scenario.input['demographic file name'])
datasetStore.saveNodesCSV(sourcePath)
#delete the tmp dataset store
os.remove(targetPath)
def create(self):
'POST /scenarios: Create a new item'
# Initialize
personID = h.getPersonID()
if not personID:
return redirect(url('person_login', targetURL=h.encodeURL(h.url('new_scenario'))))
# Load
try:
demographicDatabase_h = int(request.POST.get('demographicDatabase_h', 0))
except ValueError:
demographicDatabase_h = 0
if not demographicDatabase_h and 'demographicDatabase' not in request.POST:
return cjson.encode(dict(isOk=0, message='The demographicDatabase field is required'))
scenarioName = request.POST.get('scenarioName') or 'Untitled'
try:
scenarioScope = int(request.POST.get('scenarioScope', model.scopePrivate))
except ValueError:
scenarioScope = model.scopePrivate
metricModelName = request.POST.get('metricModelName', metric.getModelNames()[0])
networkModelName = request.POST.get('networkModelName', network.getModelNames()[0])
callbackURL = request.POST.get('callbackURL')
# Create scenario
scenario = model.Scenario(personID, scenarioName, scenarioScope)
Session.add(scenario)
Session.commit()
scenarioFolder = scenario.getFolder()
if os.path.exists(scenarioFolder):
shutil.rmtree(scenarioFolder)
store.makeFolderSafely(scenarioFolder)
# If the user is using an existing demographicDatabase,
if demographicDatabase_h:
# Copy source in case it is deleted
sourceScenario = Session.query(model.Scenario).get(demographicDatabase_h)
sourceScenarioFolder = sourceScenario.getFolder()
demographicFileName = sourceScenario.input['demographic file name']
demographicPath = os.path.join(scenarioFolder, demographicFileName)
shutil.copyfile(os.path.join(sourceScenarioFolder, demographicFileName), demographicPath)
# If the user is uploading a new demographicDatabase,
else:
# Save original demographicDatabase in case the user wants it later
demographicDatabase = request.POST['demographicDatabase']
demographicFileExtension = os.path.splitext(demographicDatabase.filename)[1]
demographicFileName = 'demographics' + demographicFileExtension
demographicPath = os.path.join(scenarioFolder, demographicFileName)
shutil.copyfileobj(demographicDatabase.file, open(demographicPath, 'wb'))
demographicDatabase.file.close()
# Store input
configurationByName = extractConfigurationByName(request.POST, scenarioFolder)
scenario.input = {
'demographic file name': str(demographicFileName),
'metric model name': metricModelName,
'metric configuration': configurationByName.get('metric', {}),
'network model name': networkModelName,
'network configuration': configurationByName.get('network', {}),
'callback url': callbackURL,
'host url': request.host_url,
}
Session.commit()
store.zipFolder(scenarioFolder + '.zip', scenarioFolder)
# Redirect
redirect(url('scenario', id=scenario.id))
