def save_job_input_file(scenario): """ Save scenario leve inputs """ scenarioFolder = scenario.getFolder() store.makeFolderSafely(scenarioFolder) expandPath = lambda x: os.path.join(scenarioFolder, x) metricConfiguration = scenario.input['metric configuration'] metric.saveMetricsConfigurationCSV(expandPath('metrics-job-input'), metricConfiguration) store.zipFolder(scenarioFolder + '.zip', scenarioFolder)
def extractConfigurationByName(valueByName, scenarioFolder): # Initialize configuration = {} # For each value, for key, value in valueByName.iteritems(): # Parse key keyTerms = variable_store.parseKey(key) # If the key is compound, if len(keyTerms) > 1: # Extract modelType, section, option = keyTerms # If the value already exists, then it must have been overridden if modelType in configuration: if section in configuration[modelType]: if option in configuration[modelType][section]: continue # If we have a hidden field, if option.endswith('_h'): # If the hidden field was overridden, skip this and wait until we find the real value if int(value) == 0: continue # Remove suffix option = option[:-2] # Prepare sourceScenario = Session.query(model.Scenario).get(value) relativePath = sourceScenario.input['%s configuration' % modelType][section][option] # If the old scenario did not specify a file here, if not relativePath: value = '' else: # Prepare sourcePath = os.path.join(sourceScenario.getFolder(), relativePath) # Copy source in case it is deleted store.makeFolderSafely(os.path.join(scenarioFolder, os.path.dirname(relativePath))) targetPath = os.path.join(scenarioFolder, relativePath) shutil.copyfile(sourcePath, targetPath) value = relativePath # If the user wants to use a new file and the value is an upload, elif hasattr(value, 'file'): # Prepare relativePath = os.path.join(modelType, section, option + os.path.splitext(value.filename)[1]) # Save it store.makeFolderSafely(os.path.join(scenarioFolder, os.path.dirname(relativePath))) targetPath = os.path.join(scenarioFolder, relativePath) shutil.copyfileobj(value.file, open(targetPath, 'wb')) value = relativePath # Store if modelType not in configuration: configuration[modelType] = {} if section not in configuration[modelType]: configuration[modelType][section] = {} configuration[modelType][section][option] = value # Return return configuration
def run(step, scriptPath, convertByName): 'Runs the script on a configuration file' # Import custom modules from np.lib import store, folder_store # Parse options and arguments optionParser = optparse.OptionParser(usage='%prog QUEUE-PATH') optionParser.add_option('-t', '--test', dest='isTest', default=False, action='store_true', help='test your QUEUE-PATH') optionParser.add_option('-d', '--directory', dest='directory', default='.', metavar='DIRECTORY', help='save results in DIRECTORY') options, arguments = optionParser.parse_args() if len(arguments) != 1: return optionParser.print_help() filePath = arguments[0] # Go try: # Load information parameterByNameByTask = store.loadQueue(filePath, convertByName) # Initialize if options.isTest: options.directory = store.makeFolderSafely(os.path.join(options.directory, 'test')) folderStore = folder_store.Store(options.directory) # Step through each task for taskName, parameterByName in parameterByNameByTask.iteritems(): step(taskName, parameterByName, folderStore, options.isTest) except (store.StoreError, folder_store.FolderError, ScriptError), error: print '%s %s\n%s' % (store.extractFileBaseName(scriptPath), filePath, error)
def regenerate_demographic_file(scenario, proj4=geometry_store.proj4LL): """ In case demographic input file is lost, this regenerates it NOTE: This uses the node dict stored in scenario.output. The nodes there are all nodes from the original datasetStore Which may contain some "FAKE" nodes if the scenario had an input network. """ scenarioFolder = scenario.getFolder() store.makeFolderSafely(scenarioFolder) expandPath = lambda x: os.path.join(scenarioFolder, x) # recreate datasetStore targetPath = scenario.getDatasetPath() targetPath = targetPath.replace(".", "_tmp.") #Don't overwrite existing store if it's there datasetStore = dataset_store.Store(targetPath, proj4) nodeDict = scenario.output['variables']['node'] datasetStore.addNodesFromNodeDict(nodeDict) sourcePath = expandPath(scenario.input['demographic file name']) datasetStore.saveNodesCSV(sourcePath) #delete the tmp dataset store os.remove(targetPath)
def create(self): 'POST /scenarios: Create a new item' # Initialize personID = h.getPersonID() if not personID: return redirect(url('person_login', targetURL=h.encodeURL(h.url('new_scenario')))) # Load try: demographicDatabase_h = int(request.POST.get('demographicDatabase_h', 0)) except ValueError: demographicDatabase_h = 0 if not demographicDatabase_h and 'demographicDatabase' not in request.POST: return cjson.encode(dict(isOk=0, message='The demographicDatabase field is required')) scenarioName = request.POST.get('scenarioName') or 'Untitled' try: scenarioScope = int(request.POST.get('scenarioScope', model.scopePrivate)) except ValueError: scenarioScope = model.scopePrivate metricModelName = request.POST.get('metricModelName', metric.getModelNames()[0]) networkModelName = request.POST.get('networkModelName', network.getModelNames()[0]) callbackURL = request.POST.get('callbackURL') # Create scenario scenario = model.Scenario(personID, scenarioName, scenarioScope) Session.add(scenario) Session.commit() scenarioFolder = scenario.getFolder() if os.path.exists(scenarioFolder): shutil.rmtree(scenarioFolder) store.makeFolderSafely(scenarioFolder) # If the user is using an existing demographicDatabase, if demographicDatabase_h: # Copy source in case it is deleted sourceScenario = Session.query(model.Scenario).get(demographicDatabase_h) sourceScenarioFolder = sourceScenario.getFolder() demographicFileName = sourceScenario.input['demographic file name'] demographicPath = os.path.join(scenarioFolder, demographicFileName) shutil.copyfile(os.path.join(sourceScenarioFolder, demographicFileName), demographicPath) # If the user is uploading a new demographicDatabase, else: # Save original demographicDatabase in case the user wants it later demographicDatabase = request.POST['demographicDatabase'] demographicFileExtension = os.path.splitext(demographicDatabase.filename)[1] demographicFileName = 'demographics' + demographicFileExtension demographicPath = os.path.join(scenarioFolder, demographicFileName) shutil.copyfileobj(demographicDatabase.file, open(demographicPath, 'wb')) demographicDatabase.file.close() # Store input configurationByName = extractConfigurationByName(request.POST, scenarioFolder) scenario.input = { 'demographic file name': str(demographicFileName), 'metric model name': metricModelName, 'metric configuration': configurationByName.get('metric', {}), 'network model name': networkModelName, 'network configuration': configurationByName.get('network', {}), 'callback url': callbackURL, 'host url': request.host_url, } Session.commit() store.zipFolder(scenarioFolder + '.zip', scenarioFolder) # Redirect redirect(url('scenario', id=scenario.id))