def cancel(modelDir): ''' Try to cancel a currently running model. ''' # Kill GLD process if already been created try: with web.locked_open(pJoin(modelDir,"PID.txt"),"r") as pidFile: pid = int(pidFile.read()) # print "pid " + str(pid) os.kill(pid, 15) print("PID KILLED") except: pass # Kill runForeground process try: with web.locked_open(pJoin(modelDir, "PPID.txt"), "r") as pPidFile: pPid = int(pPidFile.read()) os.kill(pPid, 15) print("PPID KILLED") except: pass # Remove PID, PPID, and allOutputData file if existed for fName in ["PID.txt","PPID.txt","allOutputData.json"]: try: os.remove(pJoin(modelDir,fName)) except: pass print("CANCELED", modelDir)
def cancel(modelDir): ''' Try to cancel a currently running model. ''' # Kill the GridLAB-D process if it has already been created. If GridLAB-D hasn't created a PID.txt file, or GridLAB-D never ran, keep going try: with web.locked_open(pJoin(modelDir, "PID.txt"), "r") as pidFile: pid = int(pidFile.read()) # print "pid " + str(pid) os.kill(pid, 15) print("PID KILLED") except: pass # Kill the runForeground process if it has already been created. If __neoMetaModel__.py hasn't created a PPID.txt file yet, keep going try: with web.locked_open(pJoin(modelDir, "PPID.txt"), "r") as pPidFile: pPid = int(pPidFile.read()) os.kill(pPid, 15) print("PPID KILLED") except: pass # Remove PID, PPID, and allOutputData file if existed for fName in ['PID.txt', 'PPID.txt', 'allOutputData.json']: try: os.remove(pJoin(modelDir, fName)) except: pass print("CANCELED", modelDir)
def renderTemplate(modelDir, absolutePaths=False, datastoreNames={}): ''' Render the model template to an HTML string. By default render a blank one for new input. If modelDir is valid, render results post-model-run. If absolutePaths, the HTML can be opened without a server. ''' try: with web.locked_open(pJoin(modelDir, 'allInputData.json')) as f: inJson = json.load(f) modelPath, modelName = pSplit(modelDir) deepPath, user = pSplit(modelPath) inJson["modelName"] = modelName inJson["user"] = user modelType = inJson["modelType"] template = getattr(omf.models, modelType).template allInputData = json.dumps(inJson) # Get hashes for model python and html files with open(pJoin(_myDir, modelType+".html")) as f: htmlFile = f.read() currentHtmlHash = hashlib.sha256(htmlFile.encode('utf-8')).hexdigest() with open(pJoin(_myDir, modelType+".py")) as f: pythonFile = f.read() currentPythonHash = hashlib.sha256(pythonFile.encode('utf-8')).hexdigest() except IOError: allInputData = None inJson = None try: with web.locked_open(pJoin(modelDir,"allOutputData.json")) as f: allOutputData = f.read() with web.locked_open(pJoin(modelDir, "allOutputData.json")) as f: outJson = json.load(f) try: #Needed? Should this be handled a different way? Add hashes to the output if they are not yet present if ('pythonHash' not in outJson) or ('htmlHash' not in outJson): print('new model') outJson['htmlHash'] = currentHtmlHash outJson['pythonHash'] = currentPythonHash outJson['oldVersion'] = False #If the hashes do not match, mark the model as an old version elif outJson['htmlHash'] != currentHtmlHash or outJson['pythonHash'] != currentPythonHash: outJson['oldVersion'] = True #If the hashes match, mark the model as up to date else: outJson['oldVersion'] = False except (UnboundLocalError, KeyError) as e: print((traceback.print_exc())) print(('error:' + str(e))) except IOError: allOutputData = None outJson = None if absolutePaths: # Parent of current folder. pathPrefix = _omfDir else: pathPrefix = "" # Raw input output include. return template.render(allInputData=allInputData, allOutputData=allOutputData, modelStatus=getStatus(modelDir), pathPrefix=pathPrefix, datastoreNames=datastoreNames, modelName=modelType, allInputDataDict=inJson, allOutputDataDict=outJson)
def heavyProcessing(modelDir): ''' Wrapper to handle model running safely and uniformly. ''' try: # Start a timer. startTime = datetime.datetime.now() # Get the inputs. with web.locked_open(pJoin(modelDir, 'allInputData.json')) as f: inputDict = json.load(f) # Remove old outputs. try: os.remove(pJoin(modelDir,"allOutputData.json")) except Exception as e: pass # Get the function and run it. work = getattr(omf.models, inputDict['modelType']).work #This grabs the new outData model outData = work(modelDir, inputDict) except: # If input range wasn't valid delete output, write error to disk. cancel(modelDir) thisErr = traceback.format_exc() print('ERROR IN MODEL', modelDir, thisErr) inputDict['stderr'] = thisErr with web.locked_open(os.path.join(modelDir,'stderr.txt'),'w') as errorFile: errorFile.write(thisErr) else: # No errors, so update the runTime in the input file. endTime = datetime.datetime.now() inputDict["runTime"] = str(datetime.timedelta(seconds=int((endTime - startTime).total_seconds()))) # Write output. modelType = inputDict["modelType"] #Get current file hashes and dd to the output with open(pJoin(_myDir, modelType+".html")) as f: htmlFile = f.read() currentHtmlHash = hashlib.sha256(htmlFile.encode('utf-8')).hexdigest() with open(pJoin(_myDir, modelType+".py")) as f: pythonFile = f.read() currentPythonHash = hashlib.sha256(pythonFile.encode('utf-8')).hexdigest() outData['htmlHash'] = currentHtmlHash outData['pythonHash'] = currentPythonHash outData['oldVersion'] = False # Raw input/output file names. outData['fileNames'] = os.listdir(modelDir) outData['fileNames'].append('allOutputData.json') with web.locked_open(pJoin(modelDir, "allOutputData.json"),"w") as outFile: json.dump(outData, outFile, indent=4) finally: # Clean up by updating input data. try: with web.locked_open(pJoin(modelDir,"allInputData.json"),"w") as inFile: json.dump(inputDict, inFile, indent=4) except: pass try: os.remove(pJoin(modelDir,"PPID.txt")) except: pass
def run(modelDir): ''' Run the model in a separate process. web.py calls this to run the model. This function will return fast, but results take a while to hit the file system.''' backProc = multiprocessing.Process(target = heavyProcessing, args = (modelDir,)) backProc.start() with web.locked_open(pJoin(modelDir, "PPID.txt"),"w+") as pPidFile: pPidFile.write(str(backProc.pid)) print("SENT TO BACKGROUND", modelDir)
def runForeground(modelDir): ''' Run all model work immediately in the same thread. ''' with web.locked_open(pJoin(modelDir, "PPID.txt"), "w+") as pPidFile: pPidFile.write( '-999' ) # HACK: put in an invalid PID to indicate the model is running. print("FOREGROUND RUNNING", modelDir) heavyProcessing(modelDir)
def new(modelDir, defaultInputs): ''' Create a new instance of a model. Returns true on success, false on failure. ''' alreadyThere = os.path.isdir(modelDir) or os.path.isfile(modelDir) try: if not alreadyThere: os.makedirs(modelDir) else: return False defaultInputs["created"] = str(datetime.datetime.now()) with web.locked_open(pJoin(modelDir, "allInputData.json"),"w") as inputFile: json.dump(defaultInputs, inputFile, indent = 4) return True except: return False
def renderTemplateToFile(modelDir, datastoreNames={}): ''' Render and open a template (blank or with output) in a local browser. ''' with tempfile.NamedTemporaryFile('w+', suffix=".html", delete=False) as baseTemplate: baseTemplate.write(renderTemplate(modelDir, absolutePaths=False)) baseTemplate.flush() baseTemplate.seek(0) with web.locked_open(pJoin(modelDir, 'inlineTemplate.html'), 'w', encoding='utf-8') as inlineTemplate: for line in baseTemplate: #add backslash to regex between signle and double quote matchObj = re.match(r"(.*)/static(.+?)(['\"])(.+?)", line, re.M | re.I) scriptTags = re.match(r"(.*)<script(.*)static/(.*)</script>", line, re.M | re.I) styleTags = re.match(r"(.*)<link(.*)stylesheet", line, re.M | re.I) if scriptTags: with open(_omfDir + "/static" + matchObj.group(2)) as f: sourceFile = f.read() with open(_omfDir + "/static" + matchObj.group(2), 'r', encoding='utf-8') as yFile: ttempfile = yFile.readlines() tmp = '<script>' + sourceFile + '</script>' inlineTemplate.write('<script>') for i in ttempfile: try: inlineTemplate.write(i) except (UnicodeEncodeError): print(i) inlineTemplate.write('</script>') elif styleTags: with open(_omfDir + "/static" + matchObj.group(2), 'r', encoding='utf-8') as yFile: ttempfile = yFile.readlines() inlineTemplate.write('<style>') for i in ttempfile: try: inlineTemplate.write(i) except (UnicodeEncodeError): print(i) inlineTemplate.write('</style>') else: inlineTemplate.write(str(line))
def renderTemplate(modelDir, absolutePaths=False, datastoreNames={}): ''' Render the model template to an HTML string. By default render a blank one for new input. If modelDir is valid, render results post-model-run. If absolutePaths, the HTML can be opened without a server. ''' try: with web.locked_open(pJoin(modelDir, 'allInputData.json')) as f: inJson = json.load(f) modelPath, modelName = pSplit(modelDir) deepPath, modelOwner = pSplit(modelPath) inJson["modelName"] = modelName inJson["user"] = modelOwner modelType = inJson["modelType"] template = getattr(omf.models, modelType).template allInputData = json.dumps(inJson) # Get hashes for model python and html files with open(pJoin(_myDir, modelType + ".html")) as f: htmlFile = f.read() currentHtmlHash = hashlib.sha256(htmlFile.encode('utf-8')).hexdigest() with open(pJoin(_myDir, modelType + ".py")) as f: pythonFile = f.read() currentPythonHash = hashlib.sha256( pythonFile.encode('utf-8')).hexdigest() except IOError: allInputData = None inJson = None try: with web.locked_open(pJoin(modelDir, "allOutputData.json")) as f: allOutputData = f.read() with web.locked_open(pJoin(modelDir, "allOutputData.json")) as f: outJson = json.load(f) try: #Needed? Should this be handled a different way? Add hashes to the output if they are not yet present if ('pythonHash' not in outJson) or ('htmlHash' not in outJson): outJson['htmlHash'] = currentHtmlHash outJson['pythonHash'] = currentPythonHash outJson['oldVersion'] = False #If the hashes do not match, mark the model as an old version elif outJson['htmlHash'] != currentHtmlHash or outJson[ 'pythonHash'] != currentPythonHash: outJson['oldVersion'] = True #If the hashes match, mark the model as up to date else: outJson['oldVersion'] = False except (UnboundLocalError, KeyError) as e: print((traceback.print_exc())) print(('error:' + str(e))) except IOError: allOutputData = None outJson = None if absolutePaths: # Parent of current folder. pathPrefix = _omfDir else: pathPrefix = "" # Generate standard raw output files. rawFilesTemplate = ''' <p class="reportTitle">Raw Input and Output Files</p> <div id="rawOutput" class="content" style="margin-top:0px"> {% for name in allOutputDataDict['fileNames'] %} {% if loop.index > 1 %}— {% endif %}<a href="/downloadModelData/{{allInputDataDict['user']}}/{{allInputDataDict['modelName']}}/{{name}}">{{name}}</a> {% endfor %} </div> ''' rawOutputFiles = Template(rawFilesTemplate).render( allOutputDataDict=outJson, allInputDataDict=inJson) # Generate standard model buttons. omfModelButtonsTemplate = ''' <div class="wideInput" style="text-align:right"> {% if modelStatus != 'running' and (loggedInUser == modelOwner or loggedInUser == 'admin') %} <button id="deleteButton" type="button" onclick="deleteModel()">Delete</button> <button id="runButton" type="submit">Run Model</button> {% endif %} {% if modelStatus == "finished" %} <button id="shareButton" type="button" onclick="shareModel()">Share</button> <button id="duplicateButton" type="button" onclick="duplicateModel()">Duplicate</button> {% endif %} {% if modelStatus == "running" and (loggedInUser == modelOwner or loggedInUser == 'admin') %} <button id="cancelButton" type="button" onclick="cancelModel()">Cancel Run</button> {% endif %} </div> ''' # Generate standard status content. loggedInUser = datastoreNames.get('currentUser', 'test') modelStatus = getStatus(modelDir) omfModelButtons = Template(omfModelButtonsTemplate).render( modelStatus=modelStatus, loggedInUser=loggedInUser, modelOwner=modelOwner) now = datetime.datetime.now() try: mod_start = datetime.datetime.fromisoformat(inJson.get('runStartTime')) except: mod_start = now elapsed_dt = now - mod_start elapsed_min = elapsed_dt.total_seconds() / 60.0 model_estimate_min = float(inJson.get('runtimeEst_min', '2.0')) remain_min = model_estimate_min - elapsed_min runDebugTemplate = ''' {% if modelStatus == 'running' %} <div id ="runIndicator" class="content"> Model has run for {{elapsed_min}} minutes. {{remain_min}} minutes estimated until completion. Results updated every 5 seconds. </div> {% endif %} {% if modelStatus == 'stopped' and stderr != '' %} <div id ="stopIndicator" class="content"> <pre id='errorText' style='overflow-x:scroll'>MODEL ENCOUNTERED AN ERROR AS FOLLOWS:\n\n{{stderr}}</pre> </div> {% endif %} ''' omfRunDebugBlock = Template(runDebugTemplate).render( modelStatus=modelStatus, stderr=inJson.get('stderr', ''), elapsed_min=round(elapsed_min, 2), remain_min=round(remain_min, 2)) # Raw input output include. return template.render(allInputData=allInputData, allOutputData=allOutputData, modelStatus=modelStatus, pathPrefix=pathPrefix, datastoreNames=datastoreNames, modelName=modelType, allInputDataDict=inJson, allOutputDataDict=outJson, rawOutputFiles=rawOutputFiles, omfModelButtons=omfModelButtons, omfRunDebugBlock=omfRunDebugBlock)