def cancel(modelDir):
''' Try to cancel a currently running model. '''
# Kill GLD process if already been created
try:
with web.locked_open(pJoin(modelDir,"PID.txt"),"r") as pidFile:
pid = int(pidFile.read())
# print "pid " + str(pid)
os.kill(pid, 15)
print("PID KILLED")
except:
pass
# Kill runForeground process
try:
with web.locked_open(pJoin(modelDir, "PPID.txt"), "r") as pPidFile:
pPid = int(pPidFile.read())
os.kill(pPid, 15)
print("PPID KILLED")
except:
pass
# Remove PID, PPID, and allOutputData file if existed
for fName in ["PID.txt","PPID.txt","allOutputData.json"]:
try:
os.remove(pJoin(modelDir,fName))
except:
pass
print("CANCELED", modelDir)
def cancel(modelDir):
''' Try to cancel a currently running model. '''
# Kill the GridLAB-D process if it has already been created. If GridLAB-D hasn't created a PID.txt file, or GridLAB-D never ran, keep going
try:
with web.locked_open(pJoin(modelDir, "PID.txt"), "r") as pidFile:
pid = int(pidFile.read())
# print "pid " + str(pid)
os.kill(pid, 15)
print("PID KILLED")
except:
pass
# Kill the runForeground process if it has already been created. If __neoMetaModel__.py hasn't created a PPID.txt file yet, keep going
try:
with web.locked_open(pJoin(modelDir, "PPID.txt"), "r") as pPidFile:
pPid = int(pPidFile.read())
os.kill(pPid, 15)
print("PPID KILLED")
except:
pass
# Remove PID, PPID, and allOutputData file if existed
for fName in ['PID.txt', 'PPID.txt', 'allOutputData.json']:
try:
os.remove(pJoin(modelDir, fName))
except:
pass
print("CANCELED", modelDir)
def renderTemplate(modelDir, absolutePaths=False, datastoreNames={}):
''' Render the model template to an HTML string.
By default render a blank one for new input.
If modelDir is valid, render results post-model-run.
If absolutePaths, the HTML can be opened without a server. '''
try:
with web.locked_open(pJoin(modelDir, 'allInputData.json')) as f:
inJson = json.load(f)
modelPath, modelName = pSplit(modelDir)
deepPath, user = pSplit(modelPath)
inJson["modelName"] = modelName
inJson["user"] = user
modelType = inJson["modelType"]
template = getattr(omf.models, modelType).template
allInputData = json.dumps(inJson)
# Get hashes for model python and html files
with open(pJoin(_myDir, modelType+".html")) as f:
htmlFile = f.read()
currentHtmlHash = hashlib.sha256(htmlFile.encode('utf-8')).hexdigest()
with open(pJoin(_myDir, modelType+".py")) as f:
pythonFile = f.read()
currentPythonHash = hashlib.sha256(pythonFile.encode('utf-8')).hexdigest()
except IOError:
allInputData = None
inJson = None
try:
with web.locked_open(pJoin(modelDir,"allOutputData.json")) as f:
allOutputData = f.read()
with web.locked_open(pJoin(modelDir, "allOutputData.json")) as f:
outJson = json.load(f)
try:
#Needed? Should this be handled a different way? Add hashes to the output if they are not yet present
if ('pythonHash' not in outJson) or ('htmlHash' not in outJson):
print('new model')
outJson['htmlHash'] = currentHtmlHash
outJson['pythonHash'] = currentPythonHash
outJson['oldVersion'] = False
#If the hashes do not match, mark the model as an old version
elif outJson['htmlHash'] != currentHtmlHash or outJson['pythonHash'] != currentPythonHash:
outJson['oldVersion'] = True
#If the hashes match, mark the model as up to date
else:
outJson['oldVersion'] = False
except (UnboundLocalError, KeyError) as e:
print((traceback.print_exc()))
print(('error:' + str(e)))
except IOError:
allOutputData = None
outJson = None
if absolutePaths:
# Parent of current folder.
pathPrefix = _omfDir
else:
pathPrefix = ""
# Raw input output include.
return template.render(allInputData=allInputData, allOutputData=allOutputData, modelStatus=getStatus(modelDir), pathPrefix=pathPrefix,
datastoreNames=datastoreNames, modelName=modelType, allInputDataDict=inJson, allOutputDataDict=outJson)
def heavyProcessing(modelDir):
''' Wrapper to handle model running safely and uniformly. '''
try:
# Start a timer.
startTime = datetime.datetime.now()
# Get the inputs.
with web.locked_open(pJoin(modelDir, 'allInputData.json')) as f:
inputDict = json.load(f)
# Remove old outputs.
try: os.remove(pJoin(modelDir,"allOutputData.json"))
except Exception as e: pass
# Get the function and run it.
work = getattr(omf.models, inputDict['modelType']).work
#This grabs the new outData model
outData = work(modelDir, inputDict)
except:
# If input range wasn't valid delete output, write error to disk.
cancel(modelDir)
thisErr = traceback.format_exc()
print('ERROR IN MODEL', modelDir, thisErr)
inputDict['stderr'] = thisErr
with web.locked_open(os.path.join(modelDir,'stderr.txt'),'w') as errorFile:
errorFile.write(thisErr)
else:
# No errors, so update the runTime in the input file.
endTime = datetime.datetime.now()
inputDict["runTime"] = str(datetime.timedelta(seconds=int((endTime - startTime).total_seconds())))
# Write output.
modelType = inputDict["modelType"]
#Get current file hashes and dd to the output
with open(pJoin(_myDir, modelType+".html")) as f:
htmlFile = f.read()
currentHtmlHash = hashlib.sha256(htmlFile.encode('utf-8')).hexdigest()
with open(pJoin(_myDir, modelType+".py")) as f:
pythonFile = f.read()
currentPythonHash = hashlib.sha256(pythonFile.encode('utf-8')).hexdigest()
outData['htmlHash'] = currentHtmlHash
outData['pythonHash'] = currentPythonHash
outData['oldVersion'] = False
# Raw input/output file names.
outData['fileNames'] = os.listdir(modelDir)
outData['fileNames'].append('allOutputData.json')
with web.locked_open(pJoin(modelDir, "allOutputData.json"),"w") as outFile:
json.dump(outData, outFile, indent=4)
finally:
# Clean up by updating input data.
try:
with web.locked_open(pJoin(modelDir,"allInputData.json"),"w") as inFile:
json.dump(inputDict, inFile, indent=4)
except: pass
try: os.remove(pJoin(modelDir,"PPID.txt"))
except: pass
def run(modelDir):
''' Run the model in a separate process. web.py calls this to run the model.
This function will return fast, but results take a while to hit the file system.'''
backProc = multiprocessing.Process(target = heavyProcessing, args = (modelDir,))
backProc.start()
with web.locked_open(pJoin(modelDir, "PPID.txt"),"w+") as pPidFile:
pPidFile.write(str(backProc.pid))
print("SENT TO BACKGROUND", modelDir)
def runForeground(modelDir):
''' Run all model work immediately in the same thread. '''
with web.locked_open(pJoin(modelDir, "PPID.txt"), "w+") as pPidFile:
pPidFile.write(
'-999'
) # HACK: put in an invalid PID to indicate the model is running.
print("FOREGROUND RUNNING", modelDir)
heavyProcessing(modelDir)
def new(modelDir, defaultInputs): ''' Create a new instance of a model. Returns true on success, false on failure. ''' alreadyThere = os.path.isdir(modelDir) or os.path.isfile(modelDir) try: if not alreadyThere: os.makedirs(modelDir) else: return False defaultInputs["created"] = str(datetime.datetime.now()) with web.locked_open(pJoin(modelDir, "allInputData.json"),"w") as inputFile: json.dump(defaultInputs, inputFile, indent = 4) return True except: return False
def renderTemplateToFile(modelDir, datastoreNames={}):
''' Render and open a template (blank or with output) in a local browser. '''
with tempfile.NamedTemporaryFile('w+', suffix=".html",
delete=False) as baseTemplate:
baseTemplate.write(renderTemplate(modelDir, absolutePaths=False))
baseTemplate.flush()
baseTemplate.seek(0)
with web.locked_open(pJoin(modelDir, 'inlineTemplate.html'),
'w',
encoding='utf-8') as inlineTemplate:
for line in baseTemplate:
#add backslash to regex between signle and double quote
matchObj = re.match(r"(.*)/static(.+?)(['\"])(.+?)", line,
re.M | re.I)
scriptTags = re.match(r"(.*)<script(.*)static/(.*)</script>",
line, re.M | re.I)
styleTags = re.match(r"(.*)<link(.*)stylesheet", line,
re.M | re.I)
if scriptTags:
with open(_omfDir + "/static" + matchObj.group(2)) as f:
sourceFile = f.read()
with open(_omfDir + "/static" + matchObj.group(2),
'r',
encoding='utf-8') as yFile:
ttempfile = yFile.readlines()
tmp = '<script>' + sourceFile + '</script>'
inlineTemplate.write('<script>')
for i in ttempfile:
try:
inlineTemplate.write(i)
except (UnicodeEncodeError):
print(i)
inlineTemplate.write('</script>')
elif styleTags:
with open(_omfDir + "/static" + matchObj.group(2),
'r',
encoding='utf-8') as yFile:
ttempfile = yFile.readlines()
inlineTemplate.write('<style>')
for i in ttempfile:
try:
inlineTemplate.write(i)
except (UnicodeEncodeError):
print(i)
inlineTemplate.write('</style>')
else:
inlineTemplate.write(str(line))
def renderTemplate(modelDir, absolutePaths=False, datastoreNames={}):
''' Render the model template to an HTML string.
By default render a blank one for new input.
If modelDir is valid, render results post-model-run.
If absolutePaths, the HTML can be opened without a server. '''
try:
with web.locked_open(pJoin(modelDir, 'allInputData.json')) as f:
inJson = json.load(f)
modelPath, modelName = pSplit(modelDir)
deepPath, modelOwner = pSplit(modelPath)
inJson["modelName"] = modelName
inJson["user"] = modelOwner
modelType = inJson["modelType"]
template = getattr(omf.models, modelType).template
allInputData = json.dumps(inJson)
# Get hashes for model python and html files
with open(pJoin(_myDir, modelType + ".html")) as f:
htmlFile = f.read()
currentHtmlHash = hashlib.sha256(htmlFile.encode('utf-8')).hexdigest()
with open(pJoin(_myDir, modelType + ".py")) as f:
pythonFile = f.read()
currentPythonHash = hashlib.sha256(
pythonFile.encode('utf-8')).hexdigest()
except IOError:
allInputData = None
inJson = None
try:
with web.locked_open(pJoin(modelDir, "allOutputData.json")) as f:
allOutputData = f.read()
with web.locked_open(pJoin(modelDir, "allOutputData.json")) as f:
outJson = json.load(f)
try:
#Needed? Should this be handled a different way? Add hashes to the output if they are not yet present
if ('pythonHash' not in outJson) or ('htmlHash' not in outJson):
outJson['htmlHash'] = currentHtmlHash
outJson['pythonHash'] = currentPythonHash
outJson['oldVersion'] = False
#If the hashes do not match, mark the model as an old version
elif outJson['htmlHash'] != currentHtmlHash or outJson[
'pythonHash'] != currentPythonHash:
outJson['oldVersion'] = True
#If the hashes match, mark the model as up to date
else:
outJson['oldVersion'] = False
except (UnboundLocalError, KeyError) as e:
print((traceback.print_exc()))
print(('error:' + str(e)))
except IOError:
allOutputData = None
outJson = None
if absolutePaths:
# Parent of current folder.
pathPrefix = _omfDir
else:
pathPrefix = ""
# Generate standard raw output files.
rawFilesTemplate = '''
<p class="reportTitle">Raw Input and Output Files</p>
<div id="rawOutput" class="content" style="margin-top:0px">
{% for name in allOutputDataDict['fileNames'] %}
{% if loop.index > 1 %}— {% endif %}<a href="/downloadModelData/{{allInputDataDict['user']}}/{{allInputDataDict['modelName']}}/{{name}}">{{name}}</a>
{% endfor %}
</div>
'''
rawOutputFiles = Template(rawFilesTemplate).render(
allOutputDataDict=outJson, allInputDataDict=inJson)
# Generate standard model buttons.
omfModelButtonsTemplate = '''
<div class="wideInput" style="text-align:right">
{% if modelStatus != 'running' and (loggedInUser == modelOwner or loggedInUser == 'admin') %}
<button id="deleteButton" type="button" onclick="deleteModel()">Delete</button>
<button id="runButton" type="submit">Run Model</button>
{% endif %}
{% if modelStatus == "finished" %}
<button id="shareButton" type="button" onclick="shareModel()">Share</button>
<button id="duplicateButton" type="button" onclick="duplicateModel()">Duplicate</button>
{% endif %}
{% if modelStatus == "running" and (loggedInUser == modelOwner or loggedInUser == 'admin') %}
<button id="cancelButton" type="button" onclick="cancelModel()">Cancel Run</button>
{% endif %}
</div>
'''
# Generate standard status content.
loggedInUser = datastoreNames.get('currentUser', 'test')
modelStatus = getStatus(modelDir)
omfModelButtons = Template(omfModelButtonsTemplate).render(
modelStatus=modelStatus,
loggedInUser=loggedInUser,
modelOwner=modelOwner)
now = datetime.datetime.now()
try:
mod_start = datetime.datetime.fromisoformat(inJson.get('runStartTime'))
except:
mod_start = now
elapsed_dt = now - mod_start
elapsed_min = elapsed_dt.total_seconds() / 60.0
model_estimate_min = float(inJson.get('runtimeEst_min', '2.0'))
remain_min = model_estimate_min - elapsed_min
runDebugTemplate = '''
{% if modelStatus == 'running' %}
<div id ="runIndicator" class="content">
Model has run for {{elapsed_min}} minutes. {{remain_min}} minutes estimated until completion. Results updated every 5 seconds.
</div>
{% endif %}
{% if modelStatus == 'stopped' and stderr != '' %}
<div id ="stopIndicator" class="content">
<pre id='errorText' style='overflow-x:scroll'>MODEL ENCOUNTERED AN ERROR AS FOLLOWS:\n\n{{stderr}}</pre>
</div>
{% endif %}
'''
omfRunDebugBlock = Template(runDebugTemplate).render(
modelStatus=modelStatus,
stderr=inJson.get('stderr', ''),
elapsed_min=round(elapsed_min, 2),
remain_min=round(remain_min, 2))
# Raw input output include.
return template.render(allInputData=allInputData,
allOutputData=allOutputData,
modelStatus=modelStatus,
pathPrefix=pathPrefix,
datastoreNames=datastoreNames,
modelName=modelType,
allInputDataDict=inJson,
allOutputDataDict=outJson,
rawOutputFiles=rawOutputFiles,
omfModelButtons=omfModelButtons,
omfRunDebugBlock=omfRunDebugBlock)