def __init__(self): self.matrices = MatrixManager.MatrixManager(Config) self.statStore = StatManager() self.mapGenerator = MapGenerator() self.mapGenerator.start() self.archiver = MapArchiver() self.archiver.start()
class ModelManager():
matrices = None #matrix manager
local = True
STEPCOMPLETE = "stepcomplete"
STEPWRITTEN = "write"
MODELCOMPLETE = "modelcomplete"
def __init__(self):
self.matrices = MatrixManager.MatrixManager(Config)
self.statStore = StatManager()
self.mapGenerator = MapGenerator()
self.mapGenerator.start()
self.archiver = MapArchiver()
self.archiver.start()
def __del__(self):
self.mapGenerator.stop()
self.archiver.stop()
def stop(self):
self.mapGenerator.stop()
self.archiver.stop()
#run the model with a specific identifier
def runModel(self, ids):
#default id to current if non existant
if (ids == None):
id = self.getCurrentID(True)
log.error('Identifier Defaulted to: ' + str(id))
try:
self.__setupDirectory(str(ids["run"]))
except DirectoryExists:
log.info('directory '+str(ids["run"])+' already exists')
if not Config.debug:
return 0 #mark as completed already
self.__writeInputFiles(ids)
log.debug('Input files written')
self.__setupParamaterFile(ids)
log.debug('Paramater files written')
# #run model
self.__runModelJob(ids)
return 1
def getState(self,id):
pass
def getCurrentID(self, force=False):
return self.matrices.checkIdentity(force)
def emit(self,message):
if self.emitCallback:
self.emitCallback(message)
def onMessage(self, func):
self.emitCallback = func
#run the model executable and process output
def __runModelJob(self, ids):
log.info('Model Kicked off')
path = Config.runPath + '/' + ids["run"]
#outputPath = Config.rasterPath + '/' + ids["run"]
outputPath = path + "/aggs/"
print "outputpath: " + outputPath
inputPath = path + '/'
paramPath = path + '/params.txt'
#set prevention and protection status
self.statStore.updateTime(ids["species"],ids["timeline"],0,{"prevention":ids["prevention"], "protection":ids["protection"]})
#if running modelling locally
if (self.local):
cmd = "stdbuf -oL " + Config.migExecutable + " " + paramPath + " " + inputPath + " " + outputPath
print cmd
#run process
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=0, shell=True)
#capture output
while True:
output = proc.stdout.readline()
#exit if process finished
if output == '' and proc.poll() is not None:
self.emit('{"event":"model_complete"}')
break
#process output
if output:
#print output.strip()
result = self.processOutput(output.strip())
if result:
time = result[1]
speciesID = ids["species"]
timelineID = ids["timeline"]
if result[0] == self.STEPCOMPLETE:
#update stats
print result[2]
self.statStore.updateTime(speciesID,timelineID,time,result[2])
#emit
state = self.statStore.getTimeline(speciesID,timelineID)
self.emit('{"event":"timeline_state","data":{"speciesID":"'+speciesID+'","timelineID":"'+timelineID+'","state":'+state+'}}')
if result[0] == self.STEPWRITTEN:
runFile = inputPath + "aggs/agg" + time + ".tif"
renderedFile = Config.rasterPath + ids["run"] + "/agg" + time + ".tif"
#render map
#self.mapGenerator.renderFile(runFile,renderedFile)
self.archiver.convertFile(runFile,renderedFile)
#emit
self.emit('{"event":"time_rendered","data":{"speciesID":"'+speciesID+'","timelineID":"'+timelineID+'","time":'+time+'}}')
if result[0] == self.MODELCOMPLETE:
pass
#self.emit('{"event":"model_complete"}')
#cleanup
self.cleanupDirectory(path)
else: #running remotely using dispy
jobs = []
cluster = dispy.JobCluster(Config.migExecutable)
job = cluster.submit(paramPath, inputPath, outputPath);
n = job()
log.debug('job %s at %s with %s' % (job.id, job.start_time, n))
log.info(job.stdout)
def cleanupDirectory(self,path):
pass
os.remove(path + "/ma.tif")
os.remove(path + "/max_pre1.tif")
os.remove(path + "/dist_p.tif")
def processOutput(self,output):
line = output.lower()
#process file write completion
if line.startswith(self.STEPWRITTEN):
if (":" in line):
parts = line.split(":",1)
if len(parts) == 2:
#determine year number
fileNumber = parts[0].replace(self.STEPWRITTEN,"").strip(" ")
#process output
statsJson = parts[1].strip()
stats = json.loads(statsJson)
# print statsJson
return [self.STEPWRITTEN,fileNumber, stats]
if line.startswith(self.STEPCOMPLETE):
if (":" in line):
parts = line.split(":",1)
if len(parts) == 2:
#determine year number
fileNumber = parts[0].replace(self.STEPCOMPLETE,"").strip(" ")
#process output
statsJson = parts[1].strip()
stats = json.loads(statsJson)
return [self.STEPCOMPLETE,fileNumber, stats]
if line.startswith(self.MODELCOMPLETE):
return [self.MODELCOMPLETE,0]
def getDataPath(self):
return Config.dataPath + 'siam'
def __setupParamaterFile(self, settings):
newPath = Config.runPath + '/' + settings['run'] + '/params.txt'
fInit = open (Config.paramsFilePath, "r")
fNew = open(newPath, "w")
print settings
for line in fInit:
if (line.startswith("fullOutput")):
if (settings["full"] == "1"):
fNew.write("fullOutput true\n")
if (settings["full"] == "0"):
fNew.write("fullOutput false\n")
elif (line.startswith("lddFreq")):
print settings["prevention"]
if (settings["prevention"] == "20"):
fNew.write("lddFreq 0.05\n")
if (settings["prevention"] == "10"):
fNew.write("lddFreq 0.08\n")
if (settings["prevention"] == "0"):
fNew.write("lddFreq 0.1\n")
else:
fNew.write(line)
fInit.close()
fNew.close()
def __setupDirectory(self, id):
path = Config.runPath + '/'+str(id)
if not os.path.exists(path):
os.makedirs(path)
outputPath = path + Config.aggregatesPath
if not os.path.exists(outputPath):
os.makedirs(outputPath)
else:
raise DirectoryExists
def __writeInputFiles(self, ids):
dest = Config.runPath + '/' + str(ids['run'])
#write out distribution files
src = Config.initialFiles + '/dist_p.asc'
dmDest = dest + '/dist_p.asc'
#run async, export current distribution
distTifPath = dest+'/dist_p.tif'
distAscPath = dest+"/dist_p.asc"
distCmd = 'gdal_rasterize -burn 1 -a_nodata 0 -init 0 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM distribution WHERE species = \'siam\'" -of GTiff '+distTifPath
#distCmd += " ; "
#distCmd += "gdal_translate -a_nodata -9999 -of AAIGrid "+distTifPath+" "+distAscPath
proc = subprocess.Popen(distCmd, stdout=subprocess.PIPE, bufsize=0, shell=True)
#rasterize Control Mechanisms
adjustTifPath = dest+'/ma.tif'
status = subprocess.call('gdal_rasterize -a "controlMechanism" -a_nodata -9999 -init -9999 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM nyc_buildings WHERE timeline = \''+ids['timeline']+'\'" -of GTiff '+adjustTifPath, shell=True)
#translate to AAIGrid
#maxPath = dest+'/ma.asc'
#status = subprocess.call('gdal_translate -a_nodata -9999 -of AAIGrid '+adjustTifPath+' '+maxPath, shell=True)
#
# #cleanup intermediary files
# os.remove(mergedTifPath)
#os.remove(adjustTifPath)
#wait for distribution to finish
while True:
output = proc.stdout.readline()
if output == '' and proc.poll() is not None:
print "finished"
break
if output:
print output
#cleanup intermediary file
#os.remove(distTifPath)
#copy initial distribution map over
src = Config.initialFiles + '/max_pre1.tif'
dmDest = dest + '/max_pre1.tif'
shutil.copyfile(src,dmDest)