Beispiel #1
0
	def __init__(self):
		self.matrices = MatrixManager.MatrixManager(Config)
		self.statStore = StatManager()
		self.mapGenerator = MapGenerator()
		self.mapGenerator.start()
		self.archiver = MapArchiver()
		self.archiver.start()
Beispiel #2
0
class ModelManager():

	matrices = None #matrix manager
	local = True
	STEPCOMPLETE = "stepcomplete"
	STEPWRITTEN = "write"
	MODELCOMPLETE = "modelcomplete"

	def __init__(self):
		self.matrices = MatrixManager.MatrixManager(Config)
		self.statStore = StatManager()
		self.mapGenerator = MapGenerator()
		self.mapGenerator.start()
		self.archiver = MapArchiver()
		self.archiver.start()

	def __del__(self):
		self.mapGenerator.stop()
		self.archiver.stop()

	def stop(self):
		self.mapGenerator.stop()
		self.archiver.stop()

	#run the model with a specific identifier
	def runModel(self, ids):
		#default id to current if non existant
		if (ids == None):
			id = self.getCurrentID(True)
			log.error('Identifier Defaulted to: ' + str(id))

		try:
			self.__setupDirectory(str(ids["run"]))
		except DirectoryExists:
			log.info('directory '+str(ids["run"])+' already exists')
			if not Config.debug:
				return 0 #mark as completed already

		self.__writeInputFiles(ids)
		log.debug('Input files written')
		self.__setupParamaterFile(ids)
		log.debug('Paramater files written')

		# #run model
		self.__runModelJob(ids)

		return 1

	def getState(self,id):
		pass

	def getCurrentID(self, force=False):
		return self.matrices.checkIdentity(force)

	def emit(self,message):
		if self.emitCallback:
			self.emitCallback(message)

	def onMessage(self, func):
		self.emitCallback = func


	#run the model executable and process output
	def __runModelJob(self, ids):
		log.info('Model Kicked off')
		path = Config.runPath + '/' + ids["run"]
		#outputPath = Config.rasterPath + '/' + ids["run"]
		outputPath = path + "/aggs/"
		print "outputpath: " + outputPath
		inputPath = path + '/'
		paramPath = path + '/params.txt'

		#set prevention and protection status
		self.statStore.updateTime(ids["species"],ids["timeline"],0,{"prevention":ids["prevention"], "protection":ids["protection"]})

		#if running modelling locally
		if (self.local):
			cmd = "stdbuf -oL " + Config.migExecutable + " " + paramPath + " " + inputPath + " " + outputPath

			print cmd

			#run process
			proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, bufsize=0, shell=True)

			#capture output
			while True:
				output = proc.stdout.readline()

				#exit if process finished
				if output == '' and proc.poll() is not None:
					self.emit('{"event":"model_complete"}')
					break

				#process output
				if output:
					#print output.strip()
					result = self.processOutput(output.strip())
					if result:
						time = result[1]
						speciesID = ids["species"]
						timelineID = ids["timeline"]

						if result[0] == self.STEPCOMPLETE:
							#update stats
							print result[2]
							self.statStore.updateTime(speciesID,timelineID,time,result[2])

							#emit
							state = self.statStore.getTimeline(speciesID,timelineID)
							self.emit('{"event":"timeline_state","data":{"speciesID":"'+speciesID+'","timelineID":"'+timelineID+'","state":'+state+'}}')

						if result[0] == self.STEPWRITTEN:
							runFile = inputPath + "aggs/agg" + time + ".tif"
							renderedFile = Config.rasterPath + ids["run"] + "/agg" + time + ".tif"

							#render map
							#self.mapGenerator.renderFile(runFile,renderedFile)
							self.archiver.convertFile(runFile,renderedFile)

							#emit
							self.emit('{"event":"time_rendered","data":{"speciesID":"'+speciesID+'","timelineID":"'+timelineID+'","time":'+time+'}}')

						if result[0] == self.MODELCOMPLETE:
							pass
							#self.emit('{"event":"model_complete"}')
			#cleanup
			self.cleanupDirectory(path)

		else: #running remotely using dispy
			jobs = []

			cluster = dispy.JobCluster(Config.migExecutable)

			job = cluster.submit(paramPath, inputPath, outputPath);
			n = job()
			log.debug('job %s at %s with %s' % (job.id, job.start_time, n))
			log.info(job.stdout)

	def cleanupDirectory(self,path):
		pass
		os.remove(path + "/ma.tif")
		os.remove(path + "/max_pre1.tif")
		os.remove(path + "/dist_p.tif")

	def processOutput(self,output):
		line = output.lower()

		#process file write completion
		if line.startswith(self.STEPWRITTEN):
			if (":" in line):
				parts = line.split(":",1)

				if len(parts) == 2:
					#determine year number
					fileNumber = parts[0].replace(self.STEPWRITTEN,"").strip(" ")

					#process output
					statsJson = parts[1].strip()
					stats = json.loads(statsJson)
					# print statsJson

					return [self.STEPWRITTEN,fileNumber, stats]

		if line.startswith(self.STEPCOMPLETE):
			if (":" in line):
				parts = line.split(":",1)

				if len(parts) == 2:
					#determine year number
					fileNumber = parts[0].replace(self.STEPCOMPLETE,"").strip(" ")

					#process output
					statsJson = parts[1].strip()
					stats = json.loads(statsJson)

					return [self.STEPCOMPLETE,fileNumber, stats]
		if line.startswith(self.MODELCOMPLETE):
			return [self.MODELCOMPLETE,0]

	def getDataPath(self):
		return Config.dataPath + 'siam'

	def __setupParamaterFile(self, settings):
		newPath = Config.runPath + '/' + settings['run'] + '/params.txt'
		fInit = open (Config.paramsFilePath, "r")
		fNew = open(newPath, "w")

		print settings

		for line in fInit:
			if (line.startswith("fullOutput")):
				if (settings["full"] == "1"):
					fNew.write("fullOutput true\n")
				if (settings["full"] == "0"):
					fNew.write("fullOutput false\n")
			elif (line.startswith("lddFreq")):
				print settings["prevention"]
				if (settings["prevention"] == "20"):
					fNew.write("lddFreq 0.05\n")
				if (settings["prevention"] == "10"):
					fNew.write("lddFreq 0.08\n")
				if (settings["prevention"] == "0"):
					fNew.write("lddFreq 0.1\n")
			else:
				fNew.write(line)

		fInit.close()
		fNew.close()

	def __setupDirectory(self, id):
		path = Config.runPath + '/'+str(id)
		if not os.path.exists(path):
			os.makedirs(path)
			outputPath = path + Config.aggregatesPath
			if not os.path.exists(outputPath):
				os.makedirs(outputPath)
		else:
			raise DirectoryExists

	def __writeInputFiles(self, ids):
		dest = Config.runPath + '/' + str(ids['run'])

		#write out distribution files
		src = Config.initialFiles + '/dist_p.asc'
		dmDest = dest + '/dist_p.asc'

		#run async, export current distribution
		distTifPath = dest+'/dist_p.tif'
		distAscPath = dest+"/dist_p.asc"
		distCmd = 'gdal_rasterize -burn 1 -a_nodata 0 -init 0 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM distribution WHERE species = \'siam\'" -of GTiff '+distTifPath
		#distCmd += " ; "
		#distCmd += "gdal_translate -a_nodata -9999 -of AAIGrid "+distTifPath+" "+distAscPath
		proc = subprocess.Popen(distCmd, stdout=subprocess.PIPE, bufsize=0, shell=True)

		#rasterize Control Mechanisms
		adjustTifPath = dest+'/ma.tif'
		status = subprocess.call('gdal_rasterize -a "controlMechanism" -a_nodata -9999 -init -9999 -te 143.9165675170000043 -20.0251072599999986 147.0005675170000075 -14.9801072599999987 -ts 3084 5045 -a_srs "EPSG:3857" PG:"host=localhost user=postgres dbname=nyc password=password1" -sql "SELECT * FROM nyc_buildings WHERE timeline = \''+ids['timeline']+'\'" -of GTiff '+adjustTifPath, shell=True)

		#translate to AAIGrid
		#maxPath = dest+'/ma.asc'
		#status = subprocess.call('gdal_translate -a_nodata -9999 -of AAIGrid '+adjustTifPath+' '+maxPath, shell=True)
		#
		# #cleanup intermediary files
		# os.remove(mergedTifPath)
		#os.remove(adjustTifPath)

		#wait for distribution to finish
		while True:
			output = proc.stdout.readline()
			if output == '' and proc.poll() is not None:
				print "finished"
				break
			if output:
				print output

		#cleanup intermediary file
		#os.remove(distTifPath)

		#copy initial distribution map over
		src = Config.initialFiles + '/max_pre1.tif'
		dmDest = dest + '/max_pre1.tif'
		shutil.copyfile(src,dmDest)