Exemple #1
0
	def run(self,Test=True, Verbose=False, UseGtlike=False):

		# Convert everything to a float
		ra = float(self.xref)
		dec = float(self.yref)
		nxdeg = float(self.nxdeg)
		nydeg = float(self.nydeg)
		binsize = float(self.binsz)

		# Get the current working directory
		WorkingDirectory = os.getcwd()

		# Define the output, log, and job directories
		if self.outdir == None:
			OutputDirectory = WorkingDirectory
			LogDirectory = "%s/dtsmap/" % WorkingDirectory
			JobsDirectory = "%s/dtsmap/" % WorkingDirectory
		else:
			OutputDirectory = self.outdir
			LogDirectory = self.outdir
			JobsDirectory = self.outdir

		# Define the directory where this script is located
		try:
			ScriptDirectory = os.path.dirname(os.path.realpath(__file__))
		except:
			ScriptDirectory = None

		# Creating the necessary directories
		if(os.path.isdir(OutputDirectory)==False):
			print "\nCreating Directory: " + OutputDirectory
			cmd = "mkdir " + OutputDirectory
			os.system(cmd)
			
		if(os.path.isdir(LogDirectory)==False):
			print "\nCreating Directory: " + LogDirectory
			cmd = "mkdir " + LogDirectory
			os.system(cmd)	

		if(os.path.isdir(JobsDirectory)==False):
			print "\nCreating Directory: " + JobsDirectory
			cmd = "mkdir " + JobsDirectory
			os.system(cmd)

		# Calculate tsmap grid 
		ra_min = ra - self.nxdeg/2.0
		ra_max = ra + self.nxdeg/2.0
		dec_min = dec - self.nydeg/2.0
		dec_max = dec + self.nydeg/2.0

		# Calcualate the range of ra and dec values
		ra_range = numpy.arange(ra_min,ra_max+binsize,binsize)
		dec_range = numpy.arange(dec_min,dec_max+binsize,binsize)
		xsize = len(ra_range)
		ysize = len(dec_range)

		# Make sure that we don't have any ra values below zero or greater than 360, they should wrap ra instead.
		for i in range(len(ra_range)):
			if ra_range[i] < 0:
				ra_range[i] = ra_range[i] + 360.0
			if ra_range[i] > 360:
				ra_range[i] = ra_range[i] - 360.0

		# Make sure that we don't have any dec values below or above +/- 90, they should instead wrap in both ra and dec.
		for i in range(len(dec_range)):
			if dec_range[i] < -90:
				dec_range[i] = ((dec_range[i] + 90) + 90)*-1
				ra_range[i] = ra_range[i] + 180.0
			if dec_range[i] > 90:
				dec_range[i] = 90 - (dec_range[i] - 90)
				ra_range[i] = ra_range[i] + 180


				
		print "\nTS map grid size: %s x %s" % (len(ra_range), len(dec_range))
		print "RA: %s to %s, Dec: %s to %s" % (ra_range[-1], ra_range[0], dec_range[0], dec_range[-1])
		print "Bin size: %s deg" % binsize
		print " "
		print "(%.2f, %.2f)   (%.2f, %.2f)" % (ra_range[-1], dec_range[-1], ra_range[0],  dec_range[-1])
		print " -----------------------------"
		print " |                           |"
		print " |                           |"
		print " |                           |"
		print " |                           |" 
		print " |       (%.2f, %.2f)      |" % (float(ra), float(dec))
		print " |             x             |"
		print " |                           |"
		print " |                           |"
		print " |                           |"
		print " |                           |"					
		print " -----------------------------"
		print "(%.2f, %.2f)    (%.2f, %.2f)" % (ra_range[-1], dec_range[0], ra_range[0],  dec_range[-1])


		print "\nSubmitting a total of %s Jobs" % (xsize * ysize)

		# Mark the start time
		startTime = time.time()
		
		# Loop through all combinations of Ra and Dec and submit a job for each one
		RaDecPairs = {}
		binNumbers = []
		binNumber = 0

		# Keep track of the jobs submitted. This allows the user to specify how many jobs should be running at any given time.
		JobsInQueue = 0

		for raStep in ra_range:
			for decStep in dec_range:

				# Check to see if the number of jobs in the queue is greater than the max.  If so, wait for the jobs to leave the queue before submitting more.
				while JobsInQueue >= int(self.maxJobs):

					print "\nMaximum number of submitted jobs (%s) reached.  Waiting..." % self.maxJobs
					print "Total number of remaining jobs to submit: %s" % remainingJobs

					# Wait 30 seconds before polling the job statuses
					time.sleep(60)

					# Get the number of jobs actively in the queue
					command = "bjobs -g %s | wc" % JobsDirectory	
					process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
					lines = process.stdout.readlines()
					
					# Extract the number of jobs that are running
					JobsInQueue = int(lines[0].split()[0])
						
				# Setup the job
				logfile = LogDirectory + "/dtsmap_bin%s.log" % binNumber

				# Make the job name
				try:
					sourceNumber = int(self.outdir.split('/')[-1].replace('dtsmap_Source',''))
					jobName = "dts_S%sb%s" % (sourceNumber, binNumber)
				except:
					jobName = "dtsmap_b%s" % binNumber
				
				# Construct the command		
				command = """dtsmap.py scfile=%s evfile=%s expmap=%s expcube=%s cmap=%s srcmdl=%s irfs=%s source=%s optimizer=%s ftol=%s nxdeg=%s nydeg=%s binsz=%s coordsys=%s xref=%s yref=%s proj=%s statistic=%s binNumber=%s outfile=%s outdir=%s PerformAnalysis=1""" % (self.scfile, 
				 			 	self.evfile, self.expmap, self.expcube, self.cmap, self.srcmdl, self.irfs, self.source, self.optimizer, self.ftol, self.nxdeg, 
				 			 	self.nydeg, self.binsz, self.coordsys, raStep, decStep, self.proj, self.statistic, binNumber, self.outfile, OutputDirectory)

				# Specify where to find the python script
				if ScriptDirectory != None:
					command = ScriptDirectory + "/" + command 

				# Construct the process call
				process = 'bsub -o ' + logfile + ' -J ' + jobName + ' -q xlong -R rhel60 -g ' + JobsDirectory + ' "' + command + '"'
			
				# Display the command
				if Verbose == True:
					print process

				# Start the process
				if Test == False:
					if self.batch == True:
						subprocess.call(process, shell=True)
					else:
						os.system(command)

				# Put the ra, dec, and bin info in a dictionary for easy retrieval later
				RaDecPairs[binNumber] = [raStep,decStep]

				# Increment the bin number
				binNumbers.append(binNumber)
				binNumber = binNumber + 1

				# Increment the bin number
				JobsInQueue = JobsInQueue + 1

				# Get the number of remaining jobs
				remainingJobs = ((xsize * ysize) - binNumber)


		print "\nTotal number of jobs submitted: %s" % binNumber
		print "All jobs submitted."
		
		nJobs = -1
		while nJobs != 0:		

			# Determine the number of jobs still running
			command = "bjobs -g %s | wc" % JobsDirectory	
			process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
			lines = process.stdout.readlines()
		
			# Display the results and the elapsed time since the start of the analysis
			if 'No unfinished job found' in lines[0]:
				nJobs = 0
				print '\nAll Jobs Complete.'

				# Display the elapsed time
				elapsedTimeSeconds = time.time() - startTime
				elapsedTimeMinutes = elapsedTimeSeconds/60.0
				elapsedTimeHours = elapsedTimeMinutes/60.0

				# Modify the units depending on the amount of time that has elapsed
				if elapsedTimeMinutes > 60:				
					print "Total Elapsed Time: %.2f Hours" % elapsedTimeHours
				else:
					print "Total Elapsed Time: %.2f Minutes" % elapsedTimeMinutes

				# Make sure we actually break out of this while loop!
				break

			else:
				
				# Get the number of jobs in the queue from the stdout
				nJobs = int(lines[0].split()[0])
				print "\nJobs Remaining: %s" % nJobs
				
				# Display the elapsed time
				elapsedTimeSeconds = time.time() - startTime
				elapsedTimeMinutes = elapsedTimeSeconds/60.0
				elapsedTimeHours = elapsedTimeMinutes/60.0

				if elapsedTimeMinutes > 60:				
					print "Elapsed Time: %.2f Hours" % elapsedTimeHours
				else:
					print "Elapsed Time: %.2f Minutes" % elapsedTimeMinutes 

			# Wait 120 seconds before polling the job statuses
			time.sleep(120)

		if UseGtlike == True:
			# Gather the results (gtlike)
			command = "grep 'TS value' %s/likelihoodResults_bin*.txt" % LogDirectory	
			process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
			lines = process.stdout.readlines()
		else:
			# Gather the results (pylikelihood)
			command = "grep 'TS = ' %s/dtsmap_bin*.log" % LogDirectory	
			process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
			lines = process.stdout.readlines()

		# Read in the results
		binNumbersReadIn = []
		TSs = []
		for line in lines:
			if ('No match' in line) or ('grep' in line):
				print "Error: *** No available likelihood results ***"
				print "Check your science tools setup and try again..."
				print "Exiting.\n"

				Results = {'MaxTS':None, 'MaxRA':None, 'MaxDec':None, 'Error':None, 'Index':None, 'IndexError':None, 'Flux':None,'FluxError':None, 'MaxBin':None}
				return Results
			else:

				if UseGtlike == True:

					# For use with gtlike results
					binNumber = int(line.split()[0].split('/')[-1].replace('.txt:\'TS','').replace('likelihoodResults_bin','').strip())
					TS = float("%.2f" % float(line.split()[2].strip().replace("'","").replace(',','')))
					binNumbersReadIn.append(binNumber)
					TSs.append(TS)

				else:

					# For use with pylikelihood results
					binNumber = int(line.split()[0].split('/')[-1].replace('.log:TS','').replace('dtsmap_bin','').strip())
					TS = float("%.2f" % float(line.split()[2].strip().replace("'","").replace(',','')))
					binNumbersReadIn.append(binNumber)
					TSs.append(TS)


		# Put the results in a dictionary for easy retrieval later
		LikelihoodResults = {key:value for key, value in zip(binNumbersReadIn,TSs)}

		# Make an matrix to store the values.  Saving any missing values as NaNs
		TSMap = numpy.zeros(shape=(xsize, ysize))
		binNumber = 0
		xStep = 0
		yStep = 0
		for raStep in ra_range:
			yStep = 0
			for decStep in dec_range:
				if binNumber in LikelihoodResults:
					TSMap[xStep][yStep] = LikelihoodResults[binNumber]
					pass
				else:
					print "Bad bin: %s" % binNumber
					TSMap[xStep][yStep] = numpy.nan
					pass
				if TSMap[xStep][yStep] < -1:
					TSMap[xStep][yStep] = numpy.nan
					pass
				binNumber = binNumber + 1
				yStep = yStep + 1
				pass
			xStep = xStep + 1
			pass


		# # Loop through the results matrix and fill any missing values with interpolations from nearby pixels
		# binNumber = 0
		# xStep = 0
		# yStep = 0
		# for raStep in ra_range:
		# 	yStep = 0
		# 	for decStep in dec_range:
		# 		if numpy.isnan(TSMap[xStep][yStep]) == True:

		# #			try:
		# #				TSMap[xStep][yStep] = numpy.mean([TSMap[xStep-1,yStep-1],TSMap[xStep-1,yStep],TSMap[xStep-1,yStep+1],TSMap[xStep,yStep-1],TSMap[xStep,yStep+1],TSMap[xStep+1,yStep-1],TSMap[xStep+1,yStep],TSMap[xStep+1,yStep+1]])
		# 			try:					
		# 				if numpy.isnan(TSMap[xStep-1][yStep]) == False and  numpy.isnan(TSMap[xStep+1][yStep]) == False:
		# 					TSMap[xStep][yStep] = (TSMap[xStep-1][yStep] + TSMap[xStep+1][yStep]) / 2.0
		# 				elif numpy.isnan(TSMap[xStep][yStep-1]) == False and  numpy.isnan(TSMap[xStep][yStep+1]) == False:
		# 					TSMap[xStep][yStep] = (TSMap[xStep][yStep-1] + TSMap[xStep][yStep+1]) / 2.0
		# 				else:
		# 					TSMap[xStep][yStep] = numpy.nan
		# 			except:
		# 				TSMap[xStep][yStep] = numpy.nan

		# 		yStep = yStep + 1
		# 		pass
		# 	xStep = xStep + 1

		# Finding the maximum TS
		MaxTS = numpy.nanmax(TSMap)
		MaxBin = numpy.nanargmax(TSMap)

		# Check to see if a maximum couldn't be found.
		if numpy.isnan(MaxBin) == True:
			print "\nAnalysis Complete."
			print "Maximum TS: %s" % 'None'
			print "Coordinates: RA = %s, Dec = %s" % ('NA', 'NA')
			print "*** Unable to locate maximum TS ***"

			print '\nCleaning up...'
			# Cat the gtlike log files
			cmd = "cat %s/likelihoodResults_bin*.txt > dtsmap_LikelihoodResults.txt" % self.outdir
#			print cmd
			os.system(cmd)

			# Cat the log files
			cmd = "cat %s/dtsmap_bin*.log > dtsmap_LikelihoodFits.log" % self.outdir
#			print cmd
			os.system(cmd)

			# Erase the individual xml files
			cmd = "rm %s/ModelSource_bin*.xml" % self.outdir
#			print cmd
			os.system(cmd)

			print 'Done.'

			Results = {'MaxTS':None, 'MaxRA':None, 'MaxDec':None, 'Error':None, 'Index':None, 'IndexError':None, 'Flux':None,'FluxError':None, 'MaxBin':None}
			return Results

		else:

			MaxRa = RaDecPairs[MaxBin][0]
			MaxDec = RaDecPairs[MaxBin][1]

		# Define default spectral parameters
		index = 'NA'
		indexError = 'NA'
		flux = 'NA'
		fluxError = 'NA'

		if UseGtlike == True:

			# Extract the fit parameters for the bin with the maximum TS (gtlike)
			MaxBinFile = "%s/dtsmap_bin%s.log" % (LogDirectory, MaxBin)
			for line in fileinput.input([MaxBinFile]):
				if 'Index:' in line: 
					lineContents = line.split()	
					index = lineContents[1]
					indexError = lineContents[3]
				if 'Flux:' in line: 
					lineContents = line.split()	
					flux = lineContents[1]
					fluxError = lineContents[3]
					break
			fileinput.close()

		else:

			# Extract the spectral fit parameters for the bin with the maximum TS (pyLikelihood)
			MaxBinFile = "%s/dtsmap_bin%s.log" % (LogDirectory, MaxBin)
			for line in fileinput.input([MaxBinFile]):
				if 'Flux =' in line: 
					lineContents = line.split()	
					flux = float(lineContents[2])
					fluxError = float(lineContents[4])
				if 'Index =' in line: 
					lineContents = line.split()	
					index = float(lineContents[2])
					indexError = float(lineContents[4])
			fileinput.close()


		# Rotate and flip the matrix in order to it to match ra and dec ordering conventions
		TSMapRotated = numpy.rot90(TSMap)
		TSMapFlippedUp = numpy.flipud(TSMapRotated)
		TSMapFlippedLeft2Right = numpy.fliplr(TSMapFlippedUp)
		MaxRa = RaDecPairs[numpy.nanargmax(TSMap)][0]
		MaxDec = RaDecPairs[numpy.nanargmax(TSMap)][1]

		# Import the basemap module
		sys.path.append("/nfs/slac/g/ki/ki08/kocevski/LATBA/lib/python_rhel6-64/")
		from mpl_toolkits.basemap import Basemap

		# Create the figure
		fig = plot.figure()
		ax = fig.add_subplot(111)

		# Create a base map on which to plot the results
		#m = Basemap(llcrnrlon=ra_range[-1], llcrnrlat=dec_range[0], urcrnrlon=ra_range[0], urcrnrlat=dec_range[-1], projection='laea', lon_0 = ra, lat_0 = dec,resolution ='l',area_thresh=1000.)# ,celestial=True )
		m = Basemap(height=5.5e5,width=5.5e5, projection='laea', lon_0 = ra*-1, lat_0 = dec, resolution ='l', area_thresh=1000., celestial=True )

		# Set the plot limits (in map coordinates)
		xMin, yMin = m(ra_range[0], dec_range[0])
		xMax, yMax = m(ra_range[-1], dec_range[-1])
		m.lonmin = xMin
		m.lonmax = xMax
		m.latmin = yMin
		m.latmax = yMax

		# Plot the matrix as an image
		extent=[xMax, xMin, yMin, yMax]
		#m.imshow(TSMapFlippedLeft2Right, origin='lower', extent=extent)
		m.imshow(TSMapFlippedLeft2Right, origin='lower')

		# Setup the map grid
		m.drawmapboundary(fill_color='#ffffff')
		m.drawparallels(numpy.arange(181)-90,labels=[1,0,0,0], fmt=customDecLabel, color='grey', linewidth=0)
		m.drawmeridians(numpy.arange(0,360,1),labels=[0,0,0,1], fmt=customRALabel, color='grey', linewidth=0)
		m.suppress_ticks = False
		m.fix_aspect = False
	
		# Force the aspect ratio to be 1:1
		try:
			forceAspect(ax,aspect=1)
			#extent=[xMax, xMin, yMin, yMax]
			#ax.set_aspect(abs((extent[1]-extent[0])/(extent[3]-extent[2])))
		except Exception, message:
			print traceback.format_exc()
Exemple #2
0
    def run(self, Test=True, Verbose=False, UseGtlike=False):

        # Convert everything to a float
        ra = float(self.xref)
        dec = float(self.yref)
        nxdeg = float(self.nxdeg)
        nydeg = float(self.nydeg)
        binsize = float(self.binsz)

        # Get the current working directory
        WorkingDirectory = os.getcwd()

        # Define the output, log, and job directories
        if self.outdir == None:
            OutputDirectory = WorkingDirectory
            LogDirectory = "%s/dtsmap/" % WorkingDirectory
            JobsDirectory = "%s/dtsmap/" % WorkingDirectory
        else:
            OutputDirectory = self.outdir
            LogDirectory = self.outdir
            JobsDirectory = self.outdir

        # Define the directory where this script is located
        try:
            ScriptDirectory = os.path.dirname(os.path.realpath(__file__))
        except:
            ScriptDirectory = None

        # Creating the necessary directories
        if (os.path.isdir(OutputDirectory) == False):
            print "\nCreating Directory: " + OutputDirectory
            cmd = "mkdir " + OutputDirectory
            os.system(cmd)

        if (os.path.isdir(LogDirectory) == False):
            print "\nCreating Directory: " + LogDirectory
            cmd = "mkdir " + LogDirectory
            os.system(cmd)

        if (os.path.isdir(JobsDirectory) == False):
            print "\nCreating Directory: " + JobsDirectory
            cmd = "mkdir " + JobsDirectory
            os.system(cmd)

        # Calculate tsmap grid
        ra_min = ra - self.nxdeg / 2.0
        ra_max = ra + self.nxdeg / 2.0
        dec_min = dec - self.nydeg / 2.0
        dec_max = dec + self.nydeg / 2.0

        # Calcualate the range of ra and dec values
        ra_range = numpy.arange(ra_min, ra_max + binsize, binsize)
        dec_range = numpy.arange(dec_min, dec_max + binsize, binsize)
        xsize = len(ra_range)
        ysize = len(dec_range)

        # Make sure that we don't have any ra values below zero or greater than 360, they should wrap ra instead.
        for i in range(len(ra_range)):
            if ra_range[i] < 0:
                ra_range[i] = ra_range[i] + 360.0
            if ra_range[i] > 360:
                ra_range[i] = ra_range[i] - 360.0

        # Make sure that we don't have any dec values below or above +/- 90, they should instead wrap in both ra and dec.
        for i in range(len(dec_range)):
            if dec_range[i] < -90:
                dec_range[i] = ((dec_range[i] + 90) + 90) * -1
                ra_range[i] = ra_range[i] + 180.0
            if dec_range[i] > 90:
                dec_range[i] = 90 - (dec_range[i] - 90)
                ra_range[i] = ra_range[i] + 180

        print "\nTS map grid size: %s x %s" % (len(ra_range), len(dec_range))
        print "RA: %s to %s, Dec: %s to %s" % (ra_range[-1], ra_range[0],
                                               dec_range[0], dec_range[-1])
        print "Bin size: %s deg" % binsize
        print " "
        print "(%.2f, %.2f)   (%.2f, %.2f)" % (ra_range[-1], dec_range[-1],
                                               ra_range[0], dec_range[-1])
        print " -----------------------------"
        print " |                           |"
        print " |                           |"
        print " |                           |"
        print " |                           |"
        print " |       (%.2f, %.2f)      |" % (float(ra), float(dec))
        print " |             x             |"
        print " |                           |"
        print " |                           |"
        print " |                           |"
        print " |                           |"
        print " -----------------------------"
        print "(%.2f, %.2f)    (%.2f, %.2f)" % (ra_range[-1], dec_range[0],
                                                ra_range[0], dec_range[-1])

        print "\nSubmitting a total of %s Jobs" % (xsize * ysize)

        # Mark the start time
        startTime = time.time()

        # Loop through all combinations of Ra and Dec and submit a job for each one
        RaDecPairs = {}
        binNumbers = []
        binNumber = 0

        # Keep track of the jobs submitted. This allows the user to specify how many jobs should be running at any given time.
        JobsInQueue = 0

        for raStep in ra_range:
            for decStep in dec_range:

                # Check to see if the number of jobs in the queue is greater than the max.  If so, wait for the jobs to leave the queue before submitting more.
                while JobsInQueue >= int(self.maxJobs):

                    print "\nMaximum number of submitted jobs (%s) reached.  Waiting..." % self.maxJobs
                    print "Total number of remaining jobs to submit: %s" % remainingJobs

                    # Wait 30 seconds before polling the job statuses
                    time.sleep(60)

                    # Get the number of jobs actively in the queue
                    command = "bjobs -g %s | wc" % JobsDirectory
                    process = subprocess.Popen(command,
                                               shell=True,
                                               stdin=subprocess.PIPE,
                                               stdout=subprocess.PIPE,
                                               stderr=subprocess.STDOUT,
                                               close_fds=True)
                    lines = process.stdout.readlines()

                    # Extract the number of jobs that are running
                    JobsInQueue = int(lines[0].split()[0])

                # Setup the job
                logfile = LogDirectory + "/dtsmap_bin%s.log" % binNumber

                # Make the job name
                try:
                    sourceNumber = int(
                        self.outdir.split('/')[-1].replace(
                            'dtsmap_Source', ''))
                    jobName = "dts_S%sb%s" % (sourceNumber, binNumber)
                except:
                    jobName = "dtsmap_b%s" % binNumber

                # Construct the command
                command = """dtsmap.py scfile=%s evfile=%s expmap=%s expcube=%s cmap=%s srcmdl=%s irfs=%s source=%s optimizer=%s ftol=%s nxdeg=%s nydeg=%s binsz=%s coordsys=%s xref=%s yref=%s proj=%s statistic=%s binNumber=%s outfile=%s outdir=%s PerformAnalysis=1""" % (
                    self.scfile, self.evfile, self.expmap, self.expcube,
                    self.cmap, self.srcmdl, self.irfs, self.source,
                    self.optimizer, self.ftol, self.nxdeg, self.nydeg,
                    self.binsz, self.coordsys, raStep, decStep, self.proj,
                    self.statistic, binNumber, self.outfile, OutputDirectory)

                # Specify where to find the python script
                if ScriptDirectory != None:
                    command = ScriptDirectory + "/" + command

                # Construct the process call
                process = 'bsub -o ' + logfile + ' -J ' + jobName + ' -q xlong -R rhel60 -g ' + JobsDirectory + ' "' + command + '"'

                # Display the command
                if Verbose == True:
                    print process

                # Start the process
                if Test == False:
                    if self.batch == True:
                        subprocess.call(process, shell=True)
                    else:
                        os.system(command)

                # Put the ra, dec, and bin info in a dictionary for easy retrieval later
                RaDecPairs[binNumber] = [raStep, decStep]

                # Increment the bin number
                binNumbers.append(binNumber)
                binNumber = binNumber + 1

                # Increment the bin number
                JobsInQueue = JobsInQueue + 1

                # Get the number of remaining jobs
                remainingJobs = ((xsize * ysize) - binNumber)

        print "\nTotal number of jobs submitted: %s" % binNumber
        print "All jobs submitted."

        nJobs = -1
        while nJobs != 0:

            # Determine the number of jobs still running
            command = "bjobs -g %s | wc" % JobsDirectory
            process = subprocess.Popen(command,
                                       shell=True,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT,
                                       close_fds=True)
            lines = process.stdout.readlines()

            # Display the results and the elapsed time since the start of the analysis
            if 'No unfinished job found' in lines[0]:
                nJobs = 0
                print '\nAll Jobs Complete.'

                # Display the elapsed time
                elapsedTimeSeconds = time.time() - startTime
                elapsedTimeMinutes = elapsedTimeSeconds / 60.0
                elapsedTimeHours = elapsedTimeMinutes / 60.0

                # Modify the units depending on the amount of time that has elapsed
                if elapsedTimeMinutes > 60:
                    print "Total Elapsed Time: %.2f Hours" % elapsedTimeHours
                else:
                    print "Total Elapsed Time: %.2f Minutes" % elapsedTimeMinutes

                # Make sure we actually break out of this while loop!
                break

            else:

                # Get the number of jobs in the queue from the stdout
                nJobs = int(lines[0].split()[0])
                print "\nJobs Remaining: %s" % nJobs

                # Display the elapsed time
                elapsedTimeSeconds = time.time() - startTime
                elapsedTimeMinutes = elapsedTimeSeconds / 60.0
                elapsedTimeHours = elapsedTimeMinutes / 60.0

                if elapsedTimeMinutes > 60:
                    print "Elapsed Time: %.2f Hours" % elapsedTimeHours
                else:
                    print "Elapsed Time: %.2f Minutes" % elapsedTimeMinutes

            # Wait 120 seconds before polling the job statuses
            time.sleep(120)

        if UseGtlike == True:
            # Gather the results (gtlike)
            command = "grep 'TS value' %s/likelihoodResults_bin*.txt" % LogDirectory
            process = subprocess.Popen(command,
                                       shell=True,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT,
                                       close_fds=True)
            lines = process.stdout.readlines()
        else:
            # Gather the results (pylikelihood)
            command = "grep 'TS = ' %s/dtsmap_bin*.log" % LogDirectory
            process = subprocess.Popen(command,
                                       shell=True,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.STDOUT,
                                       close_fds=True)
            lines = process.stdout.readlines()

        # Read in the results
        binNumbersReadIn = []
        TSs = []
        for line in lines:
            if ('No match' in line) or ('grep' in line):
                print "Error: *** No available likelihood results ***"
                print "Check your science tools setup and try again..."
                print "Exiting.\n"

                Results = {
                    'MaxTS': None,
                    'MaxRA': None,
                    'MaxDec': None,
                    'Error': None,
                    'Index': None,
                    'IndexError': None,
                    'Flux': None,
                    'FluxError': None,
                    'MaxBin': None
                }
                return Results
            else:

                if UseGtlike == True:

                    # For use with gtlike results
                    binNumber = int(line.split()[0].split('/')[-1].replace(
                        '.txt:\'TS', '').replace('likelihoodResults_bin',
                                                 '').strip())
                    TS = float("%.2f" % float(line.split()[2].strip().replace(
                        "'", "").replace(',', '')))
                    binNumbersReadIn.append(binNumber)
                    TSs.append(TS)

                else:

                    # For use with pylikelihood results
                    binNumber = int(line.split()[0].split('/')[-1].replace(
                        '.log:TS', '').replace('dtsmap_bin', '').strip())
                    TS = float("%.2f" % float(line.split()[2].strip().replace(
                        "'", "").replace(',', '')))
                    binNumbersReadIn.append(binNumber)
                    TSs.append(TS)

        # Put the results in a dictionary for easy retrieval later
        LikelihoodResults = {
            key: value
            for key, value in zip(binNumbersReadIn, TSs)
        }

        # Make an matrix to store the values.  Saving any missing values as NaNs
        TSMap = numpy.zeros(shape=(xsize, ysize))
        binNumber = 0
        xStep = 0
        yStep = 0
        for raStep in ra_range:
            yStep = 0
            for decStep in dec_range:
                if binNumber in LikelihoodResults:
                    TSMap[xStep][yStep] = LikelihoodResults[binNumber]
                    pass
                else:
                    print "Bad bin: %s" % binNumber
                    TSMap[xStep][yStep] = numpy.nan
                    pass
                if TSMap[xStep][yStep] < -1:
                    TSMap[xStep][yStep] = numpy.nan
                    pass
                binNumber = binNumber + 1
                yStep = yStep + 1
                pass
            xStep = xStep + 1
            pass

        # # Loop through the results matrix and fill any missing values with interpolations from nearby pixels
        # binNumber = 0
        # xStep = 0
        # yStep = 0
        # for raStep in ra_range:
        # 	yStep = 0
        # 	for decStep in dec_range:
        # 		if numpy.isnan(TSMap[xStep][yStep]) == True:

        # #			try:
        # #				TSMap[xStep][yStep] = numpy.mean([TSMap[xStep-1,yStep-1],TSMap[xStep-1,yStep],TSMap[xStep-1,yStep+1],TSMap[xStep,yStep-1],TSMap[xStep,yStep+1],TSMap[xStep+1,yStep-1],TSMap[xStep+1,yStep],TSMap[xStep+1,yStep+1]])
        # 			try:
        # 				if numpy.isnan(TSMap[xStep-1][yStep]) == False and  numpy.isnan(TSMap[xStep+1][yStep]) == False:
        # 					TSMap[xStep][yStep] = (TSMap[xStep-1][yStep] + TSMap[xStep+1][yStep]) / 2.0
        # 				elif numpy.isnan(TSMap[xStep][yStep-1]) == False and  numpy.isnan(TSMap[xStep][yStep+1]) == False:
        # 					TSMap[xStep][yStep] = (TSMap[xStep][yStep-1] + TSMap[xStep][yStep+1]) / 2.0
        # 				else:
        # 					TSMap[xStep][yStep] = numpy.nan
        # 			except:
        # 				TSMap[xStep][yStep] = numpy.nan

        # 		yStep = yStep + 1
        # 		pass
        # 	xStep = xStep + 1

        # Finding the maximum TS
        MaxTS = numpy.nanmax(TSMap)
        MaxBin = numpy.nanargmax(TSMap)

        # Check to see if a maximum couldn't be found.
        if numpy.isnan(MaxBin) == True:
            print "\nAnalysis Complete."
            print "Maximum TS: %s" % 'None'
            print "Coordinates: RA = %s, Dec = %s" % ('NA', 'NA')
            print "*** Unable to locate maximum TS ***"

            print '\nCleaning up...'
            # Cat the gtlike log files
            cmd = "cat %s/likelihoodResults_bin*.txt > dtsmap_LikelihoodResults.txt" % self.outdir
            #			print cmd
            os.system(cmd)

            # Cat the log files
            cmd = "cat %s/dtsmap_bin*.log > dtsmap_LikelihoodFits.log" % self.outdir
            #			print cmd
            os.system(cmd)

            # Erase the individual xml files
            cmd = "rm %s/ModelSource_bin*.xml" % self.outdir
            #			print cmd
            os.system(cmd)

            print 'Done.'

            Results = {
                'MaxTS': None,
                'MaxRA': None,
                'MaxDec': None,
                'Error': None,
                'Index': None,
                'IndexError': None,
                'Flux': None,
                'FluxError': None,
                'MaxBin': None
            }
            return Results

        else:

            MaxRa = RaDecPairs[MaxBin][0]
            MaxDec = RaDecPairs[MaxBin][1]

        # Define default spectral parameters
        index = 'NA'
        indexError = 'NA'
        flux = 'NA'
        fluxError = 'NA'

        if UseGtlike == True:

            # Extract the fit parameters for the bin with the maximum TS (gtlike)
            MaxBinFile = "%s/dtsmap_bin%s.log" % (LogDirectory, MaxBin)
            for line in fileinput.input([MaxBinFile]):
                if 'Index:' in line:
                    lineContents = line.split()
                    index = lineContents[1]
                    indexError = lineContents[3]
                if 'Flux:' in line:
                    lineContents = line.split()
                    flux = lineContents[1]
                    fluxError = lineContents[3]
                    break
            fileinput.close()

        else:

            # Extract the spectral fit parameters for the bin with the maximum TS (pyLikelihood)
            MaxBinFile = "%s/dtsmap_bin%s.log" % (LogDirectory, MaxBin)
            for line in fileinput.input([MaxBinFile]):
                if 'Flux =' in line:
                    lineContents = line.split()
                    flux = float(lineContents[2])
                    fluxError = float(lineContents[4])
                if 'Index =' in line:
                    lineContents = line.split()
                    index = float(lineContents[2])
                    indexError = float(lineContents[4])
            fileinput.close()

        # Rotate and flip the matrix in order to it to match ra and dec ordering conventions
        TSMapRotated = numpy.rot90(TSMap)
        TSMapFlippedUp = numpy.flipud(TSMapRotated)
        TSMapFlippedLeft2Right = numpy.fliplr(TSMapFlippedUp)
        MaxRa = RaDecPairs[numpy.nanargmax(TSMap)][0]
        MaxDec = RaDecPairs[numpy.nanargmax(TSMap)][1]

        # Import the basemap module
        sys.path.append(
            "/nfs/slac/g/ki/ki08/kocevski/LATBA/lib/python_rhel6-64/")
        from mpl_toolkits.basemap import Basemap

        # Create the figure
        fig = plot.figure()
        ax = fig.add_subplot(111)

        # Create a base map on which to plot the results
        #m = Basemap(llcrnrlon=ra_range[-1], llcrnrlat=dec_range[0], urcrnrlon=ra_range[0], urcrnrlat=dec_range[-1], projection='laea', lon_0 = ra, lat_0 = dec,resolution ='l',area_thresh=1000.)# ,celestial=True )
        m = Basemap(height=5.5e5,
                    width=5.5e5,
                    projection='laea',
                    lon_0=ra * -1,
                    lat_0=dec,
                    resolution='l',
                    area_thresh=1000.,
                    celestial=True)

        # Set the plot limits (in map coordinates)
        xMin, yMin = m(ra_range[0], dec_range[0])
        xMax, yMax = m(ra_range[-1], dec_range[-1])
        m.lonmin = xMin
        m.lonmax = xMax
        m.latmin = yMin
        m.latmax = yMax

        # Plot the matrix as an image
        extent = [xMax, xMin, yMin, yMax]
        #m.imshow(TSMapFlippedLeft2Right, origin='lower', extent=extent)
        m.imshow(TSMapFlippedLeft2Right, origin='lower')

        # Setup the map grid
        m.drawmapboundary(fill_color='#ffffff')
        m.drawparallels(numpy.arange(181) - 90,
                        labels=[1, 0, 0, 0],
                        fmt=customDecLabel,
                        color='grey',
                        linewidth=0)
        m.drawmeridians(numpy.arange(0, 360, 1),
                        labels=[0, 0, 0, 1],
                        fmt=customRALabel,
                        color='grey',
                        linewidth=0)
        m.suppress_ticks = False
        m.fix_aspect = False

        # Force the aspect ratio to be 1:1
        try:
            forceAspect(ax, aspect=1)
            #extent=[xMax, xMin, yMin, yMax]
            #ax.set_aspect(abs((extent[1]-extent[0])/(extent[3]-extent[2])))
        except Exception, message:
            print traceback.format_exc()
Exemple #3
0
def Plot(UseGtlike=False):

#	ra = 260.21
#	dec = -77.92
#	binsize = 0.25
#	LogDirectory = '/Users/kocevski/Research/Analysis/FAVA/Results/dtsmap_Source18'

	ra = 99.2
	dec = -3.92
	binsize = 0.25
	LogDirectory = '/Users/kocevski/Research/Analysis/FAVA/Results/dtsmap_Source27'

	nxdeg = 5
	nydeg = 5

	ra_min = ra - nxdeg/2.0
	ra_max = ra + nxdeg/2.0
	dec_min = dec - nydeg/2.0
	dec_max = dec + nydeg/2.0

	# Calcualate the range of ra and dec values
	ra_range = numpy.arange(ra_min,ra_max+binsize,binsize)
	dec_range = numpy.arange(dec_min,dec_max+binsize,binsize)
	xsize = len(ra_range)
	ysize = len(dec_range)

	# Make sure that we don't have any ra values below zero or greater than 360, they should wrap ra instead.
	for i in range(len(ra_range)):
		if ra_range[i] < 0:
			ra_range[i] = ra_range[i] + 360.0
		if ra_range[i] > 360:
			ra_range[i] = ra_range[i] - 360.0

	# Make sure that we don't have any dec values below or above +/- 90, they should instead wrap in both ra and dec.
	for i in range(len(dec_range)):
		if dec_range[i] < -90:
			dec_range[i] = ((dec_range[i] + 90) + 90)*-1
			ra_range[i] = ra_range[i] + 180.0
		if dec_range[i] > 90:
			dec_range[i] = 90 - (dec_range[i] - 90)
			ra_range[i] = ra_range[i] + 180

	RaDecMap = numpy.zeros(shape=(xsize, ysize))
	RaDecMap = RaDecMap.astype(str)
	RaDecPairs = {}
	binNumbers = []
	binNumber = 0
	xStep = 0
	yStep = 0
	for raStep in ra_range:
		yStep = 0
		for decStep in dec_range:
			RaDecPairs[binNumber] = [raStep,decStep]
			RaDecMap[xStep][yStep] = "%s,%s" % (raStep,decStep)
			binNumbers.append(binNumber)
			binNumber = binNumber + 1
			yStep = yStep + 1
			pass
		xStep = xStep + 1
		pass

	# Gather the results (pylikelihood)
	command = "grep 'TS = ' %s/dtsmap_bin*.log" % LogDirectory	
	process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
	lines = process.stdout.readlines()
	
	# Read in the results
	binNumbersReadIn = []
	TSs = []
	for line in lines:
		if ('No match' in line) or ('grep' in line):
			print "Error: *** No available likelihood results ***"
			print "Check your science tools setup and try again..."
			print "Exiting.\n"

			Results = {'MaxTS':None, 'MaxRA':None, 'MaxDec':None, 'Error':None, 'Index':None, 'IndexError':None, 'Flux':None,'FluxError':None, 'MaxBin':None}
			return Results
		else:
			# For use with pylikelihood results
			binNumber = int(line.split()[0].split('/')[-1].replace('.log:TS','').replace('dtsmap_bin','').strip())
			TS = float("%.2f" % float(line.split()[2].strip().replace("'","").replace(',','')))
			binNumbersReadIn.append(binNumber)
			TSs.append(TS)


	# Put the results in a dictionary for easy retrieval later
	LikelihoodResults = {key:value for key, value in zip(binNumbersReadIn,TSs)}

	# Make an matrix to store the values.  Saving any missing values as NaNs
	TSMap = numpy.zeros(shape=(xsize, ysize))
	binNumber = 0
	xStep = 0
	yStep = 0
	for raStep in ra_range:
		yStep = 0
		for decStep in dec_range:
			if binNumber in LikelihoodResults:
				TSMap[xStep][yStep] = LikelihoodResults[binNumber]
				pass
			else:
				print "Bad bin: %s" % binNumber
				TSMap[xStep][yStep] = numpy.nan
				pass
			if TSMap[xStep][yStep] < -1:
				TSMap[xStep][yStep] = numpy.nan
				pass
			binNumber = binNumber + 1
			yStep = yStep + 1
			pass
		xStep = xStep + 1
		pass


	# Loop through the results matrix and fill any missing values with interpolations from nearby pixels
	binNumber = 0
	xStep = 0
	yStep = 0
	for raStep in ra_range:
		yStep = 0
		for decStep in dec_range:
			if numpy.isnan(TSMap[xStep][yStep]) == True:

	#			try:
	#				TSMap[xStep][yStep] = numpy.mean([TSMap[xStep-1,yStep-1],TSMap[xStep-1,yStep],TSMap[xStep-1,yStep+1],TSMap[xStep,yStep-1],TSMap[xStep,yStep+1],TSMap[xStep+1,yStep-1],TSMap[xStep+1,yStep],TSMap[xStep+1,yStep+1]])
				try:					
					if numpy.isnan(TSMap[xStep-1][yStep]) == False and  numpy.isnan(TSMap[xStep+1][yStep]) == False:
						TSMap[xStep][yStep] = (TSMap[xStep-1][yStep] + TSMap[xStep+1][yStep]) / 2.0
					elif numpy.isnan(TSMap[xStep][yStep-1]) == False and  numpy.isnan(TSMap[xStep][yStep+1]) == False:
						TSMap[xStep][yStep] = (TSMap[xStep][yStep-1] + TSMap[xStep][yStep+1]) / 2.0
					else:
						TSMap[xStep][yStep] = numpy.nan
				except:
					TSMap[xStep][yStep] = numpy.nan

			yStep = yStep + 1
			pass
		xStep = xStep + 1

	# Finding the maximum TS
	MaxTS = numpy.nanmax(TSMap)
	MaxBin = numpy.nanargmax(TSMap)

	# Check to see if a maximum couldn't be found.
	if numpy.isnan(MaxBin) == True:
		print "\nAnalysis Complete."
		print "Maximum TS: %s" % 'None'
		print "Coordinates: RA = %s, Dec = %s" % ('NA', 'NA')
		print "*** Unable to locate maximum TS ***"

		print '\nCleaning up...'
		# Cat the gtlike log files
		cmd = "cat %s/likelihoodResults_bin*.txt > dtsmap_LikelihoodResults.txt" % self.outdir
#			print cmd
		os.system(cmd)

		# Cat the log files
		cmd = "cat %s/dtsmap_bin*.log > dtsmap_LikelihoodFits.log" % self.outdir
#			print cmd
		os.system(cmd)

		# Erase the individual xml files
		cmd = "rm %s/ModelSource_bin*.xml" % self.outdir
#			print cmd
		os.system(cmd)

		print 'Done.'

		Results = {'MaxTS':None, 'MaxRA':None, 'MaxDec':None, 'Error':None, 'Index':None, 'IndexError':None, 'Flux':None,'FluxError':None, 'MaxBin':None}
		return Results

	else:

		MaxRa = RaDecPairs[MaxBin][0]
		MaxDec = RaDecPairs[MaxBin][1]

	# Define default spectral parameters
	index = 'NA'
	indexError = 'NA'
	flux = 'NA'
	fluxError = 'NA'

	if UseGtlike == True:

		# Extract the fit parameters for the bin with the maximum TS (gtlike)
		MaxBinFile = "%s/dtsmap_bin%s.log" % (LogDirectory, MaxBin)
		for line in fileinput.input([MaxBinFile]):
			if 'Index:' in line: 
				lineContents = line.split()	
				index = lineContents[1]
				indexError = lineContents[3]
			if 'Flux:' in line: 
				lineContents = line.split()	
				flux = lineContents[1]
				fluxError = lineContents[3]
				break
		fileinput.close()

	else:

		# Extract the spectral fit parameters for the bin with the maximum TS (pyLikelihood)
		MaxBinFile = "%s/dtsmap_bin%s.log" % (LogDirectory, MaxBin)
		for line in fileinput.input([MaxBinFile]):
			if 'Flux =' in line: 
				lineContents = line.split()	
				flux = float(lineContents[2])
				fluxError = float(lineContents[4])
			if 'Index =' in line: 
				lineContents = line.split()	
				index = float(lineContents[2])
				indexError = float(lineContents[4])
		fileinput.close()


	# Rotate and flip the matrix in order to it to match ra and dec ordering conventions
	TSMapRotated = numpy.rot90(TSMap)
	TSMapFlippedUp = numpy.flipud(TSMapRotated)
	TSMapFlippedLeft2Right = numpy.fliplr(TSMapFlippedUp)
	MaxRa = RaDecPairs[numpy.nanargmax(TSMap)][0]
	MaxDec = RaDecPairs[numpy.nanargmax(TSMap)][1]

	# Import the basemap module
	sys.path.append("/nfs/slac/g/ki/ki08/kocevski/LATBA/lib/python_rhel6-64/")
	from mpl_toolkits.basemap import Basemap

	# Create the figure
	fig = plot.figure()
	ax = fig.add_subplot(111)

	# Create a base map on which to plot the results
	m = Basemap(height=5.5e5,width=5.5e5, projection='laea', lon_0 = ra*-1, lat_0 = dec, resolution ='l', area_thresh=1000., celestial=True )

	# Set the plot limits (in map coordinates)
	xMin, yMin = m(ra_range[0], dec_range[0])
	xMax, yMax = m(ra_range[-1], dec_range[-1])
	m.lonmin = xMin
	m.lonmax = xMax
	m.latmin = yMin
	m.latmax = yMax

	# Plot the matrix as an image
	extent=[xMax, xMin, yMin, yMax]
	#m.imshow(TSMapFlippedLeft2Right, origin='lower', extent=extent)
	m.imshow(TSMapFlippedLeft2Right, origin='lower')

	# Setup the map grid
	m.drawmapboundary(fill_color='#ffffff')
	m.drawparallels(numpy.arange(181)-90,labels=[1,0,0,0], fmt=customDecLabel, color='grey', linewidth=0.25)
	m.drawmeridians(numpy.arange(0,360,1),labels=[0,0,0,1], fmt=customRALabel, color='grey', linewidth=0.25)
	m.suppress_ticks = False
	m.fix_aspect = False

	# Force the aspect ratio to be 1:1
	try:
		forceAspect(ax,aspect=1)
		#extent=[xMax, xMin, yMin, yMax]
		#ax.set_aspect(abs((extent[1]-extent[0])/(extent[3]-extent[2])))
	except Exception, message:
		print traceback.format_exc()