def __init__(self, dictFile, cosmicDictFile):
# define which files to process
self.dictFile = dictFile
self.param = readDict()
self.param.readFromFile(dictFile)
# define cosmic ray finding parameters
self.cosmicDictFile = cosmicDictFile
self.s = readDict()
self.s.readFromFile(cosmicDictFile)
def loadTarget(self):
self.run = self.runNames[self.runNumber]
self.target = self.targetNames[self.runNumber][self.targetNumber]
try:
self.paramName = self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.paramData = readDict()
self.paramData.read_from_file(self.paramName)
self.obsTimes = np.array(self.paramData['obsTimes'])
self.utcDates = self.paramData['utcDates']
self.sunsetDates = self.paramData['sunsetDates']
self.calTimestamps = self.paramData['calTimestamps']
self.flatCalDates = self.paramData['flatCalDates']
self.RA = self.paramData['RA']
self.Dec = self.paramData['Dec']
self.hourAngleOffset = self.paramData['HA_offset']
print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.createSunsetList()
#self.createObsList()
self.createWavelengthList()
self.createFlatList()
self.paramFileExists = True
except IOError:
print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.ui.sunsetList.clear()
self.ui.obsList.clear()
self.ui.inputList.clear()
self.ui.wavelengthList.clear()
self.ui.flatList.clear()
self.paramFileExists = False
def testReadDict(self):
"""
Test reading geminga.dict
"""
params = readDict()
params.read_from_file("geminga.dict")
self.assertTrue(params['needHotPix'])
self.assertEqual(26, len(params['obsSequence']))
def loadTargetData(self):
self.targetNames = []
# Cycle through runs and extract target name information from various dictionaries.
for iRun in range(len(self.runNames)):
self.targetData = readDict()
self.targetData.read_from_file(self.displayStackPath + self.runNames[iRun] + '/' + self.runNames[iRun] + '.dict')
self.iTargets = np.array(self.targetData['targets'])
self.targetNames.append(self.iTargets)
def loadRunData(self):
# Load run data from /Scratch/DisplayStack/runList.dict
self.runData = readDict()
self.runData.read_from_file(self.displayStackPath + '/runList.dict')
self.runNames = np.array(self.runData['runs'])
# Populate runList table with run names.
for iRun in range(len(self.runNames)):
self.ui.runList.addItem(self.runNames[iRun])
def __init__(self, parent=None):
QMainWindow.__init__(self, parent)
self.setWindowTitle('Pixel Explorer')
paramFile = sys.argv[1]
self.params = readDict()
self.params.read_from_file(paramFile)
self.createMainFrame()
self.createStatusBar()
def __init__(self,fileID='test',path='/Scratch/DisplayStack/RUN_TEMPLATE/TARGET_TEMPLATE',targetName=None,run=None,verbose=False,showPlot=False):
'''
Constructs a list of obs FileName objects from the dictionary in the path.
Inputs:
fileID - identifier in filename. eg centroid_fileID.h5 or ImageStack_fileID.h5
path - path to the display stack target info
targetName - Name of target. Assumes there's a dictionary in path called targetName.dict
If None, just searches for any xxx.dict file in the path and assigns targetName=xxx
run - name of run. eg PAL2014
if None, assumes run is the second to last directory in the path
verbose
showPlot
'''
self.path = path
self.fileID=fileID
self.targetName=targetName
self.run=run
self.verbose=verbose
self.showPlot = showPlot
if self.targetName is None or not os.path.isfile(self.path+os.sep+self.targetName+'.dict'):
#Assumes there's only one '.dict' file in the directory
for f in os.listdir(path):
if f.endswith(".dict"):
if self.verbose: print 'Loading params from ',path+os.sep+f
self.targetName = f.split('.dict')[0]
try:
self.params = readDict(self.path+os.sep+self.targetName+'.dict')
self.params.readFromFile(self.path+os.sep+self.targetName+'.dict')
except:
print "Provide a target name that leads to a dictionary in the path!"
print path
print "target: ",self.targetName
raise ValueError
if self.run is None:
#run is assumed to be the second to last directory in the path
self.run = os.path.basename(os.path.dirname(self.path))
#Get images
try:
self.loadImageStack()
except:
print "Need to make image stack for this object"
#Get Centroids
try:
self.loadAllCentroidFiles()
except:
print "Need to make centroids for this object"
def __init__(self, paramFile):
# 8 bits - channel
# 12 bits - Parabola Fit Peak Height
# 12 bits - Sampled Peak Height
# 12 bits - Low pass filter baseline
# 20 bits - Microsecond timestamp
# bitmask of 12 ones
self.pulseMask = int(self.nBitsInPulseHeight * "1", 2)
# bitmask of 20 ones
self.timestampMask = int(self.nBitsInTimestamp * "1", 2)
self.deleteBaseMask = int((8 + 12 + 12) * "1" + 12 * "0" + 20 * "1", 2)
self.params = readDict()
self.params.read_from_file(paramFile)
self.run = self.params["run"]
self.smoothRun = self.run + "s"
self.timeToAverage = self.params["timeToAverage"]
self.trimFraction = self.params["trimFraction"]
self.sunsetDate = self.params["sunsetDate"]
self.obsSequence = self.params["obsSequence"]
self.obsFNs = [FileName(run=self.run, date=self.sunsetDate, tstamp=obsTstamp) for obsTstamp in self.obsSequence]
self.smoothObsFNs = [
FileName(run=self.smoothRun, date=self.sunsetDate, tstamp=obsTstamp) for obsTstamp in self.obsSequence
]
def loadParamFile(self):
try:
self.paramData = readDict()
self.paramData.read_from_file(self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict')
self.obsTimes = np.array(self.paramData['obsTimes'])
self.utcDates = self.paramData['utcDates']
self.sunsetDates = self.paramData['sunsetDates']
self.calTimestamps = self.paramData['calTimestamps']
self.flatCalDates = self.paramData['flatCalDates']
print 'Loading parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.createSunsetList()
#self.createObsList()
self.createWavelengthList()
self.createFlatList()
self.paramFileExists = True
except IOError:
print 'No existing parameter file at ' + self.displayStackPath + self.run + '/' + self.target + '/' + self.target + '.dict'
self.ui.sunsetList.clear()
self.ui.obsList.clear()
self.ui.inputList.clear()
self.ui.wavelengthList.clear()
self.ui.flatList.clear()
self.paramFileExists = False
"""
Make all the FITS files and the plot location file.
Use this to find the information needed fo ofs2.py
"""
from util.ObsFileSeq import ObsFileSeq
from util.readDict import readDict
d = readDict()
d.read_from_file("mosaic.dict")
name = 'PSN234416a'
dt = 200
ofs = ObsFileSeq(name, d['run'], d['date'], d['timeStampList'], dt)
#ofs.plotLocations(name+".png")
wvMin = 3000
wvMax = 13000
ofs.makeAllFitsFiles(wvMin, wvMax)
import numpy as np
import matplotlib.pyplot as plt
from util import utils
from util.readDict import readDict
from scipy import pi
from mpltools import style
import figureHeader
# try new plotting style for pipeline paper
#style.use('ggplot')
####LOAD COROT18 LIGHTCURVE######
param = readDict()
param.read_from_file('corot18bparams.dict')
FileName = param['npzLoadFitFile']
FramesPerFile = param['FramesPerFile']
print FramesPerFile
TotalNumFiles = param['TotalNumFiles']
NumFrames = FramesPerFile*TotalNumFiles
IntTime = param['integrationTime']
#FoldPeriod = 0.01966127 #This is in fractions of a day
t = np.load(FileName)
params = t['params']
jd = t['jd']
#params[:,0] are the height offset of the gaussian (so the number of background pulses)
amps = params[:,1]
widths = params[:,4]
xpos = params[:,2]
ypos = params[:,3]
print 'binning phases'
del pl
phases = photons['pulsePhase']
phaseProfile,phaseBinEdges = np.histogram(phases,bins=nPhaseBins,range=(0.,1.))
profileErrors = np.sqrt(phaseProfile)
print 'done folding'
del photons,phases
return {'phaseProfile':phaseProfile,'phaseBinEdges':phaseBinEdges,'profileErrors':profileErrors,'pixelImage':pixelImageDict['image'],'pixelImageXRange':pixelImageDict['xRange'],'pixelImageYRange':pixelImageDict['yRange']}
if __name__=='__main__':
savePath = '/Scratch/dataProcessing/J0337/'
paramFile = 'j0337.dict'
params = readDict()
params.read_from_file(paramFile)
run = params['run']
sunsetDates = []
flatDates = []
obsSequences = []
parFiles = []
sunsetDates.append(params['sunsetDate0'])
flatDates.append(params['flatDate0'])
parFiles.append(params['parFile0'])
obsSequences.append(params['obsSequence0'])
sunsetDates.append(params['sunsetDate1'])
flatDates.append(params['flatDate1'])
parFiles.append(params['parFile1'])
def __init__(self,paramFile,plots=False,verbose=False):
"""
Opens flux file, prepares standard spectrum, and calculates flux factors for the file.
Method is provided in param file. If 'relative' is selected, an obs file with standard star defocused over
the entire array is expected, with accompanying sky file to do sky subtraction.
If any other method is provided, 'absolute' will be done by default, wherein a point source is assumed
to be present. The obs file is then broken into spectral frames with photometry (psf or aper) performed
on each frame to generate the ARCONS observed spectrum.
"""
self.verbose=verbose
self.plots = plots
self.params = readDict()
self.params.read_from_file(paramFile)
run = self.params['run']
sunsetDate = self.params['fluxSunsetLocalDate']
self.fluxTstamp = self.params['fluxTimestamp']
skyTstamp = self.params['skyTimestamp']
wvlSunsetDate = self.params['wvlCalSunsetLocalDate']
wvlTimestamp = self.params['wvlCalTimestamp']
flatCalFileName = self.params['flatCalFileName']
needTimeAdjust = self.params['needTimeAdjust']
self.deadtime = float(self.params['deadtime']) #from firmware pulse detection
self.timeSpacingCut = self.params['timeSpacingCut']
bLoadBeammap = self.params.get('bLoadBeammap',False)
self.method = self.params['method']
self.objectName = self.params['object']
self.r = float(self.params['energyResolution'])
self.photometry = self.params['photometry']
self.centroidRow = self.params['centroidRow']
self.centroidCol = self.params['centroidCol']
self.aperture = self.params['apertureRad']
self.annulusInner = self.params['annulusInner']
self.annulusOuter = self.params['annulusOuter']
self.collectingArea = self.params['collectingArea']
self.startTime = self.params['startTime']
self.intTime = self.params['integrationTime']
fluxFN = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp)
self.fluxFileName = fluxFN.obs()
self.fluxFile = ObsFile(self.fluxFileName)
if self.plots:
self.plotSavePath = os.environ['MKID_PROC_PATH']+os.sep+'fluxCalSolnFiles'+os.sep+run+os.sep+sunsetDate+os.sep+'plots'+os.sep
if not os.path.exists(self.plotSavePath): os.mkdir(self.plotSavePath)
if self.verbose: print "Created directory %s"%self.plotSavePath
obsFNs = [fluxFN]
self.obsList = [self.fluxFile]
if self.startTime in ['',None]: self.startTime=0
if self.intTime in ['',None]: self.intTime=-1
if self.method=="relative":
try:
print "performing Relative Flux Calibration"
skyFN = FileName(run=run,date=sunsetDate,tstamp=skyTstamp)
self.skyFileName = skyFN.obs()
self.skyFile = ObsFile(self.skyFileName)
obsFNs.append(skyFN)
self.obsList.append(self.skyFile)
except:
print "For relative flux calibration a sky file must be provided in param file"
self.__del__()
else:
self.method='absolute'
print "performing Absolute Flux Calibration"
if self.photometry not in ['aperture','PSF']: self.photometry='PSF' #default to PSF fitting if no valid photometry selected
timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
timeAdjustFileName = FileName(run=run).timeAdjustments()
#make filename for output fluxCalSoln file
self.fluxCalFileName = FileName(run=run,date=sunsetDate,tstamp=self.fluxTstamp).fluxSoln()
print "Creating flux cal: %s"%self.fluxCalFileName
if wvlSunsetDate != '':
wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
if flatCalFileName =='':
flatCalFileName=FileName(obsFile=self.fluxFile).flatSoln()
#load cal files for flux file and, if necessary, sky file
for iObs,obs in enumerate(self.obsList):
if bLoadBeammap:
print 'loading beammap',os.environ['MKID_BEAMMAP_PATH']
obs.loadBeammapFile(os.environ['MKID_BEAMMAP_PATH'])
if wvlSunsetDate != '':
obs.loadWvlCalFile(wvlCalFileName)
else:
obs.loadBestWvlCalFile()
obs.loadFlatCalFile(flatCalFileName)
obs.setWvlCutoffs(-1,-1)
if needTimeAdjust:
obs.loadTimeAdjustmentFile(timeAdjustFileName)
timeMaskFileName = timeMaskFileNames[iObs]
print timeMaskFileName
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(obsFile=obs,outputFileName=timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
print "Flux cal/sky file pixel mask saved to %s"%(timeMaskFileName)
obs.loadHotPixCalFile(timeMaskFileName)
if self.verbose: print "Loaded hot pixel file %s"%timeMaskFileName
#get flat cal binning information since flux cal will need to match it
self.wvlBinEdges = self.fluxFile.flatCalFile.root.flatcal.wavelengthBins.read()
self.nWvlBins = self.fluxFile.flatWeights.shape[2]
self.binWidths = np.empty((self.nWvlBins),dtype=float)
self.binCenters = np.empty((self.nWvlBins),dtype=float)
for i in xrange(self.nWvlBins):
self.binWidths[i] = self.wvlBinEdges[i+1]-self.wvlBinEdges[i]
self.binCenters[i] = (self.wvlBinEdges[i]+(self.binWidths[i]/2.0))
if self.method=='relative':
print "Extracting ARCONS flux and sky spectra"
self.loadRelativeSpectrum()
print "Flux Spectrum loaded"
self.loadSkySpectrum()
print "Sky Spectrum loaded"
elif self.method=='absolute':
print "Extracting ARCONS point source spectrum"
self.loadAbsoluteSpectrum()
print "Loading standard spectrum"
try:
self.loadStdSpectrum(self.objectName)
except KeyError:
print "Invalid spectrum object name"
self.__del__()
sys.exit()
print "Generating sensitivity curve"
self.calculateFactors()
print "Sensitivity Curve calculated"
print "Writing fluxCal to file %s"%self.fluxCalFileName
self.writeFactors(self.fluxCalFileName)
if self.plots: self.makePlots()
print "Done"
def __init__(self,paramFile):
"""
opens flat file,sets wavelength binnning parameters, and calculates flat factors for the file
"""
self.params = readDict()
self.params.read_from_file(paramFile)
run = self.params['run']
sunsetDate = self.params['sunsetDate']
flatTstamp = self.params['flatTstamp']
wvlSunsetDate = self.params['wvlSunsetDate']
wvlTimestamp = self.params['wvlTimestamp']
obsSequence = self.params['obsSequence']
needTimeAdjust = self.params['needTimeAdjust']
self.deadtime = self.params['deadtime'] #from firmware pulse detection
self.intTime = self.params['intTime']
self.timeSpacingCut = self.params['timeSpacingCut']
self.nSigmaClip = self.params['nSigmaClip']
self.nNearest = self.params['nNearest']
obsFNs = [FileName(run=run,date=sunsetDate,tstamp=obsTstamp) for obsTstamp in obsSequence]
self.obsFileNames = [fn.obs() for fn in obsFNs]
self.obsList = [ObsFile(obsFileName) for obsFileName in self.obsFileNames]
timeMaskFileNames = [fn.timeMask() for fn in obsFNs]
timeAdjustFileName = FileName(run=run).timeAdjustments()
print len(self.obsFileNames), 'flat files to co-add'
self.flatCalFileName = FileName(run=run,date=sunsetDate,tstamp=flatTstamp).illumSoln()
if wvlSunsetDate != '':
wvlCalFileName = FileName(run=run,date=wvlSunsetDate,tstamp=wvlTimestamp).calSoln()
for iObs,obs in enumerate(self.obsList):
if wvlSunsetDate != '':
obs.loadWvlCalFile(wvlCalFileName)
else:
obs.loadBestWvlCalFile()
if needTimeAdjust:
obs.loadTimeAdjustmentFile(timeAdjustFileName)
timeMaskFileName = timeMaskFileNames[iObs]
print timeMaskFileName
#Temporary step, remove old hotpix file
#if os.path.exists(timeMaskFileName):
# os.remove(timeMaskFileName)
if not os.path.exists(timeMaskFileName):
print 'Running hotpix for ',obs
hp.findHotPixels(self.obsFileNames[iObs],timeMaskFileName,fwhm=np.inf,useLocalStdDev=True)
print "Flux file pixel mask saved to %s"%(timeMaskFileName)
obs.loadHotPixCalFile(timeMaskFileName)
self.wvlFlags = self.obsList[0].wvlFlagTable
self.nRow = self.obsList[0].nRow
self.nCol = self.obsList[0].nCol
print 'files opened'
#self.wvlBinWidth = params['wvlBinWidth'] #angstroms
self.energyBinWidth = self.params['energyBinWidth'] #eV
self.wvlStart = self.params['wvlStart'] #angstroms
self.wvlStop = self.params['wvlStop'] #angstroms
self.wvlBinEdges = ObsFile.makeWvlBins(self.energyBinWidth,self.wvlStart,self.wvlStop)
self.intTime = self.params['intTime']
self.countRateCutoff = self.params['countRateCutoff']
self.fractionOfChunksToTrim = self.params['fractionOfChunksToTrim']
#wvlBinEdges includes both lower and upper limits, so number of bins is 1 less than number of edges
self.nWvlBins = len(self.wvlBinEdges)-1
f(*args,**kwargs)
return None
else:
proc = Process(target=f,args=args,kwargs=kwargs)
proc.start()
return proc
if __name__ == "__main__":
try:
paramFile = sys.argv[1]
except IndexError:
paramFile=os.getenv('PYTHONPATH',default=os.path.expanduser('~')+'/ARCONS-pipeline').split(':')[0]+'/params/waveCal.dict'
print "Loading parameters from: "+paramFile
try:
params = readDict(paramFile)
params.readFromFile(paramFile)
except:
params = paramFile
#waveObs = ObsFile('/Scratch/LABTEST/20140911/cal_20140911-190013.h5')
#waveObs.loadBeammapFile('/Scratch/LABTEST/20140910/beammap_SCI6_B140731-Force_20140910.h5')
#QE_obs = ObsFile('/Scratch/QEData/20140911/obs_20140911-194430.h5')
#QE_obs.loadBeammapFile('/Scratch/LABTEST/20140910/beammap_SCI6_B140731-Force_20140910.h5')
#QE_data = np.loadtxt('/Scratch/QEData/20140911/QE_20140911-194430.txt')
#QE_param = np.loadtxt('/Scratch/QEData/20140911/QE_20140911-194430.param')
waveObs = ObsFile('/Scratch/LABTEST/BOBAFETT/20140915/cal_20140915-231457.h5')
#waveObs.loadBeammapFile('/Scratch/LABTEST/20140910/beammap_SCI6_B140731-Force_20140910.h5')
def writeImageStack(images, startTimes, intTimes=None, pixIntTimes=None, path='/Scratch/DisplayStack/RUN_TEMPLATE/TARGET_TEMPLATE',outputFilename='ImageStack_0.h5'):
'''
Saves an h5 file containing the image stacks. See headers.DisplayStackHeaders for variable definitions
Inputs:
images - list of images. images[number, row, col]
startTimes - list of startTimes [Julian Date]
intTimes - (optional) list of integration times for each image [Seconds]
pixIntTimes - (optional) list of pixel integration times. [number,row,col] (same shape as images)
path - path to target's data directory
outputFilename - name of imagestack file
'''
# Create and h5 output file, create header and data groups
fullFilename = path+os.sep+'ImageStacks'+os.sep+outputFilename
if os.path.isfile(fullFilename):
warnings.warn("Overwriting image stack file: "+str(fullFilename),UserWarning)
fileh = tables.openFile(fullFilename, mode='w')
#print fullFilename
headerGroup = fileh.createGroup("/", headerGroupName, 'Header')
stackGroup = fileh.createGroup("/", imagesGroupName, 'Image Stack')
# Create row for header information
headerTable = fileh.createTable(headerGroup, headerTableName, ImageStackHeaderDescription,'Header Info')
header = headerTable.row
run=os.path.basename(os.path.dirname(path))
target=''
params=None
for f in os.listdir(path):
if f.endswith(".dict"):
target = f.split('.dict')[0]
params = readDict(path+os.sep+f)
params.readFromFile(path+os.sep+f)
# Fill in the header with possibly useful information.
header[runColName] = run
header[targetColName] = target
header[RAColName] = None if params==None else params['RA']
header[DecColName] = None if params==None else params['Dec']
header[HA_offsetColName] = None if params==None else params['HA_offset']
nRows = len(images[0])
nCols = len(images[0][0])
header[nColColName] = nCols
header[nRowColName] = nRows
#header[lstColName] = self.lst
#header[expTimeColName] = self.exptime
#header[obsFileColName] = self.obsFn
#header[integrationTimeColName] = self.integrationTime
#if self.useDeadPixelMasking:
# header[deadPixColName] = self.deadPixelFilename
#if self.useHotPixelMasking:
# header[hotPixColName] = 'Default'
#if self.useWavelengthCalibration:
# header[wvlCalFileColName] = 'Default'
# header[lowWvlColName] = self.lowerWavelengthCutoff
# header[highWvlColName] = self.upperWavelengthCutoff
#if self.useFlatCalibration:
# header[flatCalFileColName] = 'Default'
header.append()
# Create an h5 array for the starttime of each frame in the image cube.
timeTable = fileh.createCArray(stackGroup, timeTableName, tables.Float64Atom(), (1,len(startTimes)))
timeTable[:] = startTimes
# Create an h5 array for the integration time of each frame in the image cube.
if intTimes!=None and len(intTimes)==len(startTimes):
intTimeTable = fileh.createCArray(stackGroup, intTimeTableName, tables.Float64Atom(), (1,len(intTimes)))
intTimeTable[:] = intTimes
# Create an h5 table for the image cube.
#print len(images)
stackTable = fileh.createCArray(stackGroup, imagesTableName, tables.Float64Atom(), (nRows,nCols, len(np.asarray(images))))
stackTable[:] = np.rollaxis(np.asarray(images),0,3) #Paul has a weird [row,col,number] format...
# Create an h5 table for the image cube.
if pixIntTimes!=None and np.asarray(pixIntTimes).shape == np.asarray(images).shape:
intTimeTable = fileh.createCArray(stackGroup, pixIntTimeTableName, tables.Float64Atom(), (nRows,nCols, len(np.asarray(pixIntTimes))))
intTimeTable[:] = np.rollaxis(np.asarray(pixIntTimes),0,3) #Paul has a weird [row,col,number] format...
# Flush the h5 output file
fileh.flush()
fileh.close()