def test_subtractOverscan(): inFiles = readFileToArr(os.path.join(testPath, 'BIAS.list')) outArrs = subtractOverscan(inFiles, overscanSection, trimSection=None, fitsFilesOut=None, overwrite=True) print('len(inFiles) = ', len(inFiles), ', len(outFiles) = ', len(outArrs)) meanDiffs = [] for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(outArrs[iFile]) meanDiff = meanA - meanB meanDiffs.append(meanDiff) print('iFile = ', iFile, ': meanA - meanB = ', meanDiff) outFiles = [addSuffixToFileName(fileName, 'o') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=None, fitsFilesOut=outFiles, overwrite=True) for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(CCDData.read(outFiles[iFile], unit="adu")) meanDiff = meanA - meanB print('iFile = ', iFile, ': meanA - meanB = ', meanDiff, ': difference to previous calculation = ', meanDiff - meanDiffs[iFile]) outFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=outFiles, overwrite=True) for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(CCDData.read(outFiles[iFile], unit="adu")) meanDiff = meanA - meanB print('iFile = ', iFile, ': meanA - meanB = ', meanDiff, ': difference to previous calculation = ', meanDiff - meanDiffs[iFile]) try: outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=outFiles, overwrite=False) except: pass
def test_subtractBias(): inFiles = readFileToArr(os.path.join(testPath, 'BIAS.list')) otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) print('len(inFiles) = ', len(inFiles), ', len(otFiles) = ', len(outArrs)) meanDiffs = [] for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(outArrs[iFile]) meanDiff = meanA - meanB meanDiffs.append(meanDiff) print('iFile = ', iFile, ': meanA - meanB = ', meanDiff) # create master bias masterBias = os.path.join(testPath, 'combinedBias_ot.fits') combinedImage = combine(otFiles, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedImage) = ', np.mean(combinedImage)) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] outArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(otFiles[iFile], unit="adu")) meanB = np.mean(CCDData.read(otzFiles[iFile], unit="adu")) meanDiff = meanA - meanB print('iFile = ', iFile, ': meanA - meanB = ', meanDiff, ': difference to previous calculation = ', meanDiff - meanDiffs[iFile])
# removeFilesFromListWithAngleNotEqualTo(inList,inList,'15.85') # STOP separateFileList(inList, suffixes, exptypes, objects, True, fluxStandardNames=fluxStandardNames) # STOP objectFiles = os.path.join(workPath, 'SCIENCE.list') # subtract overscan and trim all images for inputList in ['ARC', 'BIAS', 'FLAT', 'SCIENCE', 'FLUXSTDS']: subtractOverscan(getListOfFiles( os.path.join(workPath, inputList + '.list')), overscanSection, trimSection=trimSection, fitsFilesOut=getListOfFiles( os.path.join(workPath, inputList + '_ot.list')), overwrite=True) # create master bias masterBias = os.path.join(workPath, 'combinedBias_ot.fits') combinedBias = combine(getListOfFiles( os.path.join(workPath, 'BIAS_ot.list')), combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median
path = '/Volumes/work/azuri/spectra/saao/saao_sep2019/20190904/' allInputFitsList = os.path.join(path,'allFits.list') suffixes = ['','ot','otz','otzf','otzfi','otzfif','otzx','otzxf','otzxfi','otzxfif'] exptypes = ['BIAS','FLAT','ARC','SCIENCE'] objects = [['*'],['*','Domeflat','Skyflat'],['*'],['*','individual']] separateFileList(allInputFitsList, suffixes, exptypes, objects, True) # subtract overscan and trim images for exptype in exptypes: runOnFilesList = os.path.join(path,exptype+'.list') outFiles = [addSuffixToFileName(fileName, 'ot') for fileName in runOnFilesList] outArrs = subtractOverscan(runOnFilesList, overscanSection, trimSection=trimSection, fitsFilesOut=outFiles, overwrite=True) # create master bias otFiles = readFileToArr(os.path.join(path,'BIAS_ot.list')) masterBias = os.path.join(path,'combinedBias_ot.fits') combinedImage = combine(otFiles, combinerMethod='median', clippingMethod='sigma', clippingParameters={'niter':0, 'low_thresh':-3., 'high_thresh':3., 'func':np.ma.median}, scaling=False, fitsOutName=masterBias)
def test_skyFlatCorrect(objectFiles=os.path.join(testPath, 'SCIENCE.list')): path = objectFiles[0:objectFiles.rfind('/') + 1] inFilesBias = readFileToArr(os.path.join(path, 'BIAS.list')) if inFilesBias[0].rfind('/') == -1: inFilesBias = [ os.path.join(path, fileName) for fileName in inFilesBias ] otFilesBias = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesBias ] otArrsBias = subtractOverscan(inFilesBias, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesBias, overwrite=True) # create master bias masterBias = os.path.join(path, 'combinedBias_ot.fits') combinedBias = combine(otFilesBias, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedBias) = ', np.mean(combinedBias)) inFilesDomeflats = readFileToArr(os.path.join(path, 'FLATDomeflat.list')) otFilesDomeflats = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesDomeflats ] otArrsDomeflats = subtractOverscan(inFilesDomeflats, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesDomeflats, overwrite=True) otzFilesDomeflats = [ addSuffixToFileName(fileName, 'otz') for fileName in inFilesDomeflats ] otzArrsDomeflats = subtractBias(otFilesDomeflats, masterBias, fitsFilesOut=otzFilesDomeflats, overwrite=True) combinedFlat = os.path.join(path, 'combinedFlat.fits') flat = combine(otzFilesDomeflats, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 2, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, minVal=0.0001, fitsOutName=combinedFlat) masterFlat = os.path.join(path, 'masterDomeFlat.fits') smoothedFlat = os.path.join(path, 'smoothedDomeFlat.fits') makeMasterFlat(combinedFlat, 9, 80., outFileNameMasterFlat=masterFlat, outFileNameMasterFlatSmoothed=smoothedFlat) inFiles = readFileToArr(objectFiles) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] otzArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) # otzxFiles = [addSuffixToFileName(fileName, 'otzx') for fileName in inFiles] # otzxArrs = cleanCosmic(otzFiles, # fitsFilesOut=otzxFiles, # overwrite=True) otzfFiles = [addSuffixToFileName(fileName, 'otzf') for fileName in inFiles] otzfArrs = flatCorrect(otzFiles, masterFlat, fitsFilesOut=otzfFiles) # otzffFiles = [addSuffixToFileName(fileName, 'otzff') for fileName in inFiles] # otzffArrs = imDivide(otzFiles, smoothedFlat, otzffFiles) test_combineSkyFlats() otzfArrs = flatCorrect( [os.path.join(testPath, 'combinedSkyFlat.fits')], masterFlat, fitsFilesOut=[os.path.join(testPath, 'combinedSkyFlat_f.fits')]) # otzffArrs = imDivide([os.path.join(testPath,'combinedSkyFlat_f.fits')], smoothedFlat, [os.path.join(testPath,'combinedSkyFlat_ff.fits')]) interpolateTraceIm( [os.path.join(testPath, 'combinedSkyFlat_f.fits') ], #FLAT_Domeflat_a1061047_otz.fits os.path.join(testPath, 'database/apvertical_trace'), os.path.join(testPath, 'database/aphorizontal_tracer90flipl')) interpolateTraceIm( otzfFiles, os.path.join(testPath, 'database/apvertical_trace'), os.path.join(testPath, 'database/aphorizontal_tracer90flipl')) makeSkyFlat(os.path.join(testPath, 'combinedSkyFlat_fi.fits'), os.path.join(testPath, 'combinedSkyFlat_fi_flattened.fits'), 7) otzfiFiles = [ addSuffixToFileName(fileName, 'otzfi') for fileName in inFiles ] otzfifFiles = [ addSuffixToFileName(fileName, 'otzfif') for fileName in inFiles ] otzfifArrs = flatCorrect(otzfiFiles, os.path.join(testPath, 'combinedSkyFlat_fi_flattened.fits'), fitsFilesOut=otzfifFiles) return otzfifFiles
def test_flatCorrect(objectFiles=os.path.join(testPath, 'SCIENCE.list')): path = objectFiles[0:objectFiles.rfind('/') + 1] inFilesBias = readFileToArr(os.path.join(path, 'BIAS.list')) if inFilesBias[0].rfind('/') == -1: inFilesBias = [ os.path.join(path, fileName) for fileName in inFilesBias ] otFilesBias = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesBias ] otArrsBias = subtractOverscan(inFilesBias, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesBias, overwrite=True) # create master bias masterBias = os.path.join(path, 'combinedBias_ot.fits') combinedBias = combine(otFilesBias, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedBias) = ', np.mean(combinedBias)) inFilesDomeflats = readFileToArr(os.path.join(path, 'FLATDomeflat.list')) otFilesDomeflats = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesDomeflats ] otArrsDomeflats = subtractOverscan(inFilesDomeflats, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesDomeflats, overwrite=True) otzFilesDomeflats = [ addSuffixToFileName(fileName, 'otz') for fileName in inFilesDomeflats ] otzArrsDomeflats = subtractBias(otFilesDomeflats, masterBias, fitsFilesOut=otzFilesDomeflats, overwrite=True) combinedFlat = os.path.join(path, 'combinedFlat.fits') print('creating combinedFlat <' + combinedFlat + '>') print('otzFilesDomeflats = ', otzFilesDomeflats) flat = combine(otzFilesDomeflats, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 2, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, minVal=0.0001, fitsOutName=combinedFlat) masterFlat = os.path.join(path, 'masterDomeFlat.fits') smoothedFlat = os.path.join(path, 'smoothedDomeFlat.fits') makeMasterFlat(combinedFlat, 9, 80., outFileNameMasterFlat=masterFlat, outFileNameMasterFlatSmoothed=smoothedFlat) flatOut = addSuffixToFileName(combinedFlat, 'flattened') flat = flatCorrect([combinedFlat], masterFlat, fitsFilesOut=[flatOut]) inFiles = readFileToArr(objectFiles) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] otzArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) # otzxFiles = [addSuffixToFileName(fileName, 'otzx') for fileName in inFiles] # otzxArrs = cleanCosmic(otzFiles, # fitsFilesOut=otzxFiles, # overwrite=True) otzfFiles = [addSuffixToFileName(fileName, 'otzf') for fileName in inFiles] otzfArrs = flatCorrect(otzFiles, masterFlat, norm_value=1., fitsFilesOut=otzfFiles) return otzfFiles
def test_cleanCosmic(): inFiles = readFileToArr(os.path.join(testPath, 'BIAS.list')) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) # create master bias masterBias = os.path.join(testPath, 'combinedBias_ot.fits') combinedImage = combine(otFiles, combinerMethod='average', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedImage) = ', np.mean(combinedImage)) inFiles = readFileToArr(os.path.join(testPath, 'SCIENCE.list')) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] outArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) otzxFiles = [ addSuffixToFileName(fileName, 'otzx_std') for fileName in inFiles ] outArrs = cleanCosmic(inFiles, fitsFilesOut=otzxFiles, overwrite=True) cosmicParameters = {'sigclip': 4., 'cleantype': 'medmask'} otzxFiles = [ addSuffixToFileName(fileName, 'otzx_lacos_pars') for fileName in inFiles ] outArrs = cleanCosmic(inFiles, cosmicParameters=cosmicParameters, fitsFilesOut=otzxFiles, overwrite=True) cosmicParameters = {'sigclip': 4., 'cleantype': 'medmask', 'testkey': 0} try: outArrs = cleanCosmic(inFiles, cosmicParameters=cosmicParameters, fitsFilesOut=otzxFiles, overwrite=True) except: pass otzxFiles = [ addSuffixToFileName(fileName, 'otzx_median') for fileName in inFiles ] outArrs = cleanCosmic(inFiles, cosmicMethod='median', fitsFilesOut=otzxFiles, overwrite=True)