def test_subtractOverscan(): inFiles = readFileToArr(os.path.join(testPath, 'BIAS.list')) outArrs = subtractOverscan(inFiles, overscanSection, trimSection=None, fitsFilesOut=None, overwrite=True) print('len(inFiles) = ', len(inFiles), ', len(outFiles) = ', len(outArrs)) meanDiffs = [] for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(outArrs[iFile]) meanDiff = meanA - meanB meanDiffs.append(meanDiff) print('iFile = ', iFile, ': meanA - meanB = ', meanDiff) outFiles = [addSuffixToFileName(fileName, 'o') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=None, fitsFilesOut=outFiles, overwrite=True) for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(CCDData.read(outFiles[iFile], unit="adu")) meanDiff = meanA - meanB print('iFile = ', iFile, ': meanA - meanB = ', meanDiff, ': difference to previous calculation = ', meanDiff - meanDiffs[iFile]) outFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=outFiles, overwrite=True) for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(CCDData.read(outFiles[iFile], unit="adu")) meanDiff = meanA - meanB print('iFile = ', iFile, ': meanA - meanB = ', meanDiff, ': difference to previous calculation = ', meanDiff - meanDiffs[iFile]) try: outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=outFiles, overwrite=False) except: pass
def test_subtractBias(): inFiles = readFileToArr(os.path.join(testPath, 'BIAS.list')) otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) print('len(inFiles) = ', len(inFiles), ', len(otFiles) = ', len(outArrs)) meanDiffs = [] for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(inFiles[iFile], unit="adu")) meanB = np.mean(outArrs[iFile]) meanDiff = meanA - meanB meanDiffs.append(meanDiff) print('iFile = ', iFile, ': meanA - meanB = ', meanDiff) # create master bias masterBias = os.path.join(testPath, 'combinedBias_ot.fits') combinedImage = combine(otFiles, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedImage) = ', np.mean(combinedImage)) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] outArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) for iFile in np.arange(0, len(inFiles), 1): meanA = np.mean(CCDData.read(otFiles[iFile], unit="adu")) meanB = np.mean(CCDData.read(otzFiles[iFile], unit="adu")) meanDiff = meanA - meanB print('iFile = ', iFile, ': meanA - meanB = ', meanDiff, ': difference to previous calculation = ', meanDiff - meanDiffs[iFile])
overscanSection = '[1983:,:]' trimSection = '[17:1982,38:97]' #testPath = '/Users/azuri/spupnik/data/20190501/' path = '/Volumes/work/azuri/spectra/saao/saao_sep2019/20190904/' allInputFitsList = os.path.join(path,'allFits.list') suffixes = ['','ot','otz','otzf','otzfi','otzfif','otzx','otzxf','otzxfi','otzxfif'] exptypes = ['BIAS','FLAT','ARC','SCIENCE'] objects = [['*'],['*','Domeflat','Skyflat'],['*'],['*','individual']] separateFileList(allInputFitsList, suffixes, exptypes, objects, True) # subtract overscan and trim images for exptype in exptypes: runOnFilesList = os.path.join(path,exptype+'.list') outFiles = [addSuffixToFileName(fileName, 'ot') for fileName in runOnFilesList] outArrs = subtractOverscan(runOnFilesList, overscanSection, trimSection=trimSection, fitsFilesOut=outFiles, overwrite=True) # create master bias otFiles = readFileToArr(os.path.join(path,'BIAS_ot.list')) masterBias = os.path.join(path,'combinedBias_ot.fits') combinedImage = combine(otFiles, combinerMethod='median', clippingMethod='sigma', clippingParameters={'niter':0, 'low_thresh':-3., 'high_thresh':3.,
def main(): if False: test_separateFileList(os.path.join(testPath, 'allFits.list')) if False: test_combine() test_subtractOverscan() test_subtractBias() if False: test_cleanCosmic() if False: test_flatCorrect(os.path.join(testPath, 'allFits.list')) test_combineSkyFlats() testInterpolateTraceIm() testMakeSkyFlat() test_skyFlatCorrect() if False: test_skyFlatCorrect(os.path.join(testPath, 'ARC.list')) inFiles = readFileToArr( os.path.join(testPath, 'SCIENCEWKK98_201_otzfif.list')) wkk = combine(inFiles, combinerMethod='average', clippingMethod='sigma', clippingParameters={ 'niter': 2, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=True, minVal=0.0001, fitsOutName=os.path.join( testPath, 'SCIENCEWKK98_201_otzxfif_combined.fits')) if False: cosmicParameters = { 'sigclip': 4., 'cleantype': 'meanmask', 'gain': 0.219, 'readnoise': 3.4, 'sepmed': False, 'fsmode': 'convolve', 'psfmodel': 'moffat', 'verbose': True, 'niter': 1, 'psfbeta': 1.5 } inFiles = readFileToArr( '/Users/azuri/daten/uni/HKU/Pa30/sparsepak/spectra/do_cosmics.list' ) outFiles = [addSuffixToFileName(fileName, 'x') for fileName in inFiles] outArrs = cleanCosmic(inFiles, cosmicParameters=cosmicParameters, fitsFilesOut=outFiles, overwrite=True) if True: arc = 'ARC_MPA_J1354-6337_a1171127_otzf.fits' arcInterp = arc[:-5] + 'i.fits' if False: lineProfiles = getLineProfiles(os.path.join(testPath, arc)) bestLineProfile = getBestLineProfile(lineProfiles) if True: oneDSpec = extractSum(os.path.join(testPath, arc), 'row') oneDSpecInterp = extractSum(os.path.join(testPath, arcInterp), 'row') plt.plot(oneDSpec, label='raw') plt.plot(oneDSpecInterp, label='interpolated') plt.legend() plt.show() xSpec = np.arange(0, oneDSpecInterp.shape[0], 1.) print('xSpec = ', xSpec) if False: # xCor([xSpec,oneDSpecInterp],bestLineProfile) findGoodLines( xSpec, oneDSpecInterp, bestLineProfile, outFileNameAllLines= '/Volumes/work/azuri/spectra/saao/saao_refSpec_lines.dat', outFileNameGoodLines= '/Volumes/work/azuri/spectra/saao/saao_refSpec_lines_good.dat') if True: coeffs, rms = calcDispersion( '/Volumes/work/azuri/spectra/saao/saao_refspec_lines_identified_good.dat', xRange=[0, xSpec[xSpec.shape[0] - 1]], degree=3) xSpecNorm = normalizeX(xSpec) wLenSpec = np.polynomial.legendre.legval(xSpecNorm, coeffs) plt.plot(wLenSpec, oneDSpecInterp) plt.show() if False: extractList = [[ 'SCIENCE_CTI173557.97-271213.963_a1061524_otzfif.fits', [20, 29], [30, 35], [16, 18] ]] extractList.append([ 'SCIENCE_Pre_22_a1061516_otzfif.fits', [25, 32], [18, 24], [33, 41] ]) twoDSpec = os.path.join(testPath, extractList[1][0]) specOut = twoDSpec[:-5] + 'Ec.dat' extract(twoDSpec, specOut, extractList[1][1], extractList[1][2], extractList[1][3], 'row')
def test_skyFlatCorrect(objectFiles=os.path.join(testPath, 'SCIENCE.list')): path = objectFiles[0:objectFiles.rfind('/') + 1] inFilesBias = readFileToArr(os.path.join(path, 'BIAS.list')) if inFilesBias[0].rfind('/') == -1: inFilesBias = [ os.path.join(path, fileName) for fileName in inFilesBias ] otFilesBias = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesBias ] otArrsBias = subtractOverscan(inFilesBias, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesBias, overwrite=True) # create master bias masterBias = os.path.join(path, 'combinedBias_ot.fits') combinedBias = combine(otFilesBias, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedBias) = ', np.mean(combinedBias)) inFilesDomeflats = readFileToArr(os.path.join(path, 'FLATDomeflat.list')) otFilesDomeflats = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesDomeflats ] otArrsDomeflats = subtractOverscan(inFilesDomeflats, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesDomeflats, overwrite=True) otzFilesDomeflats = [ addSuffixToFileName(fileName, 'otz') for fileName in inFilesDomeflats ] otzArrsDomeflats = subtractBias(otFilesDomeflats, masterBias, fitsFilesOut=otzFilesDomeflats, overwrite=True) combinedFlat = os.path.join(path, 'combinedFlat.fits') flat = combine(otzFilesDomeflats, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 2, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, minVal=0.0001, fitsOutName=combinedFlat) masterFlat = os.path.join(path, 'masterDomeFlat.fits') smoothedFlat = os.path.join(path, 'smoothedDomeFlat.fits') makeMasterFlat(combinedFlat, 9, 80., outFileNameMasterFlat=masterFlat, outFileNameMasterFlatSmoothed=smoothedFlat) inFiles = readFileToArr(objectFiles) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] otzArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) # otzxFiles = [addSuffixToFileName(fileName, 'otzx') for fileName in inFiles] # otzxArrs = cleanCosmic(otzFiles, # fitsFilesOut=otzxFiles, # overwrite=True) otzfFiles = [addSuffixToFileName(fileName, 'otzf') for fileName in inFiles] otzfArrs = flatCorrect(otzFiles, masterFlat, fitsFilesOut=otzfFiles) # otzffFiles = [addSuffixToFileName(fileName, 'otzff') for fileName in inFiles] # otzffArrs = imDivide(otzFiles, smoothedFlat, otzffFiles) test_combineSkyFlats() otzfArrs = flatCorrect( [os.path.join(testPath, 'combinedSkyFlat.fits')], masterFlat, fitsFilesOut=[os.path.join(testPath, 'combinedSkyFlat_f.fits')]) # otzffArrs = imDivide([os.path.join(testPath,'combinedSkyFlat_f.fits')], smoothedFlat, [os.path.join(testPath,'combinedSkyFlat_ff.fits')]) interpolateTraceIm( [os.path.join(testPath, 'combinedSkyFlat_f.fits') ], #FLAT_Domeflat_a1061047_otz.fits os.path.join(testPath, 'database/apvertical_trace'), os.path.join(testPath, 'database/aphorizontal_tracer90flipl')) interpolateTraceIm( otzfFiles, os.path.join(testPath, 'database/apvertical_trace'), os.path.join(testPath, 'database/aphorizontal_tracer90flipl')) makeSkyFlat(os.path.join(testPath, 'combinedSkyFlat_fi.fits'), os.path.join(testPath, 'combinedSkyFlat_fi_flattened.fits'), 7) otzfiFiles = [ addSuffixToFileName(fileName, 'otzfi') for fileName in inFiles ] otzfifFiles = [ addSuffixToFileName(fileName, 'otzfif') for fileName in inFiles ] otzfifArrs = flatCorrect(otzfiFiles, os.path.join(testPath, 'combinedSkyFlat_fi_flattened.fits'), fitsFilesOut=otzfifFiles) return otzfifFiles
def test_flatCorrect(objectFiles=os.path.join(testPath, 'SCIENCE.list')): path = objectFiles[0:objectFiles.rfind('/') + 1] inFilesBias = readFileToArr(os.path.join(path, 'BIAS.list')) if inFilesBias[0].rfind('/') == -1: inFilesBias = [ os.path.join(path, fileName) for fileName in inFilesBias ] otFilesBias = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesBias ] otArrsBias = subtractOverscan(inFilesBias, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesBias, overwrite=True) # create master bias masterBias = os.path.join(path, 'combinedBias_ot.fits') combinedBias = combine(otFilesBias, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedBias) = ', np.mean(combinedBias)) inFilesDomeflats = readFileToArr(os.path.join(path, 'FLATDomeflat.list')) otFilesDomeflats = [ addSuffixToFileName(fileName, 'ot') for fileName in inFilesDomeflats ] otArrsDomeflats = subtractOverscan(inFilesDomeflats, overscanSection, trimSection=trimSection, fitsFilesOut=otFilesDomeflats, overwrite=True) otzFilesDomeflats = [ addSuffixToFileName(fileName, 'otz') for fileName in inFilesDomeflats ] otzArrsDomeflats = subtractBias(otFilesDomeflats, masterBias, fitsFilesOut=otzFilesDomeflats, overwrite=True) combinedFlat = os.path.join(path, 'combinedFlat.fits') print('creating combinedFlat <' + combinedFlat + '>') print('otzFilesDomeflats = ', otzFilesDomeflats) flat = combine(otzFilesDomeflats, combinerMethod='median', clippingMethod='sigma', clippingParameters={ 'niter': 2, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, minVal=0.0001, fitsOutName=combinedFlat) masterFlat = os.path.join(path, 'masterDomeFlat.fits') smoothedFlat = os.path.join(path, 'smoothedDomeFlat.fits') makeMasterFlat(combinedFlat, 9, 80., outFileNameMasterFlat=masterFlat, outFileNameMasterFlatSmoothed=smoothedFlat) flatOut = addSuffixToFileName(combinedFlat, 'flattened') flat = flatCorrect([combinedFlat], masterFlat, fitsFilesOut=[flatOut]) inFiles = readFileToArr(objectFiles) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] otzArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) # otzxFiles = [addSuffixToFileName(fileName, 'otzx') for fileName in inFiles] # otzxArrs = cleanCosmic(otzFiles, # fitsFilesOut=otzxFiles, # overwrite=True) otzfFiles = [addSuffixToFileName(fileName, 'otzf') for fileName in inFiles] otzfArrs = flatCorrect(otzFiles, masterFlat, norm_value=1., fitsFilesOut=otzfFiles) return otzfFiles
def test_cleanCosmic(): inFiles = readFileToArr(os.path.join(testPath, 'BIAS.list')) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) # create master bias masterBias = os.path.join(testPath, 'combinedBias_ot.fits') combinedImage = combine(otFiles, combinerMethod='average', clippingMethod='sigma', clippingParameters={ 'niter': 0, 'low_thresh': -3., 'high_thresh': 3., 'func': np.ma.median }, scaling=False, fitsOutName=masterBias) print('average sigma 0: mean(combinedImage) = ', np.mean(combinedImage)) inFiles = readFileToArr(os.path.join(testPath, 'SCIENCE.list')) if inFiles[0].rfind('/') == -1: inFiles = [os.path.join(path, fileName) for fileName in inFiles] otFiles = [addSuffixToFileName(fileName, 'ot') for fileName in inFiles] outArrs = subtractOverscan(inFiles, overscanSection, trimSection=trimSection, fitsFilesOut=otFiles, overwrite=True) otzFiles = [addSuffixToFileName(fileName, 'otz') for fileName in inFiles] outArrs = subtractBias(otFiles, masterBias, fitsFilesOut=otzFiles, overwrite=True) otzxFiles = [ addSuffixToFileName(fileName, 'otzx_std') for fileName in inFiles ] outArrs = cleanCosmic(inFiles, fitsFilesOut=otzxFiles, overwrite=True) cosmicParameters = {'sigclip': 4., 'cleantype': 'medmask'} otzxFiles = [ addSuffixToFileName(fileName, 'otzx_lacos_pars') for fileName in inFiles ] outArrs = cleanCosmic(inFiles, cosmicParameters=cosmicParameters, fitsFilesOut=otzxFiles, overwrite=True) cosmicParameters = {'sigclip': 4., 'cleantype': 'medmask', 'testkey': 0} try: outArrs = cleanCosmic(inFiles, cosmicParameters=cosmicParameters, fitsFilesOut=otzxFiles, overwrite=True) except: pass otzxFiles = [ addSuffixToFileName(fileName, 'otzx_median') for fileName in inFiles ] outArrs = cleanCosmic(inFiles, cosmicMethod='median', fitsFilesOut=otzxFiles, overwrite=True)