def GetOffset(filename):
telescope, obsparam = CheckInstrument([filename[0]])
for idx, elem in enumerate(filename):
hdulist = fits.open(elem)
Offset = hdulist[0].header['YOFFSET']
print(Offset)
def Log(filename, Outfile, verbode, Diagnostic):
telescope, obsparam = CheckInstrument([filename[0]])
print(telescope)
if Diagnostic:
diag.create_website('Log.html')
html = '<H2>Log</H2>\n'
html += "<P><TABLE BORDER=\"1\">\n<TR>\n"
if telescope == 'GMOSS' or telescope == 'GMOSN':
html += "<TH>Idx</TH><TH>Filename</TH> <TH>Images</TH></TR>\n"
else:
html += "<TH>Instrument</TH><TH>Index</TH> <TH>Obs type</TH> <TH>Object</TH> <TH>Exp time [sec]</TH> <TH>RA</TH> <TH>DEC</TH> <TH>Airmass</TH> <TH>Time</TH> <TH>Images</TH></TR>\n"
for idx, elem in enumerate(filename):
Splitted = elem.split('_')
hdulist = fits.open(elem)
if Diagnostic:
diag.Create_Image(elem)
framefilename = 'diagnostics/' + elem + '.png'
INST = obsparam['telescope_instrument']
OBSTYPE = hdulist[0].header[obsparam['obstype']]
if telescope == 'SOAR':
if elem.split('_')[3] == 'ACQ':
OBSTYPE = 'ACQ'
OBJECT = hdulist[0].header[obsparam['object']]
if telescope == 'SOAR':
if OBSTYPE == 'COMP':
OBJECT = elem.split('_')[4]
EXPTIME = hdulist[0].header[obsparam['exptime']]
ALPHA = hdulist[0].header[obsparam['ra']]
DEC = hdulist[0].header[obsparam['dec']]
AM = hdulist[0].header[obsparam['airmass']]
TIME = hdulist[0].header[obsparam['date_keyword']]
IDX = Splitted[1]
GRAT = hdulist[0].header[obsparam['grating']]
if telescope == 'GMOSS' or telescope == 'GMOSN':
WAV = hdulist[0].header[obsparam['tilt']]
print('%13s %6s %10s %6s %15s %7s %3.3f %3.3f \t %5s %25s %15s' %
(INST, IDX, OBSTYPE, str(int(WAV)), OBJECT, str(
int(EXPTIME)), ALPHA, DEC, str(AM), TIME, GRAT))
if Diagnostic:
html += "<TH>Idx</TH><TH>Filename</TH> <TH>Images</TH> <TH>Images</TH></TR>\n"
else:
print('%13s %6s %10s %15s %7s %12s %12s %5s %25s' %
(INST, IDX, OBSTYPE, OBJECT, str(
int(EXPTIME)), str(ALPHA), str(DEC), str(AM), TIME))
if Diagnostic:
html += ("<TR><TD>%s</TD><TD>%s</TD> <TD>%s</TD> <TD>%s</TD> <TD>%s</TD> <TD>%s</TD> <TD>%s</TD> <TD>%s</TD> <TD>%s</TD> <TH> <IMG SRC=\"%s\"> </TH></TR>\n") % \
(INST, IDX, OBSTYPE,OBJECT,str(int(EXPTIME)),str(ALPHA),str(DEC), str(AM), TIME,framefilename)
# print(INST + '\t' + OBSTYPE + '\t' + OBJECT + '\t'+ str(EXPTIME) + '\t' + str(ALPHA) + '\t' + str(DEC) + '\t' + str(AM) + '\t' + TIME )
html += '</TABLE>\n'
diag.append_website('Log.html', html, replace_below="<H2>Log</H2>\n")
def Prepare(filenames, Verbose=True):
logging.info('****************************************')
logging.info('****** Start of SP_Prepare script ******')
logging.info('****************************************')
# Get current directory
Directory = os.getcwd()
# Removes the './' if present in filenames
for idx, filename in enumerate(filenames):
filenames[idx] = filename.replace('./', '')
# start logging
# logging.info('Preparing data with parameters: %s',filenames)
# Create the procc folder
# now = datetime.datetime.now()
# Dir_To_Create = 'Procc_' + str(now.year) + '_' + str(now.month) + '_' + str(now.day) + '_' + str(now.hour) + '_' + str(now.minute) + '_' + str(now.second)
# os.mkdir(Dir_To_Create)
# Proc_Dir = Dir_To_Create
# Copy raw data to a /raw directory for safety
if not os.path.exists('raw'):
os.makedirs('raw')
for idx, filename in enumerate(filenames):
shutil.copy(filename, './' + 'raw/' + filename)
logging.info('All data have been backup to the ./raw directory')
telescope, obsparam = CheckInstrument(filenames)
# change FITS file extensions to .fits
for idx, filename in enumerate(filenames):
if filename.split('.')[-1] in ['fts', 'FTS', 'FITS', 'fit', 'FIT']:
os.rename(filename, '.'.join(filename.split('.')[:-1]) + '.fits')
filenames[idx] = '.'.join(filename.split('.')[:-1]) + '.fits'
logging.info('change filename from "%s" to "%s"' %
(filename, filenames[idx]))
# if telescope == 'SOAR':
# hdulist = fits.open(filename)
# hdulist[0].data = hdulist[0].data[200:-200,:]
# hdulist.writeto(filename,overwrite = True,checksum=False, output_verify='ignore')
CheckObsType(filenames, telescope, obsparam, Method='Normal')
logging.info('**************************************')
logging.info('****** end of SP_Prepare script ******')
logging.info('**************************************')
return None
def CheckGrating(filenames):
telescope, obsparam = CheckInstrument(filenames)
### read grating information from fits headers
# check that they are the same for all images
logging.info(('check for same grating for %d ' + \
'frames') % len(filenames))
grating = []
for idx, filename in enumerate(filenames):
try:
hdulist = fits.open(filename, ignore_missing_end=True)
except IOError:
logging.error('cannot open file %s' % filename)
print('ERROR: cannot open file %s' % filename)
filenames.pop(idx)
continue
header = hdulist[0].header
grating.append(header[obsparam['grating']])
print('%s \t %s' % (filename,header[obsparam['grating']]))
if len(grating) == 0:
raise KeyError('cannot identify filter; please update' + \
'setup/telescopes.py accordingly')
if len(set(grating)) == 1:
print('All the files possess the same grating')
else:
out = 0
while out == 0:
print('ERROR: multiple gratings used in dataset: %s' % str(set(grating)))
text = raw_input("Do you want to modify the gratings of all the files ? : (['y'],'n']) ")
if text == '' or text == 'y':
grating = raw_input("What grating do you want to put in the header ? : ")
out = 1
else:
if text == 'n':
print('The headers has not been changed.')
print('Exiting...')
sys.exit()
else:
print("SP did not recognized you answer, please write 'y' or 'n' ")
text = raw_input("Do you want to modify the gratings of all the files ? : (['y'],'n']) ")
if text == 'y' or '':
for idx, filename in enumerate(filenames):
try:
hdulist = fits.open(filename, mode='update')
except IOError:
logging.error('cannot open file %s' % filename)
print('ERROR: cannot open file %s' % filename)
filenames.pop(idx)
continue
hdulist[0].header[obsparam['grating']] = grating
hdulist.close()
def Detect_Spectra(filename, OutName, DetecLim):
Offset = []
telescope, obsparam = CheckInstrument([filename[0]])
for idx, elem in enumerate(filename):
hdulist = fits.open(elem)
Bin = int(hdulist[1].header['CCDSUM'][0])
data = hdulist[0].data
hdulist = fits.open(elem)
Offset.append(hdulist[0].header['YOFFSET'])
if idx == 0:
print(idx)
dataTot = data
else:
print(idx)
dataTot += data
hdulist[0].data = dataTot
FitsName = OutName + '.fits'
hdulist.writeto(FitsName)
Sig = float(DetecLim)
Spec = SP.Auto_Detect_Spectra(OutName + '.fits', Sig)
print(Spec)
PltScale = obsparam['pixscale']
Offset = np.array(Offset)
if telescope == 'GMOSS':
Off = -Offset / (PltScale * Bin)
if telescope == 'GMOSN':
Off = Offset / (PltScale * Bin)
print(Spec)
Spec_Loc = SP.Get_Spectra(Spec, Off)
if Bin == 4:
Spec_Loc = Spec_Loc + 400
if Bin == 2:
Spec_Loc = Spec_Loc + 800
SpecLocName = OutName + '_Spec_loc.txt'
with open(SpecLocName, 'w') as f:
simplejson.dump(list(Spec_Loc), f)
OffsetName = OutName + '_Offset.txt'
with open(OffsetName, 'w') as f:
simplejson.dump(list(Offset), f)
print(Spec_Loc)
print(Off)
def Coadd(filename, OutName):
telescope, obsparam = CheckInstrument([filename[0]])
for idx, elem in enumerate(filename):
hdulist = fits.open(elem)
data = hdulist[0].data
if idx == 0:
print(idx)
dataTot = data
else:
print(idx)
dataTot += data
hdulist[0].data = dataTot
hdulist.writeto(OutName)
def Subtract(filename, OutName, outFile):
telescope, obsparam = CheckInstrument([filename[0]])
Image1 = filename[0]
Image2 = SecondFile
hdulist1 = fits.open(Image1)
data1 = hdulist1[0].data
hdulist2 = fits.open(Image2)
data2 = hdulist2[0].data
Result = data1 - data2
hdulist1[0].data = Result
hdulist1.writeto(outFile)
def BckgSub(FileName,
Verbose,
Method,
Suffix,
Spec_loc,
Diagnostic,
Area=[250, 350],
test=0,
Live=False):
telescope, obsparam = CheckInstrument([FileName[0]])
Out_File = []
print('telescope')
for elem in FileName:
hdulist = fits.open(elem)
image = hdulist[0].data
Mask = np.zeros_like(image)
Mask[:, :] = 0
if telescope == 'Deveny' or telescope == 'DEVENY' or telescope == 'SOAR' or telescope == 'NOT':
if 'Deveny' or telescope == 'DEVENY':
DetecFlags = {'Instrument': 'Deveny'}
if telescope == 'NOT':
DetecFlags = {'Instrument': 'ALFOSC_FASU'}
if Method == 'auto':
Center = SP.Detect_Spectra(image, Bin=2, **DetecFlags)
(Center)
Area[0] = int(Center - 70)
Area[1] = int(Center + 70)
if Method == 'range':
Center = (Area[1] - Area[0]) / 2
print(Area)
Mask[0:Area[0], :] = 1
Mask[Area[1]:, :] = 1
Mask = Mask.astype(bool)
image2 = np.ma.masked_array(image, mask=Mask)
X = range(image.shape[0])
X = np.array(X)
Fdc = []
if Live:
imgdat = image
Intervals = ZScaleInterval()
Limits = Intervals.get_limits(imgdat)
imgdat = (numpy.clip(imgdat, Limits[0], Limits[1]) -
Limits[0]) / (old_div(Limits[1] - Limits[0], 256))
imgdat = interp.zoom(imgdat, test.zoom)
for i in range(image.shape[1]):
index = np.argwhere(np.isnan(image2[:, i]))
image2[index, i] = 0
image3 = SP.Sigma_Clip(image2[:, i])
XX = np.ma.masked_array(X, image3.mask)
z = np.ma.polyfit(XX, image3, 1)
p = np.poly1d(z)
Fdc.append(p(Center))
image[:, i] = image[:, i] - p(X)
if Live:
imgdat[:, int(i / 2)] = old_div(
np.clip(interp.zoom(image[:, i], test.zoom), 0, 100),
(old_div(100, 256)))
test.images[test.index] = Image.fromarray(imgdat)
test.canvas.update()
test.canvas.delete("all")
test.canvas.forget()
test.tkimage = ImageTk.PhotoImage(test.images[0],
palette=256)
test.canvas = Canvas(test.frame_fits,
height=test.tkimage.height(),
width=test.tkimage.width())
test.canvas.pack()
test.image = test.canvas.create_image(0,
0,
anchor='nw',
image=test.tkimage)
# select first image
test.index = 0
im = test.images[test.index]
test.tkimage.paste(im)
test.canvas.update()
Out_File.append(
elem.replace('.fits', '').replace('_Procc', '') + '_' +
Suffix + '.fits')
hdulist.writeto(elem.replace('.fits', '').replace('_Procc', '') +
'_' + Suffix + '.fits',
overwrite=True)
np.savetxt(
elem.replace('.fits', '').replace('_Procc', '') + '_' +
Suffix + '.txt', Fdc)
if telescope == 'GMOSS' or telescope == 'GMOSN':
DetecFlags = {'Instrument': 'GMOS'}
print(Spec_loc)
OffFile = Spec_loc + '_Offset.txt'
with open(OffFile, 'r') as f:
Offset = simplejson.load(f)
SpecFile = Spec_loc + '_Spec_loc.txt'
with open(SpecFile, 'r') as f:
Spec_Loc = simplejson.load(f)
Offset = np.sort(np.array(Offset).astype(int))
Spec_Loc = np.sort(Spec_Loc)
Off = int(hdulist[0].header['YOFFSET'])
print(Off)
idx = np.where(Offset == Off)[0][0]
if telescope == 'GMOSS':
Center = Spec_Loc[2 - idx]
print(Center)
if telescope == 'GMOSN':
Center = Spec_Loc[idx]
print(Center)
Area[0] = int(Center - 50)
Area[1] = int(Center + 50)
print(Area)
Mask[0:Area[0], :] = 1
Mask[Area[1]:, :] = 1
Mask = Mask.astype(bool)
image2 = np.ma.masked_array(image, mask=Mask)
X = range(image.shape[0])
X = np.array(X)
for i in range(image.shape[1]):
index = np.argwhere(np.isnan(image2[:, i]))
image2[index, i] = 0
image3 = SP.Sigma_Clip(image2[:, i], sig=3)
XX = np.ma.masked_array(X, image3.mask)
z = np.ma.polyfit(XX, image3, 1)
p = np.poly1d(z)
image[:, i] = image[:, i] - p(X)
hdulist.writeto(elem.replace('.fits', '').replace(
'_Procc', '').replace('Sub', '') + '_' + Suffix + '.fits',
overwrite=True)
Out_File.append(
elem.replace('.fits', '').replace('_Procc', '') + '_' +
Suffix + '.fits')
if Diagnostic:
diag.create_website('Background-Sub_Log.html')
diag.add_CosmCorr(Out_File, 'Background-Sub_Log.html')
def run_the_pipel(filenames, Verbose):
"""
wrapper to run the photometry pipeline
"""
# reset diagnostics for this data set
_SP_conf.dataroot, _SP_conf.diagroot, \
_SP_conf.index_filename, _SP_conf.reg_filename, _SP_conf.cal_filename, \
_SP_conf.res_filename = _SP_conf.setup_diagnostics()
## diag.create_index(filenames,_SP_conf.dataroot) # commented for development uncomment for dealing with new unprepared data
### read telescope information from fits headers
# check that they are the same for all images
logging.info('##### new spectroscopic process in %s #####' %
_SP_conf.dataroot)
logging.info(('check for same telescope/instrument for %d ' + \
'frames') % len(filenames))
telescope, obsparam = CheckInstrument(filenames)
### read grating information from fits headers
# check that they are the same for all images
logging.info(('check for same grating for %d ' + \
'frames') % len(filenames))
grating = []
for idx, filename in enumerate(filenames):
try:
hdulist = fits.open(filename, ignore_missing_end=True)
except IOError:
logging.error('cannot open file %s' % filename)
print('ERROR: cannot open file %s' % filename)
filenames.pop(idx)
continue
header = hdulist[0].header
grating.append(header[obsparam['grating']])
if len(grating) == 0:
raise KeyError('cannot identify filter; please update' + \
'setup/telescopes.py accordingly')
if len(set(grating)) > 1:
print('ERROR: multiple gratings used in dataset: %s' %
str(set(grating)))
print('Check the grating of each files by running SP_CheckGrating')
# logging.error('multiple grating used in dataset: %s' %
# str(set(grating)))
# for i in range(len(filenames)):
# logging.error('%s %s' % (filenames[i], grating[i]))
sys.exit()
""" run SP_prepare """
### Prepare(filenames) # commented for development uncomment for dealing with new unprepared data
_SP_conf.filenames = filenames # for development purpose, delete if Prepare(filenames) uncommented
### Toolbox.Get_BiasList()
""" Add the bias file list to the diagnostic.html file """
### diag.add_BiasList()
""" Creation of the Master Bias, SP_run assumed that there is only one Master Bias to be created. One can use SP_Bias manually otherwise """
### MasterBiasName = 'MasterBias.fits'
### Create_Bias(_SP_conf.Bias_filenames,MasterBiasName,Verbose)
### diag.add_BiasSummary(MasterBiasName)
### Toolbox.Get_FlatList()
""" Creation of the Master Flat """
### diag.add_FlatList()
### MasterFlatName = 'MasterFlat.fits'
### Create_Flat(_SP_conf.Flat_filenames,MasterFlatName,Verbose,MasterBiasName,'index',True)
""" processing of the target spectra """
Toolbox.Get_AsteroidList()
print(_SP_conf.Asteroid_filenames)
def CheckObsType(filenames):
telescope, obsparam = CheckInstrument(filenames)
CollFoc = []
for idx, filename in enumerate(filenames):
try:
hdulist = fits.open(filename,
mode='update',
ignore_missing_end=True)
except IOError:
logging.error('cannot open file %s' % filename)
print('ERROR: cannot open file %s' % filename)
filenames.pop(idx)
continue
data = hdulist[0].data
Med = np.median(data[10:-10, 10:-10])
std0 = np.std(np.median(data[10:-10, 10:-10], axis=0))
std1 = np.std(np.median(data[10:-10, 10:-10], axis=1))
if Med > 10000:
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']], 'FLAT'))
hdulist[0].header[obsparam['obstype']] = 'FLAT'
else:
if std0 + std1 < 10:
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']], 'BIAS'))
hdulist[0].header[obsparam['obstype']] = 'BIAS'
else:
if std0 / std1 > 100:
if telescope == 'DEVENY':
CollFoc.append((int(filename.split('.')[1]),
hdulist[0].header['COLLFOC'], idx))
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']],
'ARCS/FOCUS'))
hdulist[0].header[obsparam['obstype']] = 'ARCS/FOCUS'
else:
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']], 'ARCS'))
hdulist[0].header[obsparam['obstype']] = 'ARCS'
else:
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']], 'OBJECT'))
hdulist[0].header[obsparam['obstype']] = 'OBJECT'
hdulist.close()
Cons = toolbox.Get_Consecutive(np.array(CollFoc)[:, 0].astype(int))
Focus = []
Series = []
for idx, elem in enumerate(Cons):
for idx2, elem2 in enumerate(elem):
Focus.append(CollFoc[Cons[idx][idx2][0]][1])
if len(set(Focus)) == 1:
Series.append('ARCS')
else:
Series.append('FOCUS')
print(Focus)
Focus = []
for idx, elem in enumerate(Cons):
for idx2, elem2 in enumerate(elem):
try:
hdulist = fits.open(filenames[CollFoc[Cons[idx][idx2][0]][2]],
mode='update',
ignore_missing_end=True)
except IOError:
logging.error('cannot open file %s' % filename)
print('ERROR: cannot open file %s' % filename)
filenames.pop(idx)
continue
if Series[idx] == 'ARCS':
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']], 'ARCS'))
hdulist[0].header[obsparam['obstype']] = 'ARCS'
else:
print('%s changed to %s' %
(hdulist[0].header[obsparam['obstype']], 'FOCUS'))
hdulist[0].header[obsparam['obstype']] = 'FOCUS'
hdulist.close()
def Prepare(filenames,Verbose):
# Get current directory
Directory = os.getcwd()
# Removes the './' if present in filenames
for idx, filename in enumerate(filenames):
filenames[idx] = filename.replace('./','')
# start logging
logging.info('Preparing data with parameters: %s',filenames)
print('Preparing data with parameters: %s',filenames)
# Create the procc folder
# now = datetime.datetime.now()
# Dir_To_Create = 'Procc_' + str(now.year) + '_' + str(now.month) + '_' + str(now.day) + '_' + str(now.hour) + '_' + str(now.minute) + '_' + str(now.second)
# os.mkdir(Dir_To_Create)
# Proc_Dir = Dir_To_Create
# Copy raw data to a /raw directory for safety
if not os.path.exists('raw'):
os.makedirs('raw')
for idx, filename in enumerate(filenames):
shutil.copy(filename, './' + 'raw/' + filename)
telescope, obsparam = CheckInstrument(filenames)
# change FITS file extensions to .fits
for idx, filename in enumerate(filenames):
if filename.split('.')[-1] in ['fts', 'FTS', 'FITS', 'fit', 'FIT']:
os.rename(filename, '.'.join(filename.split('.')[:-1])+'.fits')
filenames[idx] = '.'.join(filename.split('.')[:-1])+'.fits'
logging.info('change filename from "%s" to "%s"' %
(filename, filenames[idx]))
if telescope == 'SOAR':
hdulist = fits.open(filename)
hdulist[0].data = hdulist[0].data[200:-200,:]
hdulist.writeto(filename,overwrite = True,checksum=False, output_verify='ignore')
# identify keywords for GENERIC telescopes
# open one sample image file
hdulist = fits.open(filenames[0], verify='ignore',
ignore_missing_end='True')
header = hdulist[0].header
# keywords that have to be implanted into each image
implants = {}
# prepare image headers for spectroscopic pipeline
CheckObsType(filenames)
return None
def WavCal(filenames, ArcsFile, OutFile, Verbose, Method):
Pipe_Path = _SP_conf.rootpath
Arcs = []
for elem in ArcsFile:
hdulist = fits.open(elem)
Arcs.append(hdulist[0].data)
Arcs = np.median(Arcs, axis=0)
telescope, obsparam = CheckInstrument([ArcsFile[0]])
print(telescope)
SpecA = np.loadtxt(filenames[0]).transpose()
SpecA = np.array(SpecA)
if Method == 'template':
if telescope == 'GMOSS' or telescope == 'GMOSN':
Detector = hdulist[0].header['DETECTOR']
print(Detector)
Gratting = hdulist[0].header[obsparam['grating']]
Binning = hdulist[1].header['CCDSUM'][0]
DetecFlags = {
'Instrument': telescope,
'Binning': Binning,
'Gratting': Gratting,
'Detector': Detector
}
print(DetecFlags)
if telescope == 'DEVENY':
DetecFlags = {'Instrument': 'Deveny'}
if telescope == 'SOAR':
DetecFlags = {'Instrument': 'Soar'}
print(DetecFlags)
Wav = SP.Wav_Cal2(Arcs, **DetecFlags)
Wav = np.array(Wav)
Wav = Wav / 10000
# Wav = np.flip(Wav,axis=0)
print(Wav)
# with open(filenames[0],'r') as f:
# SpecA = f.read().splitlines()
# SpecA = np.loadtxt(filenames[0]).transpose()
# SpecA = np.array(SpecA)
# SpecA = np.flip(SpecA,axis=0)
# f = open(OutFile,'w')
# for Wave,Spec in zip(Wav,SpecA):
# f.write("{} \t {} \n".format(Wave,Spec))
np.savetxt(OutFile,
np.array([Wav, SpecA[0, :], SpecA[1, :]]).transpose())
if Method == 'auto':
Dim = []
Dim.append(np.size(Arcs, 0))
Dim.append(np.size(Arcs, 1))
Arcs_L = Arcs[int(Dim[0] / 2), :]
Arcs_Loc = SP.Auto_Detect_Lines(Arcs_L,
Tresh_Det=1.5,
Tresh_Arcs=[4, 10])
f = open(Pipe_Path + '/Wav_Precomp', 'rb')
Pre = pickle.load(f)
f.close()
One = Arcs_Loc[:9]
One = np.array(One).astype(float)
O = One - np.max(One)
O = O / np.min(O)
D1_one = np.sort(O)[1]
D2_one = np.sort(O)[2]
D3_one = np.sort(O)[3]
D4_one = np.sort(O)[4]
D5_one = np.sort(O)[5]
D6_one = np.sort(O)[6]
D7_one = np.sort(O)[7]
dist = []
for elem1, elem2, elem3, elem4, elem5, elem6, elem7 in zip(
Pre[0], Pre[1], Pre[2], Pre[3], Pre[4], Pre[5], Pre[6]):
dist.append((elem1 - D1_one)**2 + (elem2 - D2_one)**2 +
(elem3 - D3_one)**2 + (elem4 - D4_one)**2 +
(elem5 - D5_one)**2 + (elem6 - D6_one)**2 +
(elem7 - D7_one)**2)
AS = np.argsort(dist)
max_index, max_value = min(enumerate(dist), key=operator.itemgetter(1))
WV = [
6965.4, 7067.2, 7147.0, 7272.9, 7384.0, 7503.9, 7635.1, 7723.8,
7948.2, 8014.8, 8115.3, 8264.5, 8408.2, 8521.4, 8667.9, 9122.9,
9224.5, 9354.2, 9657.8, 9784.5
]
comb = combinations(WV, 9)
WV_Comb = []
for elem in comb:
WV_Comb.append(elem)
a, b = np.polyfit(WV_Comb[AS[0]], np.sort(One)[::-1], 1)
Red = a * np.array(WV) + b
WV_Sol = []
for elem in Arcs_Loc:
Dist = (Red - elem)**2
SD = np.argsort(Dist)
WV_Sol.append(WV[SD[0]])
aa = np.polyfit(WV_Sol, Arcs_Loc, 1, full=True)
print(aa[1])
with open(filenames[0], 'r') as f:
SpecA = f.read().splitlines()
SpecA = np.flip(SpecA, axis=0)
Wav = (np.array(range(0, len(SpecA))) - aa[0][1]) / aa[0][0]
Wav = Wav / 10000
Wavel = Wav[::-1]
f = open(OutFile, 'w')
for Wave, Spec in zip(Wavel, SpecA):
f.write("{} \t {} \n".format(Wave, Spec))
return None
def create_index(filenames, directory, display=False, imagestretch='linear'):
"""
create index.html
diagnostic root website for one pipeline process
"""
if display:
print('create frame index table and frame images')
logging.info('create frame index table and frame images')
# obtain grating from first image file
telescope, obsparam = CheckInstrument([filenames[0]])
refheader = fits.open(filenames[0], ignore_missing_end=True)[0].header
grating = [refheader[obsparam['grating']]]
del (refheader)
html = "<H2>data directory: %s</H2>\n" % directory
html += ("<H1>%s/%s-band - Diagnostic Output</H1>\n" + \
"%d frames total, see full pipeline " + \
"<A HREF=\"%s\">log</A> for more information\n") % \
(obsparam['telescope_instrument'], grating,
len(filenames),
'.diagnostics/' +
_SP_conf.log_filename.split('.diagnostics/')[1])
### create frame information table
html += "<P><TABLE BORDER=\"1\">\n<TR>\n"
html += "<TH>Idx</TH><TH>Filename</TH><TH>Date</TH>" + \
"<TH>Objectname</TH><TH>Grating</TH>" + \
"<TH>Airmass</TH><TH>Exptime (s)</TH>" + \
"<TH>Type</TH>\n</TR>\n"
# fill table and create frames
filename = filenames
for idx, filename in enumerate(filenames):
### fill table
hdulist = fits.open(filename, ignore_missing_end=True)
header = hdulist[0].header
# read out image binning mode
binning = tb.get_binning(header, obsparam)
#framefilename = _pp_conf.diagroot + '/' + filename + '.png'
framefilename = '.diagnostics/' + filename + '.png'
try:
objectname = header[obsparam['object']]
except KeyError:
objectname = 'Unknown Target'
html += ("<TR><TD>%d</TD><TD><A HREF=\"%s\">%s</A></TD>" + \
"<TD>%s</TD><TD>%s</TD>" + \
"<TD>%s</TD><TD>%4.2f</TD><TD>%.1f</TD>" + \
"<TD>%s</TD>\n</TR>\n") % \
(idx+1, framefilename, filename, header[obsparam['date_keyword']],
objectname,
header[obsparam['grating']],
float(header[obsparam['airmass']]),
float(header[obsparam['exptime']]),
header[obsparam['obstype']])
### create frame image
imgdat = hdulist[0].data
# clip extreme values to prevent crash of imresize
imgdat = np.clip(imgdat, np.percentile(imgdat, 1),
np.percentile(imgdat, 99))
imgdat = imresize(imgdat,
min(1., 1000. / np.max(imgdat.shape)),
interp='nearest')
#resize image larger than 1000px on one side
norm = ImageNormalize(imgdat,
interval=ZScaleInterval(),
stretch={
'linear': LinearStretch(),
'log': LogStretch()
}[imagestretch])
plt.figure(figsize=(5, 5))
img = plt.imshow(imgdat, cmap='gray', norm=norm, origin='lower')
# remove axes
plt.axis('off')
img.axes.get_xaxis().set_visible(False)
img.axes.get_yaxis().set_visible(False)
plt.savefig(framefilename,
format='png',
bbox_inches='tight',
pad_inches=0,
dpi=200)
plt.close()
hdulist.close()
del (imgdat)
html += '</TABLE>\n'
create_website(_SP_conf.index_filename, html)
### add to summary website, if requested
# if _pp_conf.use_diagnostics_summary:
# add_to_summary(header[obsparam['object']], filtername,
# len(filenames))
return None
def Extract_Spectrum(filename,
Verbose,
Spec_loc,
Diagnostic,
Spec_FWHM,
Yloc,
Live=0,
Live2=False,
Bckg='Auto'):
Out_Files = []
Out_Spec = []
telescope, obsparam = CheckInstrument([filename[0]])
DetecFlags = {'Instrument': telescope}
if telescope == 'DEVENY' or telescope == 'SOAR' or telescope == 'NOT':
for elem in filename:
# Read the fits file
hdulist = fits.open(elem)
# Extract data from the fits file
data = hdulist[0].data
# Output filename for the extracted background for errorbar computation
if Bckg == 'Auto':
Backg = np.loadtxt(elem.replace('.fits', '.txt'))
print(Backg)
# Check the telescope which is used
if telescope == 'DEVENY':
DetecFlags = {'Instrument': 'Deveny'}
if telescope == 'SOAR':
DetecFlags = {'Instrument': 'Soar'}
if telescope == 'NOT':
DetecFlags = {'Instrument': 'ALFOSC_FASU'}
# Try to detect the spectrum to extract
if Yloc == False:
Center = SP.Detect_Spectra(data, Bin=2, **DetecFlags)
else:
Center = Yloc
if telescope == 'DEVENY':
Start = (1415, Center)
if telescope == 'SOAR':
Start = (1415, Center)
if telescope == 'NOT':
Start = (250, Center)
# Start = (1415,Center)
Trace, bkg, MASK1 = SP.Fit_Trace(hdulist,
Start,
Range=15,
SClip=True,
Live=Live,
Live2=Live2,
**DetecFlags)
TR = []
for idx, elem2 in enumerate(Trace):
if elem2 > 50 and elem2 < np.size(data, 0) - 50:
TR.append(data[int(elem2) - 45:int(elem2) + 45, idx])
else:
TR.append(data[0:90, idx])
TR = np.array(TR)
hdulist[0].data = np.transpose(TR)
hdulist.writeto(elem.replace('.fits', '') + 'Trace.fits',
overwrite=True)
Out_Files.append(elem.replace('.fits', '') + 'Trace.fits')
# SSpec = []
# for FW in range(20):
# Spec1 = SP.Extract_Spectrum(data,Trace,bkg,FWHM=FW,Mask = MASK1,**DetecFlags)
# SSpec.append(Spec1)
# max_index, max_value = max(enumerate(np.nanmedian(SSpec,axis=1)/np.nanstd(SSpec,axis=1)), key=operator.itemgetter(1))
Spec1 = SP.Extract_Spectrum(data,
Trace,
bkg,
FWHM=Spec_FWHM,
Mask=MASK1,
**DetecFlags)
# Spec1 = SP.Extract_Spectrum(data,Trace,bkg,FWHM=50,Mask = MASK1,**DetecFlags)
# Error bar computation
print(Bckg)
Err = np.sqrt(np.array(Spec1).astype(float) * 2.1) + np.sqrt(
Spec_FWHM * Backg * 2.1) + np.sqrt(4)
Err = Err / 2.1
# Normalization of the spectrum
print(Spec1)
if telescope == 'NOT':
Median = np.abs(np.nanmedian(Spec1[200:250]))
else:
Median = np.abs(np.nanmedian(Spec1[1500:1600]))
Spec1N = Spec1 / Median
Err = Err / Median
fname = elem.replace('_CosmCorr', '').replace('Bckg',
'Extracted').replace(
'.fits', '.txt')
np.savetxt(fname, np.array([Spec1N, Err]).transpose())
# f = open(elem.replace('.fits','').replace('_bkgSub','').replace('_Procc','') + '.txt','w')
# Out_Spec.append(elem.replace('.fits','').replace('_bkgSub','').replace('_Procc','') + '.txt')
# for item in Spec1N:
# f.write("%s\n" % item)
# f.close()
if telescope == 'GMOSS' or telescope == 'GMOSN':
# Chip gaps for
#
# Hamamatsu:
#
# GMOS S: 61 pixels
# GMOS N: 67 pixels
#
# Other chips:
#
# 37 pixels
OffFile = Spec_loc + '_Offset.txt'
with open(OffFile, 'r') as f:
Offset = simplejson.load(f)
SpecFile = Spec_loc + '_Spec_loc.txt'
with open(SpecFile, 'r') as f:
Spec_Loc = simplejson.load(f)
Offset = np.sort(np.array(Offset).astype(int))
Spec_Loc = np.sort(Spec_Loc)
for elem in filename:
# Read the fits file
hdulist = fits.open(elem)
# Extract data from the fits file
data = hdulist[0].data
data[data == 0] = np.nan # Convert 0 to np.nan
# Output filename for the extracted background for errorbar computation
if Bckg == 'Auto':
Backg = np.loadtxt(elem.replace('.fits', '.txt'))
else:
Backg = np.loadtxt(Bckg)
# Get grating information # USED ?
Gratting = hdulist[0].header[obsparam['grating']]
# Get binning information # USED ?
Binning = hdulist[1].header['CCDSUM'][0]
# Get detector information # USED ?
Detector = hdulist[0].header['DETECTOR']
# Get the detectors gain and read-out-noise
Gain = []
RON = []
for HDU in hdulist: # loop over all header in the fits file
try:
Gain.append(HDU.header['GAIN'])
RON.append(HDU.header['RDNOISE'])
except:
pass
# Use the mean as an approximation of the gain for the whole detector (only used for errorbar computation)
Gain = np.mean(Gain)
RON = np.mean(RON)
DetecFlags = {
'Instrument': telescope,
'Gratting': Gratting,
'Binning': Binning,
'Detector': Detector
}
Off = int(hdulist[0].header['YOFFSET'])
idx = np.where(Offset == Off)[0][0]
if telescope == 'GMOSS':
Spec = Spec_Loc[2 - idx]
if telescope == 'GMOSN':
Spec = Spec_Loc[idx]
if Binning == '2':
Start = (1415, Spec)
if Binning == '4':
Start = (800, Spec)
print(Start)
Trace, bkg, MASK1 = SP.Fit_Trace(hdulist,
Start,
Range=15,
SClip=True,
**DetecFlags)
# # write a fits file with the extracted trace
# TR=[]
# for idx,elem2 in enumerate(Trace):
# if elem2 > 15:
# TR.append(data[int(elem2)-15:int(elem2)+15,idx])
# else:
# TR.append(data[0:30,idx])
# TR = np.array(TR)
# hdulist[0].data = np.transpose(TR)
# hdulist.writeto(elem.replace('.fits','') + 'Trace.fits' )
# Out_Files.append(elem.replace('.fits','') + 'Trace.fits')
# SSpec = []
# for FW in range(20):
# Spec1 = SP.Extract_Spectrum(data,Trace,bkg,FWHM=FW,Mask = MASK1,**DetecFlags)
# SSpec.append(Spec1)
# max_index, max_value = max(enumerate(np.nanmedian(SSpec,axis=1)/np.nanstd(SSpec,axis=1)), key=operator.itemgetter(1))
# Spec1 = SP.Extract_Spectrum(data,Trace,bkg,FWHM=max_index+1,Mask = MASK1,**DetecFlags)
Spec1, Backgroung = SP.Extract_Spectrum(data,
Trace,
Backg,
FWHM=Spec_FWHM,
Mask=MASK1,
**DetecFlags)
# Error bar computation
Err = np.sqrt(np.array(Spec1).astype(float) * Gain) + np.sqrt(
Spec_FWHM * Gain *
np.array(Backgroung).astype(float)) + np.sqrt(RON)
Err = Err / Gain
if Binning == '2':
Spec1N = Spec1 / np.abs(np.nanmedian(Spec1[1500:1600]))
Err = Err / np.abs(np.nanmedian(Spec1[1500:1600]))
if Binning == '4':
Spec1N = Spec1 / np.abs(np.nanmedian(Spec1[800:850]))
Err = Err / np.abs(np.nanmedian(Spec1[800:850]))
# f = open(elem.replace('.fits','').replace('_bkgSub','').replace('_Procc','') + '.txt','w')
# Out_Spec.append(elem.replace('.fits','').replace('_bkgSub','').replace('_Procc','') + '.txt')
# Error bar computation
# Err = np.sqrt(np.array(Spec1).astype(float)*Gain) + np.sqrt(Spec_FWHM*Gain*np.array(Backgroung).astype(float)) + np.sqrt(RON)
# Err = Err/Gain
# Normalization of the spectrum
# Median = np.abs(np.nanmedian(Spec1[1500:1600]))
# Spec1N = Spec1/Median
# Err= Err/Median
fname = elem.replace('_CosmCorr', '').replace('Bckg',
'Extracted').replace(
'.fits', '.txt')
np.savetxt(fname, np.array([Spec1N, Err]).transpose())
# for item in Spec1N:
# f.write("%s\n" % item)
f.close()
if Diagnostic:
diag.create_website('Extract_Log.html')
diag.add_Extract(Out_Files, Out_Spec, 'Extract_Log.html')
def WavCal(filenames,ArcsFile,OutFile,Verbose,Method,Line,Full):
# logging.info('***************************************')
# logging.info('****** Start of SP_Wavcal script ******')
# logging.info('***************************************')
Pipe_Path = _SP_conf.rootpath
Arcs = []
for elem in ArcsFile:
hdulist = fits.open(elem)
Arcs.append(hdulist[0].data)
Arcs = np.median(Arcs,axis=0)
telescope, obsparam = CheckInstrument([ArcsFile[0]])
print(telescope)
SpecA = np.loadtxt(filenames[0]).transpose()
SpecA = np.array(SpecA)
if telescope == 'DEVENY':
DetecFlags = {'Instrument':'Deveny'}
try:
Grating = int(str(hdulist[0].header[obsparam['grating']]).split('/')[0])
except:
Grating = 300
if Method == 'Template':
if telescope == 'GMOSS' or telescope == 'GMOSN':
Detector = hdulist[0].header['DETECTOR']
print(Detector)
Gratting = hdulist[0].header[obsparam['grating']]
Binning = hdulist[1].header['CCDSUM'][0]
DetecFlags = {'Instrument':telescope, 'Binning' : Binning, 'Gratting':Gratting, 'Detector':Detector}
print DetecFlags
# if telescope == 'DEVENY':
# DetecFlags = {'Instrument':'Deveny'}
# Grating = int(str(hdulist[0].header[obsparam['grating']]).split('/')[0])
if telescope == 'SOAR':
DetecFlags = {'Instrument':'Soar'}
if telescope == 'NOT':
Wav = ((10.93/2)*np.array(range(np.shape(Arcs)[1]))+3609.8)/10000
print(np.shape(Arcs)[1])
np.savetxt(OutFile,np.array([Wav,SpecA[0,:],SpecA[1,:]]).transpose())
return None
print(DetecFlags)
Wav = SP.Wav_Cal2(Arcs,**DetecFlags)
Wav = np.array(Wav)
Wav = Wav/10000
# Wav = np.flip(Wav,axis=0)
print(Wav)
# with open(filenames[0],'r') as f:
# SpecA = f.read().splitlines()
# SpecA = np.loadtxt(filenames[0]).transpose()
# SpecA = np.array(SpecA)
# SpecA = np.flip(SpecA,axis=0)
# f = open(OutFile,'w')
# for Wave,Spec in zip(Wav,SpecA):
# f.write("{} \t {} \n".format(Wave,Spec))
np.savetxt(OutFile,np.array([Wav,SpecA[0,:],SpecA[1,:]]).transpose())
if Method == 'Auto':
Dim = []
Dim.append(np.size(Arcs,0))
Dim.append(np.size(Arcs,1))
Arcs_L = Arcs[Line,:]
print(Arcs_L)
Arcs_Loc = SP.Auto_Detect_Lines(Arcs_L,Sig_Clip = 2 ,Tresh_Det = 1, Tresh_Arcs = [1, 5] )
if telescope == 'DEVENY':
# if Grating == 150:
# f = open(Pipe_Path +'/Pre_Comp_Deveny')
# Pre = pickle.load(f)
# f.close()
# WV = [7503.9,7635.1,7723.8,7948.2,8014.8,8115.3,8264.5,8424.6,8521.4,9122.9,9224.5,9657.8]
# if Grating == 300:
# f = open(Pipe_Path +'/Pre_Comp_Deveny_R300')
# Pre = pickle.load(f)
# f.close()
# WV = [3261.05, 3610.51, 3650.15, 4046.56, 4358.33, 4678.16, 4799.92, 5085.82, 5460.74, 5769.6, 5790.7, 6965.5, 7067.2, 7272.9, 7384.0]
# if Grating == 1200:
f = open(Pipe_Path + '/Pre_Comp_Deveny_R1200')
Pre = pickle.load(f)
f.close()
WV = [3125.67, 3131.70, 3252.52, 3261.05, 3341.48, 3403.65, 3467, 3612, 3649.56, 3650.15, 3663.28, 4046.56, 4077.84]
else:
f = open(Pipe_Path + '/Wav_Precomp','r')
Pre = pickle.load(f)
f.close()
WV = [7067.2,7147.0, 7272.9, 7384.0 ,7503.9, 7635.1, 7723.8, 7948.2, 8014.8, 8115.3, 8264.5, 8408.2, 8521.4, 8667.9, 9122.9, 9224.5,9354.2,9657.8, 9784.5]
One = Arcs_Loc[:9]
One = np.array(One).astype(float)
O = One-np.max(One)
O = O/np.min(O)
D1_one = np.sort(O)[1]
D2_one = np.sort(O)[2]
D3_one = np.sort(O)[3]
D4_one = np.sort(O)[4]
D5_one = np.sort(O)[5]
D6_one = np.sort(O)[6]
D7_one = np.sort(O)[7]
dist = []
for elem1,elem2,elem3,elem4,elem5,elem6,elem7 in zip(Pre[0],Pre[1],Pre[2],Pre[3],Pre[4],Pre[5],Pre[6]):
dist.append((elem1-D1_one)**2 + (elem2-D2_one)**2 + (elem3-D3_one)**2 + (elem4-D4_one)**2 + (elem5-D5_one)**2 + (elem6-D6_one)**2 + (elem7-D7_one)**2)
AS = np.argsort(dist)
max_index, max_value = min(enumerate(dist), key=operator.itemgetter(1))
# WV = [7067.2,7147.0, 7272.9, 7384.0 ,7503.9, 7635.1, 7723.8, 7948.2, 8014.8, 8115.3, 8264.5, 8408.2, 8521.4, 8667.9, 9122.9, 9224.5,9354.2,9657.8, 9784.5]
comb = combinations(WV, 9)
WV_Comb = []
for elem in comb:
WV_Comb.append(elem)
a,b = np.polyfit(WV_Comb[AS[0]],np.sort(One)[::-1],1)
# Red = a*np.array(WV)+b
# WV_Sol = []
# for elem in Arcs_Loc[:9]:
# Dist = (Red-elem)**2
# SD = np.argsort(Dist)
# WV_Sol.append(WV[SD[0]])
#
# aa= np.polyfit(WV_Sol,Arcs_Loc[:9],1,full=True)
#
# print(aa[1])
with open(filenames[0],'r') as f:
SpecA = f.read().splitlines()
SpecA = np.flip(SpecA,axis=0)
print('bla')
Wav = (np.array(range(0,len(SpecA))) - b)/a
Wav = Wav/10000
Wavel = Wav[::-1]
# Wav = (np.array(range(0,len(SpecA))) - aa[0][1])/aa[0][0]
# Wav = Wav/10000
# Wavel = Wav[::-1]
f = open(OutFile,'w')
for Wave,Spec in zip(Wavel,SpecA):
f.write("{} \t {} \n".format(Wave,Spec))
#
# logging.info('*************************************')
# logging.info('****** End of SP_Wavcal script ******')
# logging.info('*************************************')
#
return None
def Detect_Spectra(filename, OutName, DetecLim):
print(DetecLim)
Offset = []
telescope, obsparam = CheckInstrument([filename[0]])
for idx, elem in enumerate(filename):
hdulist = fits.open(elem)
Bin = int(hdulist[1].header['CCDSUM'][0])
data = hdulist[0].data
hdulist = fits.open(elem)
Offset.append(hdulist[0].header['YOFFSET'])
if idx == 0:
print(idx)
dataTot = data
else:
print(idx)
dataTot += data
hdulist[0].data = dataTot
FitsName = OutName + '.fits'
hdulist.writeto(FitsName)
Sig = float(DetecLim)
Spec = SP.Auto_Detect_Spectra(OutName + '.fits', Sig)
print(Spec)
PltScale = obsparam['pixscale']
Offset = np.array(Offset)
if telescope == 'GMOSS':
Off = -Offset / (PltScale * Bin)
if telescope == 'GMOSN':
Off = Offset / (PltScale * Bin)
print(Spec)
Spec_Loc = SP.Get_Spectra(Spec, Off)
if Bin == 4:
Spec_Loc = Spec_Loc + 400
if Bin == 2:
if np.shape(hdulist[0].data)[0] > 600:
Spec_Loc = Spec_Loc + 800
else:
Spec_Loc = Spec_Loc
SpecLocName = OutName + '_Spec_loc.txt'
with open(SpecLocName, 'w') as f:
simplejson.dump(list(Spec_Loc), f)
OffsetName = OutName + '_Offset.txt'
with open(OffsetName, 'w') as f:
simplejson.dump(list(Offset), f)
Offset = np.sort(np.array(Offset).astype(int))
Spec_Loc = np.sort(Spec_Loc)
for idx, elem in enumerate(filename):
hdulist = fits.open(elem)
data = hdulist[0].data
Range = range(np.min(Spec_Loc) - 20, np.max(Spec_Loc))
bckg = np.nanmedian(data[Range, :], axis=0)
np.savetxt(
elem.replace('.fits', '').replace('_Procc', '') + '_Bckg' + '.txt',
bckg)
print(Spec_Loc)
print(Off)