/
saltobsid.py
223 lines (180 loc) · 9.36 KB
/
saltobsid.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
################################# LICENSE ##################################
# Copyright (c) 2016, South African Astronomical Observatory (SAAO) #
# All rights reserved. #
############################################################################
from pyraf import iraf
import pyfits, string, os, glob
from saltsafekey import fastmode
import saltsafeio as saltio
import saltsafekey as saltkey
import saltsafestring as saltstring
from saltsafelog import logging
from salterror import SaltError, SaltIOError
debug=True
# -----------------------------------------------------------
# core routine
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True):
"""Split data into their different data directories
"""
with logging(logfile,debug) as log:
# are the arguments defined
pids = saltio.argunpack('propcode',propcode)
# check observation log file exists
obslog = obslog.strip()
saltio.fileexists(obslog)
#open the observing log
obstruct = saltio.openfits(obslog)
obstab = saltio.readtab(obstruct[1],obslog)
saltio.closefits(obstruct)
#read in the file information
filenames = saltstring.listfunc(obstab.field('filename'),'lstrip')
instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip')
proposers = saltstring.listfunc(obstab.field('proposer'),'clean')
propids = saltstring.listfunc(obstab.field('propid'),'clean')
ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean')
ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean')
gainsets = saltstring.listfunc(obstab.field('gainset'),'clean')
rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean')
detmodes = saltstring.listfunc(obstab.field('detmode'),'clean')
filters = saltstring.listfunc(obstab.field('filter'),'clean')
gratings = saltstring.listfunc(obstab.field('grating'),'clean')
gr_angles = obstab.field('gr-angle')
ar_angles = obstab.field('ar-angle')
# Create the list of proposals
try:
pids=saltio.cleanpropcode(pids, propids)
except SaltIOError:
#throw a warning adn exit if not data needs to be filterd
log.warning('No data to filter\n', with_stdout=verbose)
return
# check paths exist, end with a "/" and convert them to absolute paths
rawpath = saltio.abspath(rawpath)
prodpath = saltio.abspath(prodpath)
outpath = saltio.abspath(outpath)
#create the symlink raw path
rawsplit=rawpath.strip().split('/')
symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2])
prodsplit=prodpath.strip().split('/')
symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2])
# create PI directories
for pid in pids:
saltio.createdir(outpath+pid)
saltio.createdir(outpath+pid+'/raw')
saltio.createdir(outpath+pid+'/product')
#copy the data that belongs to a pid into that directory
log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose)
#copy data for a given proposal to the raw and produce directories
for i in range(len(obstab)):
if os.path.exists(outpath+obstab[i]['propid']):
if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']:
fname=obstab[i]['filename']
pdir=obstab[i]['propid']
detmode=obstab[i]['detmode']
linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#look through the bias/flat/arc/standard data to see if there is any relavent data
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
caldata=['ZERO', 'FLAT', 'ARC']
biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED']
flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
calproplist=['CAL_SPST']
#Include bias frames
log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a zero, check to see what other data have the same settings
if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], biasheader_list):
prop_list.append(obstab[i]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, fprefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include calibration frames
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a flat, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a arc, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], archeader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a calibration standard, see what other data have the same settings
if obstab[i]['PROPID'].strip().upper() in calproplist:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
if pdir!=obstab[i]['propid']:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include master (bias or flat) frames
log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose)
masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*')
for bimg in masterlist:
struct=pyfits.open(bimg)
bdict={}
prop_list=[]
for k in biasheader_list:
bdict[k]=saltkey.get(k, struct[0])
for i in range(len(obstab)):
if comparefiles(obstab[i], bdict, biasheader_list):
prop_list.append(obstab[i]['PROPID'])
struct.close()
#copy the files over to the directory
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
fname=os.path.basename(bimg)
infile = symprodpath+fname
link = outpath+pdir+'/product/'+fname
saltio.symlink(infile,link,clobber)
message='Copying %s to %s' % (fname ,pdir)
log.message(message, with_header=False, with_stdout=verbose)
def comparefiles(afile, bfile, headers):
"""Compare the headers in two sets of tab entries
and see if they are the same or different
"""
for k in headers:
if isinstance(afile[k], str):
if afile[k].strip().upper()!=bfile[k].strip().upper(): return False
if isinstance(afile[k], int):
if afile[k]==bfile[k]: return False
if isinstance(afile[k], float):
if abs(afile[k]-bfile[k])>0.01: return False
return True
def linkfiles(fname, pdir, detmode, rawpath, prodpath, outpath, prefix='mbxgp', fprefix='bxgp', clobber=False):
#copy the raw data
infile=rawpath+fname
link=outpath+pdir+'/raw/'+fname
saltio.symlink(infile,link,clobber)
#copy the product data
if not fastmode(detmode):
pfname=prefix+fname
else:
pfname=fprefix+fname
infile = prodpath+pfname
link = outpath+pdir+'/product/'+pfname
if fname[0] in ['S', 'P', 'H', 'R']:
saltio.symlink(infile,link,clobber)
# -----------------------------------------------------------
# main code
if not iraf.deftask('saltobsid'):
parfile = iraf.osfn("pipetools$saltobsid.par")
t = iraf.IrafTaskFactory(taskname="saltobsid",value=parfile,function=saltobsid, pkgname='pipetools')