def copy_exposure(self, filename):
"""
Create a copy of the original input to be renamed and used for multi-visit processing.
New exposure filename needs to follow the MVM naming convention:
hst_skycell-p<PPPP>x<XX>y<YY>_<instr>_<detector>_<filter>-<layer>_<ipppssoo>_fl[ct].fits
Parameters
----------
filename : str
Original pipeline filename for input exposure
Returns
-------
sce_filename : str
New MVM-compatible HAP filename for input exposure
"""
suffix = filename.split("_")[1]
sce_filename = '_'.join([self.product_basename, suffix])
log.info("Copying {} to MVM input: \n {}".format(filename, sce_filename))
try:
shutil.copy(filename, sce_filename)
except PermissionError:
pass
# Add HAPLEVEL keyword as required by pipeline processing
fits.setval(sce_filename, 'HAPLEVEL', value=0, comment='Classificaion level of this product')
return sce_filename
def chkeywd(fits_file, keyword, value, ext_number):
if ext_number is None:
extension_number = 0
else:
extension_number = int(ext_number)
# convert the keyword value input to the right type
keyword = keyword.upper()
if extension_number == 0:
sample_kyewd_dict = shkvd.keywd_dict
new_value = keywd_in_dict_or_new_keywd(sample_kyewd_dict,
extension_number, keyword,
value)
elif extension_number == 1:
sample_kyewd_dict = shkvd.keywd_dict["wcsinfo"]
new_value = keywd_in_dict_or_new_keywd(sample_kyewd_dict,
extension_number, keyword,
value)
else:
new_value = new_keywd(extension_number, value)
# now change the the value
fits.setval(fits_file, keyword, extension_number, value=new_value)
print('\n * Script change_keywd.py finished * \n')
def setwelldepth(date, filerange, welldepth):
"""
Manually set the well depth in a NIRI FITS header.
date: date of the FITS file, e.g. 20201122
filerange: range of file numbers, e.g. 1-3,5,7-9
welldepth: the desired well depth, either 'shallow' or 'deep'
"""
biasvoltage = {'deep': -0.87, 'shallow': -0.60} # Detector bias voltage = VDDUC - VDET
for i in expand(filerange):
filename = 'N%sS%04d.fits' % (date,i)
try:
vdduc1 = fits.getval(filename, 'VDDUC')
vdduc2 = fits.getval(filename, 'A_VDDUC')
vdet1 = fits.getval(filename, 'VDET')
vdet2 = fits.getval(filename, 'A_VDET')
except:
print('%s Not found' % filename)
continue
bias1 = vdduc1 - vdet1
bias2 = vdduc2 - vdet2
if abs(bias1 - biasvoltage[welldepth]) < 0.05 and \
abs(bias2 - biasvoltage[welldepth]) < 0.05:
#print('%s %.3f - %.3f = %.3f %s' % (filename, vdduc1, vdet1, bias1, welldepth.upper()))
print('%s is already set to %s well' % (filename, welldepth.upper()))
else:
newvdet1 = vdduc1 - biasvoltage[welldepth]
newvdet2 = vdduc2 - biasvoltage[welldepth]
#print('%s %.3f - %.3f = %.3f --> setting VDET = %.2f' % (filename, vdduc1, vdet1, bias1, newvdet1))
print('%s is now set to %s well' % (filename, welldepth.upper()))
fits.setval(filename, 'VDET', value=newvdet1)
fits.setval(filename, 'A_VDET', value=newvdet2)
return
def calibrate_samples(samples, tile_key="TILE", xycorr=True):
comment = "Magnitude zero point"
zps = get_zps()
zpcorr = get_zp_correction()
for sample in samples:
wdir = os.path.join(context.data_dir, sample, "cutouts")
if not os.path.exists(wdir):
continue
galaxies = sorted(os.listdir(wdir))
desc = "Calibrating galaxies of sample {}".format(sample)
for galaxy in tqdm(galaxies, desc=desc):
galdir = os.path.join(wdir, galaxy)
stamps = sorted(
[_ for _ in os.listdir(galdir) if _.endswith(".fits")])
for stamp in stamps:
filename = os.path.join(galdir, stamp)
h = fits.getheader(filename, ext=1)
tile = h[tile_key]
filtername = h["FILTER"]
idx = np.where((zps["FIELD"] == tile))[0]
if len(idx) == 0:
print("Calibration not found for tile: {}".format(tile))
continue
zp = float(zps[idx][filtername].data[0])
if xycorr:
x0 = h["X0TILE"]
y0 = h["Y0TILE"]
zp += float(zpcorr[filtername](x0, y0)[0][0])
fits.setval(filename,
"MAGZP",
value=round(zp, 7),
comment=comment,
ext=1)
def multiple_data_products_post_save(dps):
logger.info(f'Running post save hook for multiple DataProducts: {dps}')
for dp in dps:
if dp.data.path.endswith('-e91.fits.fz'):
logger.info(f'Starting spikepipe on {dp}')
spikey_thread = threading.Thread(target=run_spikepipe, args=[dp])
spikey_thread.start()
elif dp.data.path.endswith('.tar.gz'):
logger.info(f'Saving extracted spectrum from {dp}')
with tarfile.open(dp.data.path) as f:
for member in f.getmembers():
if member.name.endswith('_2df_ex.fits'):
extracted_spectrum = DataProduct(
target=dp.target,
observation_record=dp.observation_record,
data_product_type='spectroscopy',
data=File(f.extractfile(member), name=member.name))
extracted_spectrum.save()
fits.setval(extracted_spectrum.data.path,
'ORIGIN',
value='LCOGT') # for SpectroscopyProcessor
run_custom_data_processor(extracted_spectrum, {})
else:
logger.info(f'{dp} has no post save hook')
def append_bc_to_reduced_files(date, root='/Volumes/BERGRAID/data/veloce/'):
print('Appending barycentric corrections to the reduced spectra of ' +
str(date) + '...')
acq_list, bias_list, dark_list, flat_list, skyflat_list, domeflat_list, arc_list, thxe_list, laser_list, laser_and_thxe_list, stellar_list, unknown_list = get_obstype_lists(
root + 'raw_goodonly/' + date + '/')
stellar_list.sort()
obsnames = short_filenames(stellar_list)
print('Calculating barycentric correction for stellar observation:')
for i, (file, obsname) in enumerate(zip(stellar_list, obsnames)):
print(str(i + 1) + '/' + str(len(stellar_list)))
bc = get_barycentric_correction(file)
bc = np.round(bc, 2)
if np.isnan(bc):
bc = ''
# write the barycentric correction into the FITS header of both the quick-extracted and the optimal-extracted reduced spectrum files
sublist = glob.glob(root + 'reduced/' + date + '/*' + obsname +
'*extracted*')
for outfn in sublist:
pyfits.setval(outfn,
'BARYCORR',
value=bc,
comment='barycentric velocity correction [m/s]')
print('DONE!')
return
def change_keyword2ips_value(ips_keywd_dict,
st_pipe_ready_dict,
ips_keywd,
st_pipe_ready_keywd,
st_pipe_ready_file,
verbose=False):
"""
Function changes the keyword value to match IPS value.
:param ips_keywd_dict: dictionary, IPS header
:param st_pipe_ready_dict: dictionary, STScI pipeline-ready file header
:param st_pipe_ready_keywd: string, keyword with STScI spelling
:param ips_keywd: string, keyword with IPS spelling
:param st_pipe_ready_file: string, path and name of the fits file that is STScI pipeline-ready
:param verbose: boolean
:return: nothing
"""
for key, val in st_pipe_ready_dict.items():
if key == st_pipe_ready_keywd:
if ips_keywd in ips_keywd_dict:
fits.setval(st_pipe_ready_file,
st_pipe_ready_keywd,
value=ips_keywd_dict[ips_keywd])
if verbose:
print('Modified keyword: ', key, ' old_value=', val,
' new_value=', ips_keywd_dict[ips_keywd])
else:
if verbose:
print(
'IPS keyword ', ips_keywd, ', corresponding to ',
st_pipe_ready_keywd,
'in STScI pipeline, not found in header of IPS file.')
break
def test_superdark3_pass(self):
fits.setval(self.superdark, 'PCTE_VER', value='0.1_gamma')
assert_equals(
superdark_hash(superdark=self.superdark),
superdark_hash(
pctetab=self.pctetab,
files=['abc_cte.fits', 'def_cte.fits', 'hij_cte.fits']))
def make_refbias(input_list, refbias_name='refbias.fits'):
"""Create a refbias FITS file
Parameters
----------
input_list : list
list of input bias files
refbias_name : str
name of the output bias reference file
"""
print('#-------------------------------#')
print('# Running refbias #')
print('#-------------------------------#')
print('Making refbias %s' % (refbias_name))
joined_out = refbias_name.replace('.fits', '_joined.fits')
print('Joining images to %s' % joined_out)
functions.msjoin(input_list, joined_out)
print('Checking for cosmic ray rejection')
crj_filename = functions.crreject(joined_out)
shutil.copy(crj_filename, refbias_name)
flag_hot_pixels(refbias_name)
functions.update_header_from_input(refbias_name, input_list)
fits.setval(refbias_name, 'TASKNAME', ext=0, value='refbias')
print('Cleaning up...')
functions.RemoveIfThere(crj_filename)
functions.RemoveIfThere(joined_out)
print('refbias done for {}'.format(refbias_name))
def Extract_BeamCutout(self):
beams = self.grp.get_beams(self.galaxy_id)
pa = -1
for i in beams:
if i.grism.filter == 'G102':
if pa != i.get_dispersion_PA():
pa = i.get_dispersion_PA()
i.write_fits(root='../beams/o{0}'.format(pa), clobber=True)
fits.setval('../beams/o{0}_{1}.{2}.A.fits'.format(
pa, self.galaxy_id, i.grism.filter),
'EXPTIME',
ext=0,
value=fits.open(
'../beams/o{0}_{1}.{2}.A.fits'.format(
pa, self.galaxy_id,
i.grism.filter))[1].header['EXPTIME'])
pa = -1
for i in beams:
if i.grism.filter == 'G141':
if pa != i.get_dispersion_PA():
pa = i.get_dispersion_PA()
i.write_fits(root='../beams/o{0}'.format(pa), clobber=True)
fits.setval('../beams/o{0}_{1}.{2}.A.fits'.format(
pa, self.galaxy_id, i.grism.filter),
'EXPTIME',
ext=0,
value=fits.open(
'../beams/o{0}_{1}.{2}.A.fits'.format(
pa, self.galaxy_id,
i.grism.filter))[1].header['EXPTIME'])
def write_template(filename,
flux,
wave,
header=None,
hdrref=None,
clobber=False):
if not header and hdrref: header = pyfits.getheader(hdrref)
hdu = pyfits.PrimaryHDU(header=header)
warnings.resetwarnings(
) # supress nasty overwrite warning http://pythonhosted.org/pyfits/users_guide/users_misc.html
warnings.filterwarnings('ignore', category=UserWarning, append=True)
hdu.writeto(filename, clobber=clobber, output_verify='fix')
warnings.resetwarnings()
warnings.filterwarnings('always', category=UserWarning, append=True)
if isinstance(flux, np.ndarray):
pyfits.append(filename, flux)
pyfits.append(filename, wave)
else:
# pad arrays with zero to common size
maxpix = max(arr.size for arr in flux if isinstance(arr, np.ndarray))
flux_new = np.zeros((len(flux), maxpix))
wave_new = np.zeros((len(flux), maxpix))
for o, arr in enumerate(flux):
if isinstance(arr, np.ndarray): flux_new[o, :len(arr)] = arr
for o, arr in enumerate(wave):
if isinstance(arr, np.ndarray): wave_new[o, :len(arr)] = arr
pyfits.append(filename, flux_new)
pyfits.append(filename, wave_new)
pyfits.setval(filename, 'EXTNAME', value='SPEC', ext=1)
pyfits.setval(filename, 'EXTNAME', value='WAVE', ext=2)
def get_iris(num, direc, band):
bd = str(band)
iras_number = str(int(num))
if num < 100:
iras_number = '0' + iras_number
if num < 10:
iras_number = '0' + iras_number
hcon = 0
hnum = str(hcon)
result = glob.glob(direc + '/?' + iras_number + '?' + bd + '?' + hnum +
'.*')
count = len(result)
if count > 0:
fits.setval(result[0], 'CDELT3', value=1) #TEST: modify header
ifile = fits.open(result[0])
imap = ifile[0].data[0]
header = fits.open(result[0])[0].header
#sxaddpar sets LONPOLE to 180, but already there.
bad = np.where((imap < -5) + (imap == 0))[0]
nbbad = len(bad)
if nbbad > 0:
map[bad] = -32768
else:
print("error: could not find data file for ISSA number", iras_number,
"and band", bd)
return imap, header
def main():
parser = OptionParser("usage: %prog key value image1.fits [image2 ...]")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="count",
help="Enable 'DEBUG' output in python log")
parser.add_option("-e",
"--ext",
dest="hdu",
default=0,
metavar="HDU",
help="HDU where key will be setted.")
(options, args) = parser.parse_args()
if options.verbose is None:
logger.setLevel('WARN')
elif options.verbose == 1:
logger.setLevel('INFO')
else:
logger.setLevel('DEBUG')
key = args[0]
value = args[1]
files = args[2:]
for i in files:
logger.debug("setting key:{} with value:{} on {}".format(
key, value, i))
fits.setval(i, key, value=value, ext=options.hdu)
def copy_exposure(self, filename):
"""
Create a copy of the original input to be renamed and used for single-visit processing.
New exposure filename needs to follow the convention:
hst_<propid>_<obsetid>_<instr>_<detector>_<filter>_<ipppssoo>_fl[ct].fits
Parameters
----------
filename : str
Original pipeline filename for input exposure
Returns
-------
edp_filename : str
New SVM-compatible HAP filename for input exposure
"""
suffix = filename.split("_")[1]
edp_filename = self.basename + \
"_".join(map(str, [self.filters, filename[:8], suffix]))
log.info("Copying {} to SVM input: \n {}".format(filename, edp_filename))
shutil.copy(filename, edp_filename)
# Add HAPLEVEL keyword as required by pipeline processing
fits.setval(edp_filename, 'HAPLEVEL', value=0, comment='Classificaion level of this product')
return edp_filename
def test_superdark3_reject(self):
fits.setval(self.pctetab, 'nsemodel', value=6)
assert_not_equals(
superdark_hash(superdark=self.superdark),
superdark_hash(
pctetab=self.pctetab,
files=['abc_cte.fits', 'def_cte.fits', 'hij_cte.fits']))
def test_superdark4_reject(self):
fits.setval(self.pctetab, 'PCTE_VER', value='0.2_alpha')
assert_not_equals(
superdark_hash(superdark=self.superdark),
superdark_hash(
pctetab=self.pctetab,
files=['abc_cte.fits', 'def_cte.fits', 'hij_cte.fits']))
def append_relints_to_FITS(relints, fn, nfib=19):
#prepare information on fibres
fibinfo = [
'inner ring', 'outer ring (2)', 'outer ring (1)', 'inner ring',
'outer ring (2)', 'outer ring (1)', 'inner ring', 'outer ring (2)',
'outer ring (1)', 'central', 'outer ring (2)', 'outer ring (1)',
'inner ring', 'outer ring (2)', 'outer ring (1)', 'inner ring',
'outer ring (2)', 'outer ring (1)', 'inner ring'
]
#numbering of fibres as per Jon Lawrence's diagram (email from 22/02/2018
#ie correspoding to the following layout:
# L S1 S3 S4 X I O2 O1 I O2 O1 I O2 O1 C O2 O1 I O2 O1 I O2 O1 I X S2 S5 ThXe
# L S1 S3 S4 X 2 19 8 3 9 10 4 11 12 1 13 14 5 15 16 6 17 18 7 X S2 S5 ThXe
#here O1 is the outer ring that is slightly further away from the centre!!!
fibnums = [
2, 19, 8, 3, 9, 10, 4, 11, 12, 1, 13, 14, 5, 15, 16, 6, 17, 18, 7
]
#the pseudoslit is reversed w.r.t. my simulations - we therefore turn the fibnums array around
fibnums = fibnums[::-1]
#loop over all fibres
for i in np.arange(nfib):
pyfits.setval(fn,
'RELINT' + str(i + 1).zfill(2),
value=relints[i],
comment='fibre #' + str(fibnums[i]) + ' - ' +
fibinfo[i] + ' fibre')
return
def test_superdark2_reject(self):
fits.setval(self.superdark, 'PCTENSMD', value=5)
assert_not_equals(
superdark_hash(superdark=self.superdark),
superdark_hash(
pctetab=self.pctetab,
files=['abc_cte.fits', 'def_cte.fits', 'hij_cte.fits']))
def putMJD(inim, case):
hdr = fits.getheader(inim)
if case == 'C1':
mjdval = Time(hdr['JD'], format='jd', scale='utc').mjd
fits.setval(inim, 'MJD', value=mjdval, comment='MJD appended')
print 'header updated', fits.getheader(inim)['MJD']
elif case == 'C2':
mjdval = Time(hdr['J_DATE'], format='jd', scale='utc').mjd
fits.setval(inim, 'MJD', value=mjdval, comment='MJD appended')
print 'header updated', fits.getheader(inim)['MJD']
elif case == 'C3':
mjdval = Time(hdr['DATE-OBS'], format='isot', scale='utc').mjd
fits.setval(inim, 'MJD', value=mjdval, comment='MJD appended')
print 'header updated', fits.getheader(inim)['MJD']
elif case == 'C4':
datetimestr = hdr['DATE-OBS'] + 'T' + hdr['TIME-OBS']
mjdval = Time(datetimestr, format='isot', scale='utc').mjd
fits.setval(inim, 'MJD', value=mjdval, comment='MJD appended')
elif case == 'C5':
datetimestr = hdr['DATE-OBS'] + 'T' + hdr['UT']
mjdval = Time(datetimestr, format='isot', scale='utc').mjd
fits.setval(inim, 'MJD', value=mjdval, comment='MJD appended')
else:
print 'Not in known cases.'
def make_refbias(input_list, refbias_name='refbias.fits'):
"""Create a refbias FITS file
Parameters
----------
input_list : list
list of input bias files
refbias_name : str
name of the output bias reference file
"""
print('#-------------------------------#')
print('# Running refbias #')
print('#-------------------------------#')
print('Making refbias %s' % (refbias_name))
joined_out = refbias_name.replace('.fits', '_joined.fits')
print('Joining images to %s' % joined_out)
functions.msjoin(input_list, joined_out)
print('Checking for cosmic ray rejection')
crj_filename = functions.crreject(joined_out)
shutil.copy(crj_filename, refbias_name)
flag_hot_pixels(refbias_name)
functions.update_header_from_input(refbias_name, input_list)
fits.setval(refbias_name, 'TASKNAME', ext=0, value='refbias')
print('Cleaning up...')
functions.RemoveIfThere(crj_filename)
functions.RemoveIfThere(joined_out)
print('refbias done for {}'.format(refbias_name))
def main():
global deep_check
pool = multiprocessing.Pool(processes=4)
conf = json.load(open('/data7/DEEP/python/config.json'))
deep_check = tf.keras.models.load_model(conf["ML_model"])
img_list = open('img.list').readlines()
img_list = [i.strip() for i in img_list]
list(pool.map(gen_diff, img_list))
trim_files = glob.glob('*trim*fits')
for i in trim_files:
os.remove(i)
diff_img_list = glob.glob('*.diff.fits')
check_result = check_img(diff_img_list)
diff_img_list = np.array(diff_img_list)
check_result = np.array(check_result)
for n, j in enumerate(diff_img_list):
print('{}: {}'.format(j, check_result[n]))
if check_result[n] >= 0:
fits.setval(j, 'ml_score', value=check_result[n])
else:
gen_diff(j.replace('.diff.fits', '.fits'), num=1)
ml_score = check_img([j])[0]
print('{}: {}'.format(j, ml_score))
if ml_score > -5:
fits.setval(j, 'ml_score', value=check_result[n])
else:
os.remove(j)
trim_files = glob.glob('*trim*fits')
for i in trim_files:
os.remove(i)
def make_basedark(input_list, refdark_name='basedark.fits', bias_file=None):
"""Make a monthly baseline dark from the input list.
Parameters
----------
input_list: list
list of input dark files
refdark_name: str
name of the output reference file
bias_file: str
bias file to be used in calibration
"""
print('#-------------------------------#')
print('# Running basedark #')
print('#-------------------------------#')
print('output to: %s' % refdark_name)
print('with biasfile %s' % bias_file)
#-- bias subtract data if not already done
flt_list = [functions.bias_subtract_data(item, bias_file) for item in input_list]
for filename in flt_list:
texpstrt = fits.getval(filename, 'texpstrt', 0)
if texpstrt > 52091.0:
functions.apply_dark_correction(filename, texpstrt)
joined_filename = refdark_name.replace('.fits', '_joined.fits')
crj_filename = joined_filename.replace('.fits', '_crj.fits')
#if not bias_file:
# raise IOError('No biasfile specified, this task needs one to run')
print('Joining images')
functions.msjoin(flt_list, joined_filename)
print('Performing CRREJECT')
crdone = functions.bd_crreject(joined_filename)
if not crdone:
functions.bd_calstis(joined_filename, bias_file)
functions.normalize_crj(crj_filename)
shutil.copy(crj_filename, refdark_name)
update_sci(refdark_name)
find_hotpix(refdark_name)
functions.update_header_from_input(refdark_name, input_list)
fits.setval(refdark_name, 'TASKNAME', ext=0, value='BASEDARK')
print('Cleaning...')
functions.RemoveIfThere(crj_filename)
functions.RemoveIfThere(joined_filename)
#map(functions.RemoveIfThere, flt_list)
print('basedark done for {}'.format(refdark_name))
def test_restore_simple(self):
# test restore on simple fits format
altwcs.archiveWCS(self.simplefits, ext=0, wcskey='R')
pyfits.setval(self.simplefits, ext=0, keyword='CRVAL1R', value=1)
altwcs.restoreWCS(self.simplefits, ext=0, wcskey='R')
wo = wcsutil.HSTWCS(self.simplefits, ext=0, wcskey='R')
ws = wcsutil.HSTWCS(self.simplefits, ext=0)
compare_wcs(ws, wo)
def test_restore_simple(self):
# test restore on simple fits format
altwcs.archive_wcs(self.simplefits, ext=0, wcskey='R')
pyfits.setval(self.simplefits, ext=0, keyword='CRVAL1R', value=1)
altwcs.restoreWCS(self.simplefits, ext=0, wcskey='R')
wo = wcsutil.HSTWCS(self.simplefits, ext=0, wcskey='R')
ws = wcsutil.HSTWCS(self.simplefits, ext=0)
compare_wcs(ws, wo)
def fitsfix_6pt5(fitsfile):
'''fixes the wcs header of a lenstool file generated using version 6.5'''
fits.setval(fitsfile,
'CRPIX1',
value=fits.getval(fitsfile, 'CRPIX1') + 0.5)
fits.setval(fitsfile,
'CRPIX2',
value=fits.getval(fitsfile, 'CRPIX2') + 0.5)
def destripe_plus(sy,scimask1='None',scimask2='None',cte_correct=True):
# verify that the RAW image exists in cwd
cwddir = os.getcwd()
if not os.path.exists(cwddir+'/'+sy):
sys.exit(sy + " RAW file does not exist. Quitting now!")
#check if 2K subarray or full frame
is_sub2K = False
header = fits.getheader(sy)
ctecorr = header['PCTECORR']
aperture = header['APERTURE']
subarray_list=['WFC1-2K', 'WFC1-POL0UV', 'WFC1-POL0V', 'WFC1-POL60V',
'WFC1-POL60UV', 'WFC1-POL120V', 'WFC1-POL120UV', 'WFC1-SMFL',
'WFC1-IRAMPQ', 'WFC1-MRAMPQ', 'WFC2-2K', 'WFC2-ORAMPQ',
'WFC2-SMFL', 'WFC2-POL0UV', 'WFC2-POL0V', 'WFC2-MRAMPQ']
if header['SUBARRAY'] and cte_correct:
if aperture in subarray_list:
is_sub2K = True
else:
print 'Using non-2K subarray, turning CTE correction off'
cte_correct = False
# run ACSCCD on RAW subarray
acsccd.acsccd(sy)
# execute destriping of the subarray (post-SM4 data only)
acs_destripe.clean(sy.replace('raw','blv_tmp'), 'strp', clobber=False, maxiter=15, sigrej=2.0, mask1=scimask1, mask2=scimask2)
os.rename(sy.replace('raw','blv_tmp_strp'),sy.replace('raw','blv_tmp'))
#update subarray header
if is_sub2K and cte_correct:
fits.setval(sy.replace('raw','blv_tmp'),'PCTECORR',value='PERFORM')
# perform CTE correction on destriped image
if cte_correct:
if ctecorr == 'PERFORM':
acscte.acscte(sy.replace('raw','blv_tmp'))
else:
print "PCTECORR not set to 'PERFORM', cannot run CTE correction"
cte_correct = False
# run ACS2D to get FLT and FLC images
acs2d.acs2d(sy.replace('raw','blv_tmp'))
if cte_correct:
acs2d.acs2d(sy.replace('raw','blc_tmp'))
# delete intermediate files
os.system("rm "+sy.replace('raw','blv_tmp'))
if cte_correct:
os.system("rm "+sy.replace('raw','blc_tmp'))
print '__________________________________________________________'
print " "
print "FLT : ", sy.replace('raw','flt')
if cte_correct:
print "FLC : ", sy.replace('raw','flc')
print " "
def save_fits_after_align(image_path, res_data):
hdu = fits.open(image_path)
hdu[0].data = res_data
hdu.writeto(image_path[:-4]+'_aligned'+image_path[-4:], overwrite =True)
if os.path.isfile(image_path[:-4]+'_aligned'+image_path[-4:]) :
fits.setval(image_path[:-4]+'_aligned'+image_path[-4:], 'NOTES', value='aligned by [email protected]')
return print('aligned fits file is created')
else :
return print('failure creating aligned fits file...')
def putmjd(i):
try:
hdr = fits.getheader(i)
mjdval = Time(hdr['DATE-OBS'], format='isot', scale='utc').mjd
fits.setval(i, 'MJD', value=mjdval, comment='MJD appended')
print('header updated', fits.getheader(i)['MJD'])
except:
print(i, 'DATE-OBS keyword is missing')
def Write(self):
if os.path.isfile(self.name):
os.remove(self.name)
self.hdu.writeto(self.name)
self.hdu.header['i'] = '1'
# Include data of the image in the Header of the .fits file
for j in self.hdrData.keys():
fits.setval(self.name, j, value=self.hdrData[j])
def track_versions(self, files, apply_to="_raw"):
"""Add version keywords to raw_files(files)."""
import caldp
csys_ver = os.environ.get("CSYS_VER", "UNDEFINED")
for filename in self.raw_files(files):
if apply_to in filename:
fits.setval(filename, "CSYS_VER", value=csys_ver)
fits.setval(filename, "CALDPVER", value=caldp.__version__)
def scamp_astrometry_net0(im, projection='TAN', DETECT_THRESH='3'):
contnum = scamp_net0(im, projection=projection, DETECT_THRESH='3')
headmerge(im)
fits.setval('sa' + im, 'FLXSCALE', value=1)
fits.setval('sa' + im,
'SCAMPCON',
value=contnum,
hdrcomment='SCAMP cont. num')
delhdrkey('sa' + im, kwds=kwds)
return 'Done'
def scamp_non_astrometry_net(im, projection='TAN'):
contnum = scamp_net(im, astref=False, projection=projection)
swarpcom = 'swarp -c ' + astswarpconfig + ' ' + im + ' -IMAGEOUT_NAME sa' + im
os.system(swarpcom)
fits.setval('sa' + im, 'FLXSCALE', value=1)
fits.setval('sa' + im,
'SCAMPCON',
value=contnum,
hdrcomment='SCAMP cont. num')
delhdrkey('sa' + im, kwds=kwds)
def headerRenameKey2(self, key, newkey):
for filename in self._matchingFiles:
print('rename %s: %s= %s' % (filename, key, newkey))
try:
value = fits.getval(filename, key)
fits.setval(filename, newkey, value)
fits.delval(filename, key)
except Exception as e:
print(str(e))
pass
def edit_cal_files():
for subdir in subdirlist:
mydir = os.path.join(rootdir, subdir)
rawfiles = glob.glob(os.path.join(mydir, '*_rawtag.fits'))
for ref in reffiles:
outputfolder = os.path.join(mydir, ref.split('_')[1].split('.')[0])
for myfile in rawfiles:
fits.setval(myfile, 'XTRACTAB', value = '/user/esnyder/nuvxtractabproj/new1dxfiles/' + str(ref), ext = 0)
asnfiles = glob.glob(os.path.join(mydir, '*_asn.fits'))
parallel_cal(asnfiles, outputfolder)
def header_info(fitsfiles, switch_on):
'''
setting values to "PERFORM" from "OMIT"
'''
for f in fitsfiles:
for kwd in switch_on:
fits.setval(f, kwd, value='PERFORM')
command_line_input = F'crds bestrefs --files {f} --sync-references=1 --update-bestrefs'
os.system(command_line_input)
return
def test_restore_wcs_mem(self):
# test restore on an HDUList object
altwcs.archiveWCS(self.acs_file, ext=[('SCI', 1), ('SCI', 2)], wcskey='T')
pyfits.setval(self.acs_file, ext=('SCI', 1), keyword='CRVAL1', value=1)
pyfits.setval(self.acs_file, ext=('SCI', 2), keyword='CRVAL1', value=1)
f = pyfits.open(self.acs_file, mode='update')
altwcs.restoreWCS(f, ext=1, wcskey='T')
f.close()
w1o = wcsutil.HSTWCS(self.acs_file, ext=1, wcskey='T')
w1 = wcsutil.HSTWCS(self.acs_file, ext=1)
compare_wcs(w1, w1o)
def wcs_photom(image,cat_file='nocat',conf_file='noconf',appsize=2.0,verbose=False):
if not wcs_succeeded(image):
return 'failed'
outname = image + '.phot'
casutools.imcore_list(image, cat_file, outname, confidence_map=conf_file,rcore=appsize, noell=True,
verbose=verbose)
# do some quality checks
factor = 5
size = 11
stars = 1000
side = 3 #how many subdevisions of the chip along each axis. must be odd.
fwhm_a, fwhm_b, t = call_find_fwhm(image,cat_file,factor,size,stars,tag=image,side=side)
frame_ston = compute_frame_signal_to_noise(image)
pf.setval(outname,'FRAME_SN',1,value=round(frame_ston,2),comment='A measure of bulk structure in the image (S/N)')
positions = ['Top left.','Top middle.','Top right.','Middle left.','Center.','Middle right.','Bottom left.','Bottom middle.','Bottom right.']
for val in range(1,10):
pf.setval(outname,'PSF_a_'+str(val),1,value=fwhm_a['f_'+str(val)][0],comment='[pixels] psf long axis FWHM. '+positions[val-1])
pf.setval(outname,'PSF_b_'+str(val),1,value=fwhm_b['f_'+str(val)][0],comment='[pixels] psf short axis FWHM. '+positions[val-1])
pf.setval(outname,'PSF_t_'+str(val),1,value=t['f_'+str(val)][0],comment='[degrees] psf rotation angle. '+positions[val-1])
# Compute the HJD values
append_hjd_correction_column(outname)
return 'ok'
def write_pixelscale(fits_file, value, ext=0):
"""
Write pixel scale information to a FITS file header
The input pixel scale value is given in arcseconds but is stored
in degrees since the chosen header KEYS are the linear
transformation matrix parameters CDi_ja
Parameters
----------
fits_file: str
Path to a FITS image file
value: float
Pixel scale value in arcseconds
ext: int, optional
Extension number in the FITS file
"""
pixscl = value / 3600
comment = 'Linear transformation matrix'
pyfits.setval(fits_file, 'CD1_1', value=pixscl, ext=ext, comment=comment)
pyfits.setval(fits_file, 'CD1_2', value=0.0, ext=ext, comment=comment)
pyfits.setval(fits_file, 'CD2_1', value=0.0, ext=ext, comment=comment)
pyfits.setval(fits_file, 'CD2_2', value=pixscl, ext=ext, comment=comment)
def test_remove_d2im_distortion():
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits ')
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
updatewcs.updatewcs(acs_file)
fits.setval(acs_file, keyword="D2IMFILE", value="N/A")
updatewcs.updatewcs(acs_file)
w1 = HSTWCS(acs_file, ext=("SCI", 1))
w4 = HSTWCS(acs_file, ext=("SCI", 2))
assert w1.det2im1 is None
assert w4.det2im2 is None
def test_apply_d2im():
from stwcs.updatewcs import apply_corrections as appc
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
fname = get_filepath('j94f05bgq_flt.fits', current_dir)
d2imfile = get_filepath('new_wfc_d2i.fits')
try:
os.remove(fname)
except OSError:
pass
shutil.copyfile(acs_orig_file, fname)
fits.setval(fname, ext=0, keyword="D2IMFILE", value=d2imfile)
fits.setval(fname, ext=0, keyword="IDCTAB", value='N/A')
fits.setval(fname, ext=0, keyword="NPOLFILE", value='N/A')
# If D2IMEXT does not exist, the correction should be applied
assert appc.apply_d2im_correction(fname, d2imcorr=True)
updatewcs.updatewcs(fname)
# Test the case when D2IMFILE == D2IMEXT
assert not appc.apply_d2im_correction(fname, d2imcorr=True)
assert not appc.apply_d2im_correction(fname, d2imcorr=False)
fits.setval(fname, ext=0, keyword='D2IMFILE', value="N/A")
assert not appc.apply_d2im_correction(fname, d2imcorr=True)
# No D2IMFILE keyword in primary header
fits.delval(fname, ext=0, keyword='D2IMFILE')
assert not appc.apply_d2im_correction(fname, d2imcorr=True)
def test_update_d2im_distortion():
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits')
newd2im = get_filepath('new_wfc_d2i.fits', current_dir)
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
updatewcs.updatewcs(acs_file)
d2imerr1 = fits.getval(acs_file, ext=1, keyword='D2IMERR1')
d2imerr4 = fits.getval(acs_file, ext=4, keyword='D2IMERR1')
shutil.copyfile(d2imfile, newd2im)
with fits.open(newd2im, mode='update') as newf:
for ext in newf[1:]:
ext.data = ext.data * 100
fits.setval(acs_file, keyword="D2IMFILE", value=newd2im)
updatewcs.updatewcs(acs_file)
nd2imerr1 = fits.getval(acs_file, ext=1, keyword='D2IMERR1')
nd2imerr4 = fits.getval(acs_file, ext=4, keyword='D2IMERR1')
assert np.isclose(d2imerr1 * 100, nd2imerr1)
assert np.isclose(d2imerr4 * 100, nd2imerr4)
def make_basebias(input_list, refbias_name='basebias.fits'):
""" Make the basebias for an anneal month
1- Calbrate each bias in the list
2- Average together the biases
3- Replace pixels and colums with median values
4- Set header keywords
Parameters
----------
input_list : list
list of input bias files.
refbias_name : str
name of the output reference file.
"""
print('#-------------------------------#')
print('# Running basejoint #')
print('#-------------------------------#')
print('Output to %s' % refbias_name)
print('Processing individual files')
crj_list = [calibrate(item) for item in input_list]
crj_list = [item for item in crj_list if item != None]
mean_bias, totalweight = average_biases(crj_list)
print('Replacing hot columns and pixels by median-smoothed values')
residual_image, median_image = functions.make_residual(mean_bias)
replace_hot_cols(mean_bias, median_image, residual_image)
#-- then again, but only using the lower 20% of rows
replace_hot_cols(mean_bias, median_image, residual_image, yfrac=.2)
replace_hot_pix(mean_bias, median_image)
shutil.copy(mean_bias, refbias_name)
functions.update_header_from_input(refbias_name, input_list)
fits.setval(refbias_name, 'NCOMBINE', value=totalweight, ext=1)
fits.setval(refbias_name, 'TASKNAME', ext=0, value='BASEJOIN')
print('Cleaning up...')
functions.RemoveIfThere(mean_bias)
for item in crj_list:
functions.RemoveIfThere(item)
print('basejoint done for {}'.format(refbias_name))
def test_missing_idctab():
""" Tests that an IOError is raised if an idctab file is not found on disk."""
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, keyword="IDCTAB", value="my_missing_idctab.fits")
with pytest.raises(IOError):
updatewcs.updatewcs(acs_file)
def add_comments(fits_file, values):
"""
Add comments to the FITS header
Parameters
----------
fits_file: str
Path to a FITS image file
values: str or str list
Comment(s) to add
"""
if isinstance(values, str):
pyfits.setval(fits_file, 'COMMENT', value=values)
else:
for value in values:
pyfits.setval(fits_file, 'COMMENT', value=value)
def stisExt2PrimKw(stisfiles):
"""
Several kw which are usuall yin the primary header
are in the extension header for STIS. They are copied to
the primary header for convenience.
List if kw:
'DATE-OBS', 'EXPEND', 'EXPSTART', 'EXPTIME'
"""
kw_list = ['DATE-OBS', 'EXPEND', 'EXPSTART', 'EXPTIME']
for sfile in stisfiles:
d = {}
for k in kw_list:
d[k] = fits.getval(sfile, k, ext=1)
for item in d.items():
fits.setval(sfile, item[0], value=item[1], comment='Copied from extension header')
def bd_crreject(joinedfile):
""" Check if cosmic-ray rejection has been performed on input file
if cosmic-ray rejection has already been done on the input bias image,
skip all calstis-related calibration steps
"""
print(joinedfile)
fd = pyfits.open(joinedfile)
nimset = fd[0].header['nextend'] / 3
nrptexp = fd[0].header['nrptexp']
crcorr = fd[0].header['crcorr']
crdone = 0
if (crcorr == "COMPLETE") :
crdone = 1
print('OK, CR rejection already done')
os.rename(joinedfile, joinedfile.replace('_joined', '_crj') )
else:
print(('crcorr found = ' + crcorr))
if (nimset <= 1 and not crdone):
print("Sorry, your input image seems to have only 1 imset, but it isn't cr-rejected.")
print("This task can only handle 'raw' or 'flt images with the NEXTEND keyword equal to 3*N (N > 1).")
print("Bye now... better luck next time!")
raise ValueError( 'Something bad happened' )
if not crdone:
print('FYI: CR rejection not already done')
print(('Keyword NRPTEXP = ' + str(nrptexp) + ' while nr. of imsets = ' + str(nimset)))
if (nrptexp != nimset):
pyfits.setval( joinedfile,'NRPTEXP',value=nimset)
pyfits.setval( joinedfile,'CRSPLIT',value=1)
print(('>>>> Updated keyword NRPTEXP to '+str(nimset) ))
print(' (and set keyword CRSPLIT to 1)' )
print((' in ' + joinedfile ))
return crdone
def test_missing_d2imfile():
""" Tests that an IOError is raised if a D2IMFILE file is not found on disk."""
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, keyword="D2IMFILE", value="missing_d2i.fits")
with pytest.raises(IOError):
updatewcs.updatewcs(acs_file)
fobj = fits.open(acs_file)
with pytest.raises(IOError):
updatewcs.updatewcs(fobj)
def setup_class(self):
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
simple_orig_file = get_filepath('simple.fits')
current_dir = os.path.abspath(os.path.curdir)
simple_file = get_filepath('simple.fits', current_dir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
comp_file = get_filepath('comp.fits', current_dir)
self.headerlet_name = get_filepath('acs_hlet.fits', current_dir)
try:
os.remove(acs_file)
os.remove('comp.fits')
os.remove(simple_file)
except OSError:
pass
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits ')
shutil.copyfile(acs_orig_file, acs_file)
shutil.copyfile(simple_orig_file, simple_file)
fits.setval(acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
updatewcs.updatewcs(acs_file)
shutil.copyfile(acs_file, comp_file)
self.comp_file = comp_file
def setup_class(self):
self.obsname = 'j94f05bgq_flt.fits'
refname = self.obsname.replace('_flt', '_flt_rdb')
acs_orig_file = get_filepath(self.obsname)
current_dir = os.path.abspath(os.path.curdir)
self.ref_file = get_filepath(refname, current_dir)
self.acs_file = get_filepath(self.obsname, current_dir)
try:
os.remove(self.acs_file)
os.remove(self.ref_file)
except OSError:
pass
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits ')
shutil.copyfile(acs_orig_file, self.acs_file)
fits.setval(self.acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(self.acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(self.acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
shutil.copyfile(self.acs_file, self.ref_file)
updatewcs.updatewcs(self.acs_file, use_db=False)
def setup_class(self):
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
self.ref_file = get_filepath('j94f05bgq_flt_r.fits', current_dir)
self.acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
try:
os.remove(self.acs_file)
os.remove(self.ref_file)
except OSError:
pass
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits ')
shutil.copyfile(acs_orig_file, self.acs_file)
fits.setval(self.acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(self.acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(self.acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
updatewcs.updatewcs(self.acs_file)
#self.ref_file = ref_file
shutil.copyfile(self.acs_file, self.ref_file)
self.w1 = HSTWCS(self.acs_file, ext=1)
self.w4 = HSTWCS(self.acs_file, ext=4)
self.w = wcs.WCS()
def test_found_idctab():
""" Tests the return value of apply_corrections.foundIDCTAB()."""
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits ')
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
fits.setval(acs_file, keyword="IDCTAB", value="N/A")
corrections = apply_corrections.setCorrections(acs_file)
assert('MakeWCS' not in corrections)
assert('TDDCor' not in corrections)
assert('CompSIP' not in corrections)
fobj = fits.open(acs_file)
corrections = apply_corrections.setCorrections(fobj)
assert('MakeWCS' not in corrections)
assert('TDDCor' not in corrections)
assert('CompSIP' not in corrections)
def test_restore_wcs_from_to(self):
# test restore from ... to ...
pyfits.setval(self.acs_file, ext=('SCI', 1), keyword='CRVAL1', value=1)
pyfits.setval(self.acs_file, ext=('SCI', 2), keyword='CRVAL1', value=1)
f = pyfits.open(self.acs_file, mode='update')
altwcs.restore_from_to(f, fromext='SCI', toext=['SCI', 'ERR', 'DQ'],
wcskey='T')
f.close()
w1o = wcsutil.HSTWCS(self.acs_file, ext=('SCI', 1), wcskey='T')
w1 = wcsutil.HSTWCS(self.acs_file, ext=('SCI', 1))
compare_wcs(w1, w1o)
w2 = wcsutil.HSTWCS(self.acs_file, ext=('ERR', 1))
compare_wcs(w2, w1o, exclude_keywords=['ctype'])
w3 = wcsutil.HSTWCS(self.acs_file, ext=('DQ', 1))
compare_wcs(w3, w1o, exclude_keywords=['ctype'])
w4o = wcsutil.HSTWCS(self.acs_file, ext=4, wcskey='T')
w4 = wcsutil.HSTWCS(self.acs_file, ext=('SCI', 2))
compare_wcs(w4, w4o)
w5 = wcsutil.HSTWCS(self.acs_file, ext=('ERR', 2))
compare_wcs(w5, w4o, exclude_keywords=['ctype'])
w6 = wcsutil.HSTWCS(self.acs_file, ext=('DQ', 2))
compare_wcs(w3, w1o, exclude_keywords=['ctype'])
def setval(filepath, key, value):
"""Set metadata `key` in file `filepath` to `value`."""
ftype = config.filetype(filepath)
if ftype == "fits":
if key.upper().startswith(("META.","META_")):
key = key.replace("META_", "META.")
return _dm_setval(filepath, key, value)
else:
return pyfits.setval(filepath, key, value=value)
elif ftype == "asdf":
return _dm_setval(filepath, key, value)
else:
raise NotImplementedError("setval not supported for type " + repr(ftype))
def m_frame_shift(imagelist, index):
try:
image1 = imagelist[index - 1]
image2 = imagelist[index]
RA_shift, DEC_shift, tot_shift, RA, DEC = frame_shift(image1, image2)
pf.setval(image2 + '.phot', 'RA_MOVE', 1,
value=RA_shift,
comment='RA shift from previous image [arcseconds]')
pf.setval(image2 + '.phot', 'DEC_MOVE', 1,
value=DEC_shift,
comment='Dec shift from previous image [arcseconds]')
pf.setval(image2 + '.phot', 'SKY_MOVE', 1,
value=tot_shift,
comment='Total movement on sky [arcseconds]')
pf.setval(image2 + '.phot', 'WCSF_RA', 1,
value=RA_shift,
comment='RA center pix')
pf.setval(image2 + '.phot', 'WCSF_DEC', 1,
value=DEC_shift,
comment='Dec center pix')
except Exception as err:
print "Exception handled in m_frame_shift: {}".format(str(err))
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
try:
pav3 = fits.getval(f, 'PA_V3')
except KeyError:
rootname = fits.getval(f, 'ROOTNAME')
sptfile = rootname+'_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
fits.setval(f, 'PA_V3', value=pav3)
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t',f)
return removed_files
def insert_dateobs(output, dateobs):
""" Put the DATE-OBS keyword back into the AstroDrizzle outputs.
AstroDrizzle removes this keyword, and it is needed for the database and
ephemeris later.
Parameters:
output: string
The renamed AstroDrizzle output the DATE-OBS keyword needs to be
put back into.
dateobs: string
The value of the dateobs keyword from the original _flt or _c0m
file.
Returns: nothing
Outputs:
The AstroDrizzle output file, edited to include DATE-OBS.
"""
# Since we are changing only a single keyword, using the convenience
# function is permissible.
fits.setval(filename = output,
keyword = 'date-obs',
value = dateobs)
def create_superdarks( file_list ):
print 'Creating superdarks'
all_mjd = [pyfits.getval(item, 'EXPSTART', ext=1 ) for item in file_list ]
file_list = np.array( file_list )
all_mjd = np.array( all_mjd )
step = 30
first_mjd = int(all_mjd.min())
end_mjd = int(all_mjd.max())
print 'Running from ', first_mjd, end_mjd
for start in range( first_mjd, end_mjd, step)[:-1]:
print start, '-->', start+step
file_index = np.where( (all_mjd > start) &
(all_mjd < start+step) )[0]
if not len( file_index ):
print 'WARNING, no files found for interval'
dark_reffile = SuperDark( file_list[ file_index ] )
output = '{}_{}_drk.fits'.format( start, start+step )
dark_reffile.write( outname=output )
pyfits.setval( output, 'DATASTRT', ext=0, value=start )
pyfits.setval( output, 'DATAEND', ext=0, value=start+step )
def test_add_radesys():
""" test that RADESYS was successfully added to headers."""
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits ')
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
updatewcs.updatewcs(acs_file)
for ext in [('SCI', 1), ('SCI', 2)]:
hdr = fits.getheader(acs_file, ext)
assert hdr['RADESYS'] == 'FK5'
def conv(filter):
'''
Takes data (in the form of a FITS file) and convolves with a PSF.
Keywords
filter: the filter that the data considered is imaged in
Options - SPIRE:
psw (250 micron)
pmw (350 micron)
plw (500 micron)
Options - PACS:
70 (70 micron)
100 (100 micron)
160 (160 micron)
psf: the filename of the point spread function of the filter considered
Example:
SPIRE 250 micron PSF has filename 'theoretical_spire_beam_model_psw_V0_2'
This file *must* be present in folder 'psf'
'''
################################# Read data ####################################
# Determine the current working directory
direc = os.getcwd()
# Read in the FITS file containing the simulation results
sim_result = fits.open(str(direc)+str('/')+str(filter)+str('.fits'))
# Split into image data and header data
data = sim_result[0].data
hdu = sim_result[0].header
# Pull out the total image width
image_width = hdu['IMGWIDTH'] # In cm
# Pull out the pixel width
data_theta = hdu['PIXSIZE'] # Pixel size in "/pixel
# Determine the size of one pixel
data_size = (250*data_theta)/206256 # Extent of one pixel
############################# Read in FITS data ################################
# Read in the PSF
# Given that the PSF has different file names depending on the filter under consideration, use this if statement to determine
# which is the correct one
if filter == 'psw':
psf_raw = fits.open('/Users/tomasjames/Documents/University/Project/ZiggyStarDust/Code/datafiles/psf/ken/psf_0250.fits')
elif filter == 'pmw':
psf_raw = fits.open('/Users/tomasjames/Documents/University/Project/ZiggyStarDust/Code/datafiles/psf/theoretical_spire_beam_model_pmw_V0_2.fits')
elif filter == 'plw':
psf_raw = fits.open('/Users/tomasjames/Documents/University/Project/ZiggyStarDust/Code/datafiles/psf/theoretical_spire_beam_model_plw_V0_2.fits')
# Extract the data
psf = psf_raw[0].data
psf_header = psf_raw[0].header
# Assign cards to variables
psf_theta_deg = psf_header['CDELT2'] # Pixel width in deg
psf_theta = psf_theta_deg*3600.
#image_width = hdu['IMGWIDTH'] # Image width in cm
# Plot the data for comparison purposes
subplot2grid((10,8), (0,0), colspan=8,rowspan=8)
title(r'Raw SPIRE 250$\mu m$ PSF')
matshow(psf,origin='lower')
colorbar()
xlabel('Pixel')
ylabel('Pixel')
'''
#tight_layout
#grid(color='g',linestyle='-',linewidth=1)
subplot2grid((10,8), (8,0), colspan=8,rowspan=2)
plot(np.linspace(0,len(psf),len(psf)),psf[len(psf)/2])
xlim(min(np.linspace(0,len(psf),len(psf))), max(np.linspace(0,len(psf),len(psf))))
ylabel('Pixel Value')
xlabel('Pixel')
#tight_layout
'''
savefig('psf.pdf',dpi=300)
close()
matshow(data,origin='lower')
title(str('RADMC-3D Intensity Data for SPIRE ')+str(hdu['EFF'])+str('$ \mu m$'))
#colorbar(label=r'$I_{\nu}$ [erg/s/cm/cm/Hz/ster]')
colorbar(label=r'$I$ [MJy/ster]')
savefig('data.pdf',dpi=300)
close()
############################### Rebin the PSF #################################
# Take the data pixel size and determine how much larger/smaller it is than the PSF
size_factor = data_theta/psf_theta
# Convolve requires that the array be odd in dimension
# Check to see if the size_factor is an odd number
# This ensures the PSF is centered in the frame
if (np.round(size_factor) % 2 == 0):
print 'The dimensions of the PSF are even; altering to make the dimensions odd...\n'
else:
print 'The dimensions of the PSF are odd; proceeding as normal...\n'
# Determine whether the pixel widths of the PSF and data differ
if size_factor == 1:
print 'The pixel sizes match!\n'
psf_rebin = psf
elif size_factor < 1:
print 'The pixel sizes do not match, and the PSF has larger pixel size. Necessary to increase the size of the RADMC-3D image.\n'
# This will resize the original data to be 1/2 the original size using the value closest to the original (method='neighbour')
data = congrid(data, (len(data[0])*(1./2),len(data[:,0])*(1./2)),centre=True)
imshow(data,origin='lower')
title(r'Rebinned Data')
#colorbar(label=r'$I_{\nu}$ [erg/s/cm/cm/Hz/ster]')
colorbar(label=r'$I$ [MJy/ster]')
savefig('data_rebin.png')
close()
# Determine the new angular size of each pixel
data_theta = data_theta/(1./2)
# Adjust the size factor accordingly
size_factor = data_theta/psf_theta
# Use the above rebin function to change the dimensions of the PSF to
psf_rebin_unnorm = congrid(psf, ((psf_header['NAXIS1']/size_factor)+1,(psf_header['NAXIS2']/size_factor)+1),centre=True)
# Renormalise the PSF back to unity (or close to it)
psf_rebin = psf_rebin_unnorm*(np.sum(psf)/np.sum(psf_rebin_unnorm))
elif size_factor > 1:
# Use the above rebin function to change the dimensions of the PSF to
psf_rebin_unnorm = congrid(psf, ((psf_header['NAXIS1']/size_factor),(psf_header['NAXIS2']/size_factor)),centre=True)
# Renormalise the PSF back to unity (or close to it)
psf_rebin = psf_rebin_unnorm*(np.sum(psf)/np.sum(psf_rebin_unnorm))
# Determine the new pixel angular size
new_psf_theta = psf_theta*size_factor
# Save rebinned PSF to fits file
fname = str('psf_rebin.fits')
head = fits.PrimaryHDU(psf_rebin)
head.writeto(fname)
# Assign info to header
fits.setval(fname, 'DISTANCE', value=hdu['DISTANCE'])
fits.setval(fname, 'PIXSIZE', value=new_psf_theta)
fits.setval(fname, 'EFF', value=hdu['EFF'])
# Plot the resulting data to assess accuracy of rebin
subplot2grid((10,8), (0,0), colspan=6,rowspan=6)
title(r'Rebinned SPIRE 250$\mu m$ PSF')
matshow(psf_rebin,origin='lower')
colorbar()
xlabel('Pixel')
ylabel('Pixel')
'''
# major ticks every 20, minor ticks every 5
major_ticks = np.arange(0, len(psf_rebin[0]), 10)
minor_ticks = np.arange(0, len(psf_rebin[0]), 1)
xticks(major_ticks)
xticks(minor_ticks)
yticks(major_ticks)
yticks(minor_ticks)
grid(which='minor',linestyle='-',alpha=0.2)
grid(which='major',linestyle='-',alpha=0.2)
subplot2grid((10,8), (8,0), colspan=8,rowspan=2)
plot(np.linspace(0,len(psf_rebin[len(psf_rebin)/2]),len(psf_rebin[len(psf_rebin)/2])),psf_rebin[len(psf_rebin)/2])
xlim(min(np.linspace(0,len(psf_rebin),len(psf_rebin))), max(np.linspace(0,len(psf_rebin),len(psf_rebin))))
ylabel('Pixel Value')
xlabel('Pixel')
#tight_layout
'''
savefig('psf_rebin.pdf',dpi=300)
close()
# Determine dimensions of the new image
psf_rebin_x = len(psf_rebin)
psf_rebin_y = len(psf_rebin[0])
# Print statements for debug
print 'The raw PSF "/pixel:', psf_theta, '\n'
print 'The raw data "/pixel:', data_theta, '\n'
print 'The rebinned PSF "/pixel:', new_psf_theta, '\n'
print 'The PSF image has now been rebinned to have dimensions', psf_rebin_x, 'x', psf_rebin_y, '. The new image has also been saved to the working directory.\n'
############################### Convolve the data ##############################
print 'Currently convolving...'
# Use the convolution to convolve the rebinned data with the PSF
#conv = convolve(data, psf_rebin, boundary='extend')
conv = convolve_fft(data, psf_rebin)
print 'Convolution complete!\nSaving the image...\n'
# Plot the resulting convolution
#imshow(conv,origin='lower',vmin=0,vmax=1)
matshow(conv,origin='lower')
#colorbar(label=r'$I_{\nu}$ [erg/s/cm/cm/Hz/ster]')
colorbar(label=r'$I$ [MJy/ster]')
title(str('Convolved Data for SPIRE ')+str(hdu['EFF'])+str(r'$ \mu m$'))
savefig('convolved.pdf',dpi=300)
close()
# Save to a FITS file
# Check to see whether file has already been written
# If so, delete the file (Python cannot overwrite a binary file)
if os.path.isfile(str(filter)+str('_convolved.fits')) == True:
os.remove(str(filter)+str('_convolved.fits'))
# Do the write
fname = str(filter)+str('_convolved.fits')
head = fits.PrimaryHDU(conv)
head.writeto(fname)
# Write some header information
# Write relevant information to the FITS header
fits.setval(fname, 'DISTANCE', value=hdu['DISTANCE'])
fits.setval(fname, 'IMGWIDTH', value=hdu['IMGWIDTH'])
fits.setval(fname, 'PIXWIDTH', value=hdu['PIXWIDTH'])
fits.setval(fname, 'PIXSIZE', value=data_theta)
fits.setval(fname, 'EFF', value=hdu['EFF'])
print 'The convolution has been performed and the convolved image saved to the working directory.\n'
############################### Check image power ##############################
# Begin with default, non altered image
raw_power = np.sum(np.sum(data))
# Repeat for convolved image
convolved_power = np.sum(np.sum(conv))
# Print a statement to the terminal to show overall power
print 'The initial power in the image is ', raw_power
print 'The power in the convolved image is ', convolved_power
def regress(folder):
""" Run *drk and *bia files in folder through CalSTIS to check
for errors in processing
"""
start_dir = os.getcwd()
print('#------------------#')
print('Running regression for')
print(folder)
print('#------------------#')
monitor_dir = '/grp/hst/stis/darks_biases'
test_suite = os.path.join(monitor_dir, 'test_suite')
test_dark = os.path.join(monitor_dir, 'test_dark')
print((glob.glob(os.path.join(folder, '*bia.fits'))))
reference_files = glob.glob(os.path.join(folder, '*bia.fits')) + \
glob.glob(os.path.join(folder, '*drk.fits'))
print(reference_files)
assert len(reference_files) >= 1, 'No reference files in folder'
print('Copying files and removing old files')
for testing_dir in [test_suite, test_dark]:
for oldfile in glob.glob(os.path.join(testing_dir, '*_drk.fits')) + \
glob.glob(os.path.join(testing_dir, '*_bia.fits')):
print(('removing', oldfile))
os.remove(os.path.join(testing_dir, oldfile))
for newfile in reference_files:
print(('moving', newfile, '-->', testing_dir))
shutil.copy(newfile, testing_dir)
#######################################
# Run checks in the test_suite folder #
#######################################
os.chdir(test_suite)
print((os.getcwd()))
bias_biwk_refs = glob.glob('*bias*_bi*.fits')
bias_biwk_refs.sort()
biasrefs = glob.glob('*bias*_wk*.fits')
biasrefs.sort()
darkrefs = glob.glob('*dark*.fits')
darkrefs.sort()
raws = glob.glob('*raw.fits')
print('Setting IMPHTTAB in datasets')
for dark, bias in zip(darkrefs, biasrefs):
remove_products()
print('#-------------------------------------------#')
print(('Running CalSTIS with %s %s ' % (dark, bias)))
print('#-------------------------------------------#')
for rawfile in raws:
if os.path.exists( rawfile.replace('raw.fits', 'wav.fits') ):
wavefile = rawfile.replace('raw.fits', 'wav.fits')
else:
wavefile = ''
pyfits.setval(rawfile, 'DARKFILE', value=dark, ext=0)
pyfits.setval(rawfile, 'BIASFILE', value=bias, ext=0)
pyfits.setval(rawfile, 'IMPHTTAB', value='oref$x9r1607mo_imp.fits', ext=0)
if wavefile:
pyfits.setval(wavefile, 'DARKFILE', value=dark, ext=0)
pyfits.setval(wavefile, 'BIASFILE', value=bias, ext=0)
pyfits.setval(wavefile, 'IMPHTTAB', value='oref$x9r1607mo_imp.fits', ext=0)
status = calstis(rawfile, wavecal=wavefile)
if status: sys.exit('Calstis Error detected for %s' % (dark[5:9]))
######################################
# Run checks in the test_dark folder #
######################################
os.chdir(test_dark)
print((os.getcwd()))
raws = glob.glob('*raw.fits')
print('Setting IMPHTTAB in datasets')
for item in raws:
pyfits.setval(item, 'IMPHTTAB', value='oref$x9r1607mo_imp.fits', ext=0)
for bias in bias_biwk_refs:
remove_products()
print('#------------------------------------------#')
print(('Running CalSTIS with %s' % (bias)))
print('#------------------------------------------#')
for rawfile in raws:
pyfits.setval(rawfile, 'BIASFILE', value=bias, ext=0)
pyfits.setval(rawfile, 'DARKFILE', value=darkrefs[0], ext=0)
status = calstis(input=rawfile)
if status: sys.exit('Calstis Error detected for %s' % (dark[5:9]))
os.chdir(start_dir)