def get_from_header(image_name, *args):
""" From the header of an image, get the values corresponding to the
keywords passed in args"""
if len(args) == 1:
return fits.getval(image_name, args[0])
else:
return (fits.getval(image_name, x) for x in args)
def retrieve_SFHs(filelist, i, massnorm='mformed', nsubpersfh=None, nsfhperfile=None):
'''
fetch a time array & SFH from a series of FITS archives,
each with `nsfhperfile` SFHs & `nsubpersfh` Z/tau/mu realizations for
each SFH
'''
if nsfhperfile is None:
nsfhperfile = fits.getval(filelist[0], ext=0, keyword='NSFHPER')
if nsubpersfh is None:
nsubpersfh = fits.getval(filelist[0], ext=0, keyword='NSUBPER')
fnum, fi, fii, fiii = find_sfh_ixs(i, nsfhperfile, nsubpersfh)
'''
print('trainer {0}: spectral:file-spec {1}-{2}; SFH:file-rec-subsample {1}-{3}-{4}'.format(
i, fnum, fi, fii, fiii))
'''
fname = filelist[fnum]
allts = fits.getdata(fname, extname='allts')
allsfhs = np.repeat(fits.getdata(fname, extname='allsfhs'),
nsubpersfh, axis=0)
# normalize mass either by total mass formed or current stellar mass
mtot = fits.getdata(fname, massnorm)[:, None]
return allts, allsfhs / mtot, fi
def RunAstrometryCalibration(InputImage,Outputfile=None):
""" Does Astrometry calibration of input fits image InputImage.
Returns True if calibration succeded."""
# Following lines are TIRSPEC specific for formating the time from fits header
obs_timehdr = fits.getval(InputImage,'OBSTIME')
obs_datehdr = fits.getval(InputImage,'OBSDATE')
time_str = '{0} {1}'.format('-'.join(obs_datehdr.split('.')),obs_timehdr)
time = Time(time_str,format='iso',scale='utc')
hct_hanle = EarthLocation(lat=32.77944*u.deg, lon=78.9641*u.deg, height=4500*u.m)
zenith = AltAz(location=hct_hanle, obstime=time, az=0*u.deg, alt=90*u.deg)
ZenithRaDec = zenith.transform_to(ICRS)
# Finally call astrometry.net software to calibrate fits image
Z_ra = '{0.ra}'.format(ZenithRaDec).split()[0]
Z_dec = '{0.dec}'.format(ZenithRaDec).split()[0]
ret = subprocess.call(['solve-field','--no-plots', '--ra',Z_ra, '--dec',Z_dec, '--radius','85',
'--scale-units','arcsecperpix', '--scale-low','0.28', '--scale-high','0.32',
InputImage])
print(ret)
if ret == 0:
if Outputfile is not None:
print('Copying {0} to {1}'.format(os.path.splitext(InputImage)[0]+'.new',Outputfile))
shutil.copy(os.path.splitext(InputImage)[0]+'.new',Outputfile)
return True
else:
return False
def az_zendist_of_data(image_filename, ra_dec, site=''):
'''
Returns an array with azimuth/zenith distance locations corresponding to
ra,dec information.
:param image_filename: Path of FITS file of image
:param ra_dec: Array of ra, dec information [decimal degrees]
:return: Array of azimuth, zenith distance in [radians]
'''
obstime = fits.getval(image_filename, 'DATE-OBS').replace('T', ' ')
if not site:
site = fits.getval(image_filename, 'SITE')
observer = astro_utils.observer(site, obstime)
az_zendist = np.zeros(shape=ra_dec.shape)
for i in range(len(az_zendist)):
az_zen_am = np.deg2rad(astro_utils.world2az_zendist(ra_dec[i,0], ra_dec[i,1], observer))
az_zendist[i,0] = az_zen_am[0]
az_zendist[i,1] = az_zen_am[1]
return az_zendist
def test_update_d2im_distortion():
acs_orig_file = get_filepath('j94f05bgq_flt.fits')
current_dir = os.path.abspath(os.path.curdir)
acs_file = get_filepath('j94f05bgq_flt.fits', current_dir)
idctab = get_filepath('postsm4_idc.fits')
npol_file = get_filepath('qbu16424j_npl.fits')
d2imfile = get_filepath('new_wfc_d2i.fits')
newd2im = get_filepath('new_wfc_d2i.fits', current_dir)
try:
os.remove(acs_file)
except OSError:
pass
shutil.copyfile(acs_orig_file, acs_file)
fits.setval(acs_file, ext=0, keyword="IDCTAB", value=idctab)
fits.setval(acs_file, ext=0, keyword="NPOLFILE", value=npol_file)
fits.setval(acs_file, ext=0, keyword="D2IMFILE", value=d2imfile)
updatewcs.updatewcs(acs_file)
d2imerr1 = fits.getval(acs_file, ext=1, keyword='D2IMERR1')
d2imerr4 = fits.getval(acs_file, ext=4, keyword='D2IMERR1')
shutil.copyfile(d2imfile, newd2im)
with fits.open(newd2im, mode='update') as newf:
for ext in newf[1:]:
ext.data = ext.data * 100
fits.setval(acs_file, keyword="D2IMFILE", value=newd2im)
updatewcs.updatewcs(acs_file)
nd2imerr1 = fits.getval(acs_file, ext=1, keyword='D2IMERR1')
nd2imerr4 = fits.getval(acs_file, ext=4, keyword='D2IMERR1')
assert np.isclose(d2imerr1 * 100, nd2imerr1)
assert np.isclose(d2imerr4 * 100, nd2imerr4)
def inputs(cls, gainmap):
""" return the pulse height image name from the input gainmap
"""
gainmap_path, gainmap_name = os.path.split(gainmap)
segment = fits.getval(gainmap, 'SEGMENT')
dethv = int(fits.getval(gainmap, 'DETHV'))
both_inputs = [gainmap]
if segment == 'FUVA':
other_root = gainmap.replace(FUVA_string, FUVB_string).replace('_{}_'.format(dethv), '_???_')
elif segment == 'FUVB':
other_root = gainmap.replace(FUVB_string, FUVA_string).replace('_{}_'.format(dethv), '_???_')
other_gainmap = glob.glob(other_root)
if len(other_gainmap) != 1:
raise IOError("too many gainmaps found {}".format(other_gainmap))
else:
other_gainmap = other_gainmap[0]
both_inputs.append(other_gainmap)
both_inputs.sort()
return tuple(both_inputs)
def populate_calibration_table_with_bpms(directory, db_address=_DEFAULT_DB):
with get_session(db_address=db_address) as db_session:
bpm_filenames = glob(os.path.join(directory, 'bpm*.fits*'))
for bpm_filename in bpm_filenames:
if bpm_filename[-3:] == '.fz':
extension_number = 1
else:
extension_number = 0
header = fits.getheader(bpm_filename, extension_number)
ccdsum = fits.getval(bpm_filename, 'CCDSUM', extension_number)
dateobs = date_utils.parse_date_obs(fits.getval(bpm_filename, 'DATE-OBS', extension_number))
try:
instrument = get_instrument(header, db_address=db_address)
except ValueError:
logger.error('Instrument is missing from database', extra_tags={'site': header['SITEID'],
'camera': header['INSTRUME']})
continue
bpm_attributes = {'type': 'BPM',
'filename': os.path.basename(bpm_filename),
'filepath': os.path.abspath(directory),
'dateobs': dateobs,
'datecreated': dateobs,
'instrument_id': instrument.id,
'is_master': True,
'is_bad': False,
'attributes': {'ccdsum': ccdsum}}
add_or_update_record(db_session, CalibrationImage, {'filename': bpm_attributes['filename']}, bpm_attributes)
db_session.commit()
def grab_anneal_mjds():
""" Loop over anneal directories and return a list of tuples containing
useful mjd info to be populated into anneal database
returns
-------
anneal_info
list, full of tupes containing information for database
"""
print('Getting anneal information')
anneal_info = []
for directory in get_directories():
anneal_obs = glob.glob( os.path.join(directory, '?????????_crj.fits') )
anneal_obs.sort()
if len(anneal_obs) != 2:
print('Error in ', directory)
continue
proposid = pyfits.getval( anneal_obs[0], 'PROPOSID', ext=0 )
visit_number = int(pyfits.getval(anneal_obs[1], 'OBSET_ID')) - 1
anneal_start = pyfits.getval(anneal_obs[0], 'TEXPSTRT', ext=0)
anneal_end = pyfits.getval(anneal_obs[1], 'TEXPSTRT', ext=0)
anneal_info.append( (proposid, visit_number, anneal_start, anneal_end) )
return anneal_info
def organize_files(folder, workdir=conf.work_dir, identifier=None):
'''
Organize the files inside a folder.
'''
if identifier is None:
outdir = path.join(path.expanduser(workdir),'raw')
else:
outdir = path.join(path.expanduser(workdir),path.join(identifier,'raw'))
try:
makedirs(outdir)
except:
pass
for i in glob.glob(path.join(folder,'*.fits')):
try:
f = fits.getval(i, 'OBSTYPE')
except KeyError:
try:
bitpix = fits.getval(i, 'BITPIX')
f = fits.open(i)
k = f[0].header['OBJECT']
t = identify_file(k)
f[0].header[conf.obstype_keyword] = conf.obstypes[t]
f[0].header['BITPIX'] = bitpix
f[0].data = f[0].data.astype(fits_dtype(bitpix))
f.writeto(path.join(outdir, path.basename(i)))
f.close()
except:
print('File %s failed' % i)
def make_master_bias(dc):
print "Making master BIAS"
for i in dc:
TRIM, TRIM1, VR, PS, PS1, OS, OS1 = CCD_sections((i[0], i[1]))
filelist = []
for f in glob.glob(RAW+"*BIAS*fits"):
ccd_conf = []
for KW in ['BINX', 'BINY']:
ccd_conf.append(fits.getval(f, KW))
for KW in ['NAXIS1', 'NAXIS2']:
ccd_conf.append(fits.getval(f, KW, ext=1))
if tuple(ccd_conf)==i:
filelist.append(f)
lfl = len(filelist)
if lfl > 0:
BIN=CD+'/'+str(i[0])+'x'+str(i[1])+'/'
WD=BIN+str(i[-2])+'x'+str(i[-1])+'/' # Bottom level dir with calibrated and master frames
B=check_exist(WD, 'MB.fits', i)
if B=='n':
pass
else:
hdul = fits.HDUList()
hdul.append(fits.ImageHDU())
x = np.array(range(0,i[-1]))
for EXT in (extensions):
print "##################################################"
print "Stacking "+`lfl`+' '+`i[-2]`+'x'+`i[-1]`+' channel '+`EXT`+' bias frames!'
if EXT==1:
OSC=OS1
PSC=PS1
else:
OSC=OS
PSC=PS
stack_arr = np.zeros( (len(filelist), i[-1], i[-2]) )
for n, fn in enumerate(filelist):
print "Files left:",`lfl-n`+'/'+`lfl`
im = fits.getdata(fn, ext=EXT)
row, col = [ range(0,d) for d in np.shape(im)]
mscrow = np.median(np.hstack((im[:,PSC[0]:PSC[1]], im[:,OSC[0]:OSC[1]])), axis=1)# median scan rows
crow = np.polyfit(row,mscrow, 11)
frow = np.poly1d(crow) # fit rows
stack_arr[n] = (im.T - frow(row)).T
#~ stack_arr[n] = B
median_stack = np.median(stack_arr, axis=0)
hdul.append(fits.ImageHDU(median_stack))
hdul[0].header.set("CALIBR", "T")
hdul[0].header.set("INSTRUME", "MAIA")
hdul[0].header.set("CALMODE", "MASTER BIAS")
hdul[0].header.set("BINX", i[0])
hdul[0].header.set("BINY", i[1])
hdul.writeto(WD+"MB.fits", clobber=True)
print "############################################################"
#~ for EXT in (extensions):
#~ print "Median BIAS EXT("+repr(EXT)+"):"+repr(np.round(np.median(hdul[EXT].data),2))
#~ print "STD(EXT"+repr(EXT)+"):"+repr( np.round(np.std(hdul[EXT].data),2) )
print "Completed master bias!"
def create_wavecorr_refs(wzpc_files, outname=None, author=None, description=None, useafter="2015-11-01"):
"""
Create WAVECORR reference files (Nirspec wavelength zero-point correction).
Parameters
----------
wzpc_files : list or str
A list of *slit-*wzprf.fits file names to create the Fixed Slits reference file.
A file name to create the MSA reference file.
author : str
Author. If None it will be read from the first file.
description : str
Description of the file. If None will be read from the header.
useafter : str
Useafter date.
"""
model = WaveCorrModel()
aps = []
if isinstance(wzpc_files, list):
# Create a reference file for the Fixed Slits mode.
model.meta.exposure.type = "NRS_FIXEDSLIT"
model.meta.exposure.p_exptype = "NRS_FIXEDSLIT|NRS_BRIGHTOBJ|"
for f in wzpc_files:
aps.append(_wzpc2asdf(f, author=author, description=description, useafter=useafter))
if description is None:
description = "Wavelength zero-point reference file for Nirspec fixed slits, computed using a simple toy model."
elif isinstance(wzpc_files, str):
model.meta.exposure.type = "NRS_MSASPEC"
wzpc_files = [wzpc_files]
aps.append(_wzpc2asdf(wzpc_files[0], author=author, description=description, useafter=useafter))
else:
raise ValueError("Invalid input - expected a string or a list of strings.")
f0 = wzpc_files[0]
model.apertures = aps
if author is None:
author = fits.getval(f0, 'AUTHOR')
if description is None:
description = fits.getval(f0, 'DESCRIP')
pedigree = fits.getval(f0, 'PEDIGREE')
model.meta.author = author
model.meta.pedigree = pedigree
model.meta.description = description
model.meta.useafter = useafter
model.meta.date = fits.getval(f0, 'date')
model.meta.origin = fits.getval(f0, 'author')
model.meta.instrument.p_detector = "NRS1|NRS2|"
entry = HistoryEntry({'description': "NIRSPEC wavelength zero-point correction.", 'time': datetime.datetime.utcnow()})
software = Software({'name': 'jwstreftools', 'author': 'N.Dencheva',
'homepage': 'https://github.com/spacetelescope/jwreftools', 'version': "0.7.1"})
entry['software'] = software
model.history.append(entry)
if outname is None:
outname = "nirspec_wavecorr.asdf"
model.to_asdf(outname)
def fits_to_png(infile, outfile, draw, dpi=100):
"""Plot FITS image in PNG format.
For the default ``dpi=100`` a 1:1 copy of the pixels in the FITS image
and the PNG image is achieved, i.e. they have exactly the same size.
Parameters
----------
infile : str
Input FITS file name
outfile : str
Output PNG file name
draw : callable
Callback function ``draw(figure)``
where ``figure`` is an `~aplpy.FITSFigure`.
dpi : int
Resolution
Examples
--------
>>> def draw(figure):
... x, y, width, height = 42, 0, 3, 2
... figure.recenter(x, y, width, height)
... figure.show_grayscale()
>>> from gammapy.image import fits_to_png
>>> fits_to_png('image.fits', 'image.png', draw)
"""
import matplotlib
matplotlib.use("Agg") # Prevents image popup
import matplotlib.pyplot as plt
from astropy.io import fits
from aplpy import FITSFigure
# Peak ahead just to get the figure size
NAXIS1 = float(fits.getval(infile, "NAXIS1"))
NAXIS2 = float(fits.getval(infile, "NAXIS2"))
# Note: For dpi=100 I get exactly the same FITS and PNG image size in pix.
figsize = np.array((NAXIS1, NAXIS2))
figure = plt.figure(figsize=figsize / dpi)
# Also try this:
# matplotlib.rcParams['figure.figsize'] = NAXIS1, NAXIS2
# figsize(x,y)
subplot = [0, 0, 1, 1]
figure = FITSFigure(infile, figure=figure, subplot=subplot)
draw(figure)
figure.axis_labels.hide()
figure.tick_labels.hide()
figure.ticks.set_linewidth(0)
figure.frame.set_linewidth(0)
figure.save(outfile, max_dpi=dpi, adjust_bbox=False)
def getFilter(fitsfile):
"""Return filter name"""
instdet = getInstDet(fitsfile)
if instdet == 'acswfc':
ftr = fits.getval(fitsfile, 'filter1')
if 'CLEAR' in ftr:
ftr = fits.getval(fitsfile, 'filter2')
else:
ftr = fits.getval(fitsfile, 'filter')
return ftr.lower()
def findtimes(rpcdata_dir, camera):
"""Find the initial and final times of observation for the given camera"""
evtfile = os.path.abspath(glob.glob(
rpcdata_dir+'/'+'*{0}*ImagingEvts.ds'.format(camera.upper()))[0])
tstart = fits.getval(evtfile, 'TSTART', ext=1)
tstop = fits.getval(evtfile, 'TSTOP', ext=1)
return tstart, tstop
def applyNpolCorr(fname, unpolcorr):
"""
Determines whether non-polynomial distortion lookup tables should be added
as extensions to the science file based on the 'NPOLFILE' keyword in the
primary header and NPOLEXT kw in the first extension.
This is a default correction and will always run in the pipeline.
The file used to generate the extensions is
recorded in the NPOLEXT keyword in the first science extension.
If 'NPOLFILE' in the primary header is different from 'NPOLEXT' in the
extension header and the file exists on disk and is a 'new type' npolfile,
then the lookup tables will be updated as 'WCSDVARR' extensions.
"""
applyNPOLCorr = True
try:
# get NPOLFILE kw from primary header
fnpol0 = fits.getval(fname, 'NPOLFILE')
if fnpol0 == 'N/A':
utils.remove_distortion(fname, "NPOLFILE")
return False
fnpol0 = fileutil.osfn(fnpol0)
if not fileutil.findFile(fnpol0):
msg = """\n\tKw "NPOLFILE" exists in primary header but file %s not found
Non-polynomial distortion correction will not be applied\n
""" % fnpol0
logger.critical(msg)
raise IOError("NPOLFILE {0} not found".format(fnpol0))
try:
# get NPOLEXT kw from first extension header
fnpol1 = fits.getval(fname, 'NPOLEXT', ext=1)
fnpol1 = fileutil.osfn(fnpol1)
if fnpol1 and fileutil.findFile(fnpol1):
if fnpol0 != fnpol1:
applyNPOLCorr = True
else:
msg = """\n\tNPOLEXT with the same value as NPOLFILE found in first extension.
NPOL correction will not be applied."""
logger.info(msg)
applyNPOLCorr = False
else:
# npl file defined in first extension may not be found
# but if a valid kw exists in the primary header, non-polynomial
#distortion correction should be applied.
applyNPOLCorr = True
except KeyError:
# the case of "NPOLFILE" kw present in primary header but "NPOLEXT" missing
# in first extension header
applyNPOLCorr = True
except KeyError:
logger.info('\n\t"NPOLFILE" keyword not found in primary header')
applyNPOLCorr = False
return applyNPOLCorr
if isOldStyleDGEO(fname, fnpol0):
applyNPOLCorr = False
return (applyNPOLCorr and unpolcorr)
def det_confs(DIR): # Retrieve all possible detector configurations
dc = [] # List of all detector configurations ('BINX', 'BINY', 'WINDOWED', 'NAXIS1', 'NAXIS2')
for i in glob.glob(RAW+"*fits"):
tmp = []
for KW in ['BINX', 'BINY']:
tmp.append(fits.getval(i, KW))
for KW in ['NAXIS1', 'NAXIS2']:
tmp.append(fits.getval(i, KW, ext=1))
dc.append(tmp)
print "Determining all detector configurations."
dc = set(map(tuple, dc)) # Unique detector configurations
return dc
def check_flt(flt_path):
flt = os.path.split(flt_path)[-1]
det = fits.getval(flt_path,'DETECTOR')
filt = fits.getval(flt_path,'FILTER')
if det == 'IR' and filt == 'F110W':
if not os.path.exists(flt.replace('flt.fits','raw.fits')):
print 'NO RAW FOR {}'.format(flt)
asn_id = fits.getval(flt_path, 'asn_id')
if asn_id == 'NONE':
asn_id = fits.getval(flt_path, 'rootname').upper()
return asn_id
# raise
return None
def test_exposure_collision(camera, tmpdir):
"""
Tests attempting to take an exposure while one is already in progress.
With the SBIG cameras this will generate warning but still should work. Don't do this though!
"""
fits_path_1 = str(tmpdir.join('test_exposure_collision1.fits'))
fits_path_2 = str(tmpdir.join('test_exposure_collision2.fits'))
camera.take_exposure(2 * u.second, filename=fits_path_1)
camera.take_exposure(1 * u.second, filename=fits_path_2)
time.sleep(5)
assert os.path.exists(fits_path_1)
assert os.path.exists(fits_path_2)
assert fits.getval(fits_path_1, 'EXPTIME') == 2.0
assert fits.getval(fits_path_2, 'EXPTIME') == 1.0
def __init__(self,ifile, extension = 0,data_axis = 1, error_axis =3, x1key ='CRVAL1', dxkey='CD1_1', p1key='CRPIX1',style='linear'):
super(FitsSpec,self).__init__()
data = fits.getdata(ifile,extension)
self._f = data[data_axis,0,:]
self._ef = data[error_axis,0,:]
x1 = fits.getval(ifile,x1key)
p1 = fits.getval(ifile,p1key)
dx = fits.getval(ifile,dxkey)
self.wv = self._get_x(x1,p1,dx)
self.f = self._f
self.ef = self._ef
def find_npolfile(flist,detector,filters):
""" Search a list of files for one that matches the configuration
of detector and filters used.
"""
npolfile = None
for f in flist:
fdet = fits.getval(f,'detector')
if fdet == detector:
filt1 = fits.getval(f,'filter1')
filt2 = fits.getval(f,'filter2')
fdate = fits.getval(f,'date')
if filt1 == 'ANY' or \
(filt1 == filters[0] and filt2 == filters[1]):
npolfile = f
return npolfile
def csum_existence(filename):
'''
Check for the existence of a CSUM for a given input dataset.
Parameters:
filename : string
A raw dataset name.
Returns:
existence : bool
A boolean specifying if csums exist or not.
'''
rootname = os.path.basename(filename)[:9]
dirname = os.path.dirname(filename)
exptype = pf.getval(filename, "exptype")
# If getting one header keyword, getval is faster than opening.
# The more you know.
if exptype != "ACQ/PEAKD" and exptype != "ACQ/PEAKXD":
csums = glob.glob(os.path.join(dirname,rootname+"*csum*"))
if not csums:
existence = False
else:
existence = True
return existence
def count_imsets(file_list):
"""Count the total number of imsets in a file list.
The total number of imsets is counted by dividing the total number
of extensions (NEXTEND) by 3 (SCI, ERR, DQ).
Parameters
----------
file_list : list
list of all files to be counted
Returns
-------
total : ine
number of imsets
"""
if not isinstance(file_list, list):
file_list = [file_list]
total = 0
for item in file_list:
total += pyfits.getval(item,'NEXTEND',ext=0) / 3
return total
def retrieve_meta_table(filelist, i, nsubpersfh=None, nsfhperfile=None):
'''
'''
if nsfhperfile is None:
nsfhperfile = fits.getval(filelist[0], ext=0, keyword='NSFHPER')
if nsubpersfh is None:
nsubpersfh = fits.getval(filelist[0], ext=0, keyword='NSUBPER')
fnum, fi, fii, fiii = find_sfh_ixs(i, nsfhperfile, nsubpersfh)
fname = filelist[fnum]
metadata = t.Table.read(fname)
return metadata, fii
def test_quantity_data(self):
"""Data as Quantity."""
outfile = os.path.join(self.outdir, 'outspec2.fits')
# Write it out (flux_unit should be ignored)
specio.write_fits_spec(
outfile, self.wave, self.flux, pri_header=self.prihdr,
ext_header=self.scihdr, precision='double', flux_unit='foo')
# Read it back in and check values (flux_unit should be ignored)
hdr, wave, flux = specio.read_spec(outfile, flux_unit='foo')
# Compare data (trim_zero=True, pad_zero_ends=True)
np.testing.assert_allclose(
wave.value, [500.0, 1000.0, 2000.0, 2000.0 + self.epsilon, 4000.0,
5000.0, 6250.0],
rtol=1e-06)
np.testing.assert_allclose(
flux.value, [0.0, 0.1, 100.2, 10.0, 6.5, 1.2, 0.0])
assert wave.unit == self.wave.unit
assert flux.unit == self.flux.unit
# Compare primary header
assert hdr['PEDIGREE'] == 'DUMMY'
# Compare science header
assert fits.getval(outfile, 'SPEC_SRC', ext=1) == 'RANDOM'
def _get_RedLaws():
global RedLaws
extdir = os.path.join(rootdir, EXTDIR)
# get all the fits files in EXTDIR
globstr = os.path.join(extdir, '*.fits')
files = glob.glob(globstr)
if not files:
warnings.warn('Extinction files not found in %s' % (extdir,))
return
# replace ###.fits at the end of file names with *.fits
# and get a unique set
files = set(f[:-8] + '*.fits' for f in files)
# use _refTable to get the most recent version of each extinction file
# and add that to the RedLaws dict
for f in files:
lawf = _refTable(f)
key = pyfits.getval(lawf,'shortnm')
RedLaws[key.lower()] = lawf
def get_pixscale(fits_file):
"""
Retreive the image pixel scale from its FITS header
Parameters
----------
fits_file: str
Path to a FITS image file
Returns
-------
pixel_scale: float
The pixel scale of the image in arcseconds
"""
pixel_keys = has_pixelscale(fits_file)
if not pixel_keys:
raise IOError("Pixel scale not found in {0}.".format(fits_file))
pixel_key = pixel_keys.pop()
pixel_scale = abs(pyfits.getval(fits_file, pixel_key))
if pixel_key in PIXSCL_KEY_DEG:
pixel_scale *= 3600
return round(pixel_scale, 6)
def combineDarks(darklist):
"""Combine all the dark files into a master dark"""
darkComb = ccdproc.Combiner([ccdproc.CCDData.read(adark, unit="adu") for adark in darklist])
darkComb.sigma_clipping(low_thresh=3, high_thresh=3, func=np.ma.median)
darkmaster = darkComb.average_combine()
darkmaster.header['exptime'] = fits.getval(darklist[0], 'exptime')
return darkmaster
def run_detection(args):
"""Run iterative source detection."""
# Load data
maps = dict()
for mapname in ['counts', 'background', 'exposure']:
filename = args[mapname]
logging.info('Reading {0} map: {1}'.format(mapname, filename))
maps[mapname] = fits.getdata(filename)
# Compute scales in pixel coordinates
DEG_PER_PIX = np.abs(fits.getval(args['counts'], 'CDELT1'))
scales_deg = args['scales']
scales_pix = np.array(scales_deg) / DEG_PER_PIX
logging.info('Number of scales: {0}'.format(len(scales_deg)))
logging.info('DEG_PER_PIX: {0}'.format(DEG_PER_PIX))
logging.info('Scales in deg: {0}'.format(scales_deg))
logging.info('Scales in pix: {0}'.format(scales_pix))
# Run the iterative source detection
detector = IterativeSourceDetector(maps=maps, scales=scales_pix,
debug_output_folder=args['debug_output_folder'],
clobber=True)
detector.run()
# Save the results
# detector.save_fits(args['output_fits'])
detector.save_regions(args['output_regions'])
def split_files(all_files):
"""Split file list into two smaller lists for iraf tasks
Each list will have a selection from both early and late in time.
Parameters
----------
all_files : list
full list of files
Returns
-------
super_list : list
list of lists containing the split list of all files
"""
all_info = [(fits.getval(filename,'EXPSTART', 1), filename)
for filename in all_files]
all_info.sort()
all_files = [line[1] for line in all_info]
super_list = [all_files[0::2],
all_files[1::2]]
return super_list
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
toclose = False
if isinstance(f, str):
f = fits.open(f)
toclose = True
try:
pav3 = f[0].header['PA_V3']
except KeyError:
rootname = f[0].header['ROOTNAME']
sptfile = rootname+'_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f.filename() or "")
f[0].header['PA_V3'] = pav3
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f.filename() or "")
if toclose:
f.close()
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t',f)
return removed_files
def extract_input_filenames(drzfile):
"""
Generate a list of filenames from a drizzled image's header
"""
data_kws = fits.getval(drzfile, 'd*data', ext=0)
if len(data_kws) == 0:
return None
fnames = []
for kw in data_kws.cards:
f = kw.value.split('[')[0]
if f not in fnames:
fnames.append(f)
return fnames
def crop_wfc3_img(img_file, xmin, xmax, ymin, ymax, output_filename):
img = fits.getdata(img_file, 0)
cropped_img = img[ymin:ymax, xmin:xmax]
pyplot.figure()
pyplot.imshow(cropped_img, interpolation = 'nearest', cmap = 'bone', vmin = 0, vmax = 200)
pyplot.savefig('../../multispec/ccd_multispec/{}'.format(output_filename.replace('.fits', '.pdf')))
hdu = fits.PrimaryHDU(cropped_img)
hdu.header.set('Instrume', 'WFC3')
hdu.header.set('Proposid', 11360)
hdu.header.set('xlim', '({}, {})'.format(xmin, xmax))
hdu.header.set('ylim', '({}, {})'.format(ymin, ymax))
hdu.header.set('EXPTIME', fits.getval(img_file,'exptime', 0))
hdulist = fits.HDUList([hdu])
hdulist.writeto(output_filename, clobber = True)
def run_cal(file):
#get list of bad pixel masks that can be used
#bpmlist = glob.glob('/grp/jwst/wit/nircam/cv3_reffile_conversion/bpm/*ssbspmask.fits')
bpmlist = glob.glob(
'/ifs/jwst/wit/witserv/data7/nrc/reference_files/SSB/CV3/cv3_reffile_conversion/bpm/*DMSorient.fits'
)
satlist = glob.glob(
'/ifs/jwst/wit/witserv/data7/nrc/reference_files/SSB/CV3/cv3_reffile_conversion/welldepth/*ADU*DMSorient.fits'
)
#run the DQ init and bias_drift steps on the file
detstr = fits.getval(file, 'DETECTOR')
if 'LONG' in detstr:
detstr = detstr[0:4] + '5'
#find the appropriate bad pixel mask and saturation file for the detector
bpm = [s for s in bpmlist if detstr in s]
if len(bpm) > 1:
print(
"More than one bad pixel mask found. Need a better list of possibilities."
)
stophere
welldepth = [s for s in satlist if detstr in s]
if len(welldepth) > 1:
print(
"More than one saturation map found. Need a better list of possibilities."
)
stophere
#run the dq_init step
dqfile = file[0:-5] + '_dq_init.fits'
dqstep = DQInitStep.call(file,
config_file='dq_init.cfg',
override_mask=bpm[0],
output_file=dqfile)
#run saturation flagging
satfile = dqfile[0:-5] + '_saturation.fits'
satstep = SaturationStep.call(dqstep,
config_file='saturation.cfg',
override_saturation=welldepth[0],
output_file=satfile)
#run the reference pixel subtraction step using the group 0 subtraction and re-addition
#which is what the Build 4 pipeline used to use.
refcor = refpix_g0()
refcor.infile = satfile
refcor.outfile = None
refcor.run()
def test_svm_wcs(gather_output_data):
# Check the output primary WCSNAME includes FIT_SVM_GAIA as part of the string value
tdp_files = [
files for files in gather_output_data
if files.lower().find("total") > -1 and files.lower().endswith(".fits")
]
# For this dataset, there is only one total data product. However, make the test more general to
# handle multiple total data products which all have the same simple WCSNAME.
wcsnames = [
fits.getval(tdp, "WCSNAME", ext=1).upper() for tdp in tdp_files
]
assert len(
set(wcsnames)) == 1, f"WCSNAMES are not all the same: {wcsnames}"
def main():
"""
mv spec file from current dir to spec/
if dir spec not exist, mkdir spec.
"""
if not os.path.isdir('spec'):
print('mkdir spec')
os.mkdir('spec')
ifitsname = os.listdir(os.getcwd())
ifitsname = glob.glob('Y*.fits')
fitsnames = [
i for i in ifitsname if os.path.isfile(i)
and re.match("^Y[A-Z].+[a-l][\d]{6}\\.fits$", i) is not None
]
for i in fitsnames:
if is_spec(i):
fits = pyfits.open(i)
hdr = fits[1].header
naxis1 = pyfits.getval(i, 'NAXIS1', 1)
naxis2 = pyfits.getval(i, 'NAXIS2', 1)
if naxis1 == 2148 and naxis2 == 4612:
print('mv %s to spec/' % i)
shutil.move(i, '.' + os.sep + 'spec' + os.sep + i)
def set_inandout_filenames(step, config):
"""
Set the step input and output file names, and add the step suffix to the step map.
Args:
step: string, name of the step as in the pipeline
config: object, this is the configuration file object
Returns:
step_input_filename = string
step_output_filename = string
in_file_suffix = string, suffix of the input file
out_file_suffix = string, suffix of the output file
True_steps_suffix_map = string, path to the suffix map
"""
data_directory = config.get("calwebb_spec2_input_file", "data_directory")
working_directory = config.get("calwebb_spec2_input_file",
"working_directory")
initial_input_file_basename = config.get("calwebb_spec2_input_file",
"input_file")
initial_input_file = os.path.join(data_directory,
initial_input_file_basename)
# Get the detector used
detector = fits.getval(initial_input_file, "DETECTOR", 0)
True_steps_suffix_map = "full_run_map_" + detector + ".txt"
if os.path.isfile(initial_input_file):
print("\n Taking initial input file from data_directory:")
else:
initial_input_file = os.path.join(working_directory,
initial_input_file_basename)
print("\n Taking initial file from working_directory: ")
print(" Initial input file = ", initial_input_file, "\n")
run_calwebb_spec2 = config.getboolean("run_calwebb_spec2_in_full",
"run_calwebb_spec2")
if not run_calwebb_spec2:
pytests_directory = os.getcwd()
True_steps_suffix_map = os.path.join(
pytests_directory, "True_steps_suffix_map_" + detector + ".txt")
print("Pipeline was set to run step by step. Suffix map named: ",
True_steps_suffix_map, ", located in working directory.")
else:
print(
"Pipeline was set to run in full. Suffix map named: full_run_map_DETECTOR.txt, located in working directory."
)
suffix_and_filenames = get_step_inandout_filename(step, initial_input_file,
working_directory)
in_file_suffix, out_file_suffix, step_input_filename, step_output_filename = suffix_and_filenames
print("step_input_filename = ", step_input_filename)
print("step_output_filename = ", step_output_filename)
return step_input_filename, step_output_filename, in_file_suffix, out_file_suffix, True_steps_suffix_map
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
try:
pav3 = fits.getval(f, 'PA_V3')
except KeyError:
rootname = fits.getval(f, 'ROOTNAME')
sptfile = rootname + '_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
fits.setval(f, 'PA_V3', value=pav3)
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t', f)
return removed_files
def create_poller_file(img_list, proj_cell_dict):
"""Subroutine that executes make_poller_files.generate_poller_file() to generate custom MVM poller file
for execution of hapmultisequencer.run_mvm_processing().
Parameters
----------
img_list : list
list of images to process
proj_cell_dict : dict
Dictionary containing projection cell information
Returns
-------
poller_filename : str
Name of the newly created poller_filename
"""
# Locate bottom-left most skycell
pcell_filename = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'pars', 'allsky_cells.fits')
sc_nxy = fits.getval(
pcell_filename, "SC_NXY",
ext=0) # Projection cell side length (in skycells) in X or Y
closest_skycell = ""
min_dist = calc_skycell_dist(sc_nxy, sc_nxy, 1, 1) + 2.0
for sc in proj_cell_dict.keys():
x_index = proj_cell_dict[sc].x_index
y_index = proj_cell_dict[sc].y_index
dist = calc_skycell_dist(x_index, y_index, 1, 1)
if dist < min_dist:
min_dist = dist
closest_skycell = sc
# Create poller file
poller_filename = "temp_{}_mvm.out".format(closest_skycell)
rootname_list = []
for imgname in img_list:
rootname_list.append(imgname + "\n")
tf = tempfile.NamedTemporaryFile(mode='w+t', dir=os.getcwd())
with open(tf.name, 'w') as f:
f.writelines(rootname_list)
f.close()
make_poller_files.generate_poller_file(
tf.name,
poller_file_type="mvm",
output_poller_filename=poller_filename,
skycell_name=closest_skycell)
return poller_filename
def test_svm_wcs_uvis(gather_output_data):
print("\ntest_svm_wcs_uvis.")
# Get the TDP for this detector
tdp_files = [
files for files in gather_output_data
if files.lower().find("uvis_total") > -1
and files.lower().endswith("drc.fits")
]
# Check the WCS solution is as expected
wcsname = fits.getval(tdp_files[0], "WCSNAME", ext=1).upper()
print("\ntest_svm_wcs_uvis. WCSNAME: {} Output file: {}".format(
wcsname, tdp_files[0]))
assert WCS_UVIS_SUB_NAME in wcsname, f"WCSNAME is not as expected for file {tdp_files[0]}."
def get_keywd_val(fits_file_name, keywd, ext=0):
"""
This function obtains the value corresponding to the given keyword.
Args:
fits_file_name: name of the fits file used as input for the pipeline
keywd: keyword for which to obtain the value
ext: extension in which the kwyword lives, by default it is set to the
primary extension.
Returns:
keywd_val: the value corresponding to the inputed keyword
"""
keywd_val = fits.getval(fits_file_name, keywd, ext)
return keywd_val
def setup(listfile, do_ipc, ipc_only):
ipcdir = '/grp/jwst/wit/nircam/reference_files/SSB/CV2/delivery_Dec_2015/IPC/'
ipcfiles = glob.glob(ipcdir + '*DMSorient.fits')
#maskdir = '/grp/jwst/wit/nircam/cv3_reffile_conversion/bpm/'
maskdir = '/ifs/jwst/wit/witserv/data7/nrc/reference_files/SSB/cv3_reffile_conversion/bpm/'
maskfiles = glob.glob(maskdir + '*DMSorient.fits')
satfiles = glob.glob(
'/ifs/jwst/wit/witserv/data7/nrc/reference_files/SSB/cv3_reffile_conversion/welldepth/*ADU*DMSorient.fits'
)
files = read_listfile(listfile)
inputs = []
for file in files:
det = fits.getval(file, 'DETECTOR')
if det == 'NRCBLONG':
det = 'NRCB5'
if det == 'NRCALONG':
det = 'NRCA5'
maskfile = [s for s in maskfiles if det in s][0]
if len(maskfile) > 1:
print(
"More than one bad pixel mask found. Need a better list of possibilities."
)
stophere
welldepth = [s for s in satfiles if detstr in s]
if len(welldepth) > 1:
print(
"More than one saturation map found. Need a better list of possibilities."
)
stophere
ipckernel = None
if do_ipc == True:
cfile = 'proc_superbias_inputs.cfg'
outfile = file[0:-5] + '_dq_init_bias_drift_nosidepix_ipc.fits'
if ipc_only:
outfile = file[0:-5] + '_ipc.fits'
ipckernel = [s for s in ipcfiles if det in s][0]
else:
cfile = 'proc_superbias_inputs_withIPC.cfg'
outfile = file[0:-5] + '_dq_init_bias_drift_nosidepix.fits'
inputs.append((file, cfile, outfile, maskfile, ipckernel, ipc_only))
return inputs
def _print_results(file, mean, median, std):
"""
Print the results of the analysis to the termainl.
"""
exposure_time = getval(file, 'EXPTIME')
print('\nFile:\t\t{}'.format(file))
print('\nExposure:\t{}'.format(exposure_time))
print('Mean:\t\t{:.2f}'.format(mean))
print('Median:\t\t{:.2f}'.format(median))
print('Std:\t\t{:.2f}'.format(std))
return None
def flareCompare(inst='cos_g130m', band='SiIV', nflares=3, ax=None):
if ax is None: ax = plt.gca()
ax.set_xlabel('Time [s]')
ax.set_ylabel('Normalized Flux')
flares = reduce.combine_flarecats(band, inst)
flares = flares[:nflares]
bands, dt = flares.meta['BANDS'], flares.meta['DT']
groups = [range(len(bands))]
mxpeak = max(flares['pkratio'])
# plot all curves
lines, labels = [], []
for flare in flares:
star = flare['star']
label = starprops['name tex'][star]
# curve = reduce.auto_curve(star, inst, bands=bands, dt=dt, appx=False, groups=groups)
# t0, t1, cps, cpserr = zip(*curve)[0]
ph, p = io.readphotons(star, inst)
t0, t1, cps, cpserr = sp.smooth_curve(p['time'],
p['wavelength'],
p['epsilon'],
bands=bands,
n=100)
tmid = (t0 + t1) / 2.0
t = tmid - flare['peak rel']
qcps = fits.getval(rc.flarepath(star, inst, 'SiIV'), 'QSCTCPS', ext=1)
rate, err = cps / qcps, cpserr / qcps
fac = 10.0**np.floor(np.log10(mxpeak / flare['pkratio']))
if fac > 1.0:
label += ' $\\times$%i' % fac
rate = (rate - 1.0) * fac + 1.0
err = err * fac
# line = ax.plot(t, rate, '-')[0]
# ax.errorbar(t, rate, err, fmt='.', color=line.get_color(), capsize=0)
line, poly = pu.errorpoly(t, rate, err, ax=ax)
lines.append(line)
labels.append(label)
ax.legend(lines, labels, loc='best', fontsize='small')
starts, stops = flares['start rel'] - flares['peak rel'], flares[
'stop rel'] - flares['peak rel']
ax.set_xlim(min(starts) * 1.5, max(stops))
def applyD2ImCorr(fname, d2imcorr):
applyD2IMCorr = True
try:
# get D2IMFILE kw from primary header
fd2im0 = fits.getval(fname, 'D2IMFILE')
if fd2im0 == 'N/A':
utils.remove_distortion(fname, "D2IMFILE")
return False
fd2im0 = fileutil.osfn(fd2im0)
if not fileutil.findFile(fd2im0):
msg = """\n\tKw D2IMFILE exists in primary header but file %s not found\n
Detector to image correction will not be applied\n""" % fd2im0
logger.critical(msg)
print(msg)
raise IOError("D2IMFILE {0} not found".format(fd2im0))
try:
# get D2IMEXT kw from first extension header
fd2imext = fits.getval(fname, 'D2IMEXT', ext=1)
fd2imext = fileutil.osfn(fd2imext)
if fd2imext and fileutil.findFile(fd2imext):
if fd2im0 != fd2imext:
applyD2IMCorr = True
else:
applyD2IMCorr = False
else:
# D2IM file defined in first extension may not be found
# but if a valid kw exists in the primary header,
# detector to image correction should be applied.
applyD2IMCorr = True
except KeyError:
# the case of D2IMFILE kw present in primary header but D2IMEXT missing
# in first extension header
applyD2IMCorr = True
except KeyError:
print('D2IMFILE keyword not found in primary header')
applyD2IMCorr = False
return applyD2IMCorr
def fitcoord_edge(infile, overwrite=False):
print('\n#############################')
print('Getting the distortion map.')
# entering the channel image directory.
print('\t Entering the channel image directory, \"' + fi.chimagedir +
'\".')
os.chdir(fi.chimagedir)
basename = fits.getval('../' + infile, 'FRAMEID')
for j in range(1, 25):
fitcoord_edge_each(basename + '.ch%02dedge' % j, overwrite=overwrite)
print('\t Go back to the original directory.')
os.chdir('..')
return
def populate_bpm_table(directory, db_address=_DEFAULT_DB):
db_session = get_session(db_address=db_address)
bpm_filenames = glob(os.path.join(directory, 'bpm*.fits*'))
for bpm_filename in bpm_filenames:
if bpm_filename[-3:] == '.fz':
extension_number = 1
else:
extension_number = 0
site = fits.getval(bpm_filename, 'SITEID', extension_number).lower()
instrument = fits.getval(bpm_filename, 'INSTRUME', extension_number).lower()
ccdsum = fits.getval(bpm_filename, 'CCDSUM', extension_number)
creation_date = date_utils.epoch_string_to_date(fits.getval(bpm_filename, 'DAY-OBS',
extension_number))
telescope_id = get_telescope_id(site=site, instrument=instrument, db_address=db_address)
bpm_attributes = {'telescope_id': telescope_id, 'filepath': os.path.abspath(directory),
'filename': os.path.basename(bpm_filename), 'ccdsum': ccdsum,
'creation_date': creation_date}
add_or_update_record(db_session, BadPixelMask, bpm_attributes, bpm_attributes)
db_session.commit()
db_session.close()
def get_modeused_and_rawdatrt_PTT_cfg_file(infile_name):
# get script directory and config name
out_dir = os.path.dirname(infile_name)
PPT_cfg_file = os.path.join(out_dir, "PTT_config.cfg")
if not os.path.isfile(PPT_cfg_file):
detector = fits.getval(infile_name, "DETECTOR")
PPT_cfg_file = PPT_cfg_file.replace(".cfg", "_" + detector + ".cfg")
with open(PPT_cfg_file, "r") as cfg:
for i, line in enumerate(cfg.readlines()):
if "#" not in line:
if "mode_used" in line:
mode_used = line.split()[2]
if "raw_data_root_file" in line:
raw_data_root_file = line.split()[2]
return mode_used, raw_data_root_file
def updatecomheader(extractedfiles, objname):
airmasses = []
exptimes = []
for f in extractedfiles:
airmasses.append(float(fits.getval(f, 'AIRMASS')))
exptimes.append(float(fits.getval(f, 'EXPTIME')))
fits.setval(objname + '_com.fits', 'AIRMASS', value=np.mean(airmasses))
fits.setval(objname + '_com.fits', 'SLIT', value=fits.getval(extractedfiles[0], 'MASKNAME').replace('arcsec', ''))
comhdu = fits.open(objname + '_com.fits', mode='update')
extractedhdu = fits.open(extractedfiles[0])
for k in extractedhdu[0].header.keys():
if not k in comhdu[0].header.keys():
extractedhdu[0].header.cards[k].verify('fix')
comhdu[0].header.append(extractedhdu[0].header.cards[k])
comhdu.flush(output_verify='fix')
comhdu.close()
extractedhdu.close()
dateobs = fits.getval(objname + '_com.fits', 'DATE-OBS')
dateobs += 'T' + fits.getval(objname + '_com.fits', 'TIME-OBS')
fits.setval(objname + '_com.fits', 'DATE-OBS', value=dateobs)
def test_svm_wcs_uvis_all(gather_output_data):
# Check the output primary WCSNAME
print("\ntest_svm_wcs_uvis_all.")
uvis_files = [
files for files in gather_output_data
if files.lower().find("_uvis_") > -1
and files.lower().endswith("drc.fits")
]
wcsnames = [
fits.getval(uvis, "WCSNAME", ext=1).upper() for uvis in uvis_files
]
assert len(
set(wcsnames)
) == 1, f"WCSNAMES are not all the same for the UVIS detector: {wcsnames}"
def is_Neon(fn):
"""
whether the Neon lamp light
'CLAMP2' : FeAr
'CLAMP3' : Neon
'CLAMP4' : Helium
fn : fits name
type : string
return :
type : boolean
"""
val3 = pyfits.getval(fn, 'CLAMP3')
if int(val3) == 1:
return True
return False
def test_svm_wcs(gather_output_data):
# Check the output primary WCSNAME includes FIT_SVM_GAIA as part of the string value
tdp_files = [
files for files in gather_output_data
if files.lower().find("total") > -1 and files.lower().endswith(".fits")
]
# This check is for all total data products which have the same "type" of WCSNAME -
# in this case a name akin to *-FIT_SVM_GAIA*.
wcsnames = [
fits.getval(tdp, "WCSNAME", ext=1).upper().split('-')[1]
for tdp in tdp_files
]
assert len(
set(wcsnames)) == 1, f"WCSNAMES are not all the same: {wcsnames}"
def determine_exptime_weights(self, hdr_key_exptime='EXPTIME'):
"""Setting the weights based on the exposure times (from the headers)
(weight proportional to EXPTIME)
Attributes
----------
hdr_key_exptime : str, optional
Header keyword in primary header that contains the exposure time.
(default: EXPTIME)
"""
exptimes = [fits.getval(f, hdr_key_exptime) for f in self.files]
self.max_et = max(exptimes)
self.weight_type = 'exptime'
self.weights = [et / self.max_et for et in exptimes]
def copy_dark_lp(src, dst):
"""Copies a file from source to destination if it meets the header
value criteria of ``TARGNAME = 'DARK'`` or ``'DARK-NM'``, and
``EXPTIME = 900`` or ``EXPTIME = 1800``.
Parameters
----------
src : str
The absolute path of the source location.
dst : str
The absolute path of the destination location.
"""
# Check for all files with raw in the name
for filepath in glob.glob(os.path.join(src, '*')):
# Getting just the filename
file = os.path.basename(filepath)
# If that file is raw, then check header
if 'raw' in file:
# Get header information
exptime = fits.getval(filepath, 'EXPTIME')
targname = fits.getval(filepath, 'TARGNAME')
# Only copy appropriate files
if targname == 'DARK' or targname == 'DARK-NM':
if exptime in [900.0, 1800.0]:
dstfile = os.path.join(dst, file)
if not os.path.exists(dstfile):
# print('Copying {} to {}'.format(filepath, dst+file))
print('\tLP log: Copying {} to {}'.format(
filepath, dstfile))
shutil.copyfile(filepath, dstfile)
def myalign(input_files,
updatewcs=False, clean=True, logfile=None, verbose=False,
wcsname='ALIGN_ALL', refimage=None,
**kwargs):
logger = RegularPrint() if verbose else PrintRedirect(logfile)
if isinstance(input_files, str):
input_files = [input_files]
already_updated = []
for file in input_files:
if 'WCSMAN' not in fits.getheader(file, extname='PRIMARY'):
fits.setval(file, 'WCSMAN', 'PRIMARY', value=False,
comment='Has WCS been manually updated?')
# Only update WCS if has not been performed on ANY of the inputs
elif fits.getval(file, 'WCSMAN', 'PRIMARY') is True:
already_updated.append(file.rpartition('/')[-1])
if updatewcs:
if len(already_updated) > 0:
updatewcs = False
print('--NOT UPDATING WCS because manually updated files exist: '
f'{already_updated}')
else:
print('--Updating WCS manually as requested')
# Set default options
t_kwgs = {
'conv_width': 4.0,
'threshold': 200,
'shiftfile': False,
'updatehdr': True,
'writecat': False,
'clean': True,
'residplot': 'NoPlot',
'see2dplot': False,
'reusename': True,
'interactive': False,
'updatewcs': updatewcs,
'enforce_user_order': True,
'expand_refcat': True,
'wcsname': wcsname,
'refimage': (refimage or ''),
}
t_kwgs.update(kwargs)
# Initial Alignment
with logger:
print(t_kwgs)
tweakreg.TweakReg(input_files, **t_kwgs)
if t_kwgs['updatewcs']:
for file in input_files:
fits.setval(file, 'WCSMAN', 'PRIMARY', value=True)
def open_fits(self):
"""Open CCI file and populated attributes with
header keywords and data arrays.
"""
hdu = fitsio.FITS(self.input_file)
primary = hdu[0].read_header()
assert (hdu[2].read().shape == (Y_UNBINNED, X_UNBINNED)), 'ERROR: Input CCI not standard dimensions'
self.detector = primary['DETECTOR']
self.segment = primary['SEGMENT']
self.obsmode = primary['OBSMODE']
self.expstart = primary['EXPSTART']
self.expend = primary['EXPEND']
self.exptime = primary['EXPTIME']
self.numfiles = primary['NUMFILES']
self.counts = primary['COUNTS']
self.dethv = primary.get('DETHV', -999)
try:
self.brftab = primary['brftab'].split('$')[1]
except:
self.brftab = 'x1u1459il_brf.fits'
if self.expstart:
#----Finds to most recently created HVTAB
hvtable_list = glob.glob(os.path.join(os.environ['lref'], '*hv.fits'))
HVTAB = hvtable_list[np.array([fits.getval(item, 'DATE') for item in hvtable_list]).argmax()]
hvtab = fits.open(HVTAB)
if self.segment == 'FUVA':
hv_string = 'HVLEVELA'
elif self.segment == 'FUVB':
hv_string = 'HVLEVELB'
self.file_list = [line[0].decode("utf-8") for line in hdu[1].read()]
self.big_array = np.array([rebin(hdu[i+2].read(), bins=(self.ybinning, self.xbinning)) for i in range(32)])
self.get_counts(self.big_array)
self.extracted_charge = self.pha_to_coulombs(self.big_array)
self.gain_image = np.zeros((YLEN, XLEN))
self.modal_gain_width = np.zeros((YLEN, XLEN))
self.cnt00_00 = len(self.big_array[0].nonzero()[0])
self.cnt01_01 = len(self.big_array[1].nonzero()[0])
self.cnt02_30 = len(self.big_array[2:31].nonzero()[0])
self.cnt31_31 = len(self.big_array[31:].nonzero()[0])
def get_ra_dec(fn):
"""
get the source coords
return : ra, dec (format like '12:34:56.78', '+23:45:67.89')
"""
standname, others = pyfits.getval(fn, 'OBJECT').split()
standname = standname.lower()
radeclst = open(config_path + os.sep + 'objradec.lst').readlines()
radecdic = dict([i.split(None, 1) for i in radeclst])
if standname in radecdic:
ra, dec = radecdic[standname].split()
return ra, dec
print('can not match ', standname, ', please check and edit objradec.lst')
webbrowser.open(config_path + os.sep + 'objradec.lst')
raw_input('edit ok?(y)')
return get_ra_dec(fn)
def load_flux(fname):
"""
load the 1D flux array from file
Parameters
----------
fname : str
Returns
-------
: numpy.ndarray
"""
surface = fits.getval(fname, 'PHXREFF')**2 * 4 * np.pi
flux = fits.getdata(fname).astype(np.float64) * surface
return flux
def select_images(image_list, image_types):
images = []
for filename in image_list:
try:
if os.path.splitext(filename)[1] == '.fz':
ext = 1
else:
ext = 0
if fits.getval(filename, 'OBSTYPE', ext=ext) in image_types:
images.append(filename)
except Exception as e:
logger.error('Unable to get OBSTYPE from {0}.'.format(filename))
logger.error(e)
continue
return images
def add_detector2filename(working_directory, step_input_file):
"""
This function is only used in the case when the pipeline is run in full. At the end of the run, before the files
have been moved to the working directory, the names will be changed to include the detector name.
Args:
working_directory: string, path of working directory
step_input_file: string, full name of initial input file
Returns:
nothing
"""
# dictionary of the steps and corresponding strings to be added to the file name after the step has ran
step_strings = [
"_assign_wcs", "_bkg_subtract", "_imprint_subtract", "_msa_flagging",
"_extract_2d", "_flat_field", "_srctype", "_pathloss", "_barshadow",
"_photom", "_resample_spec", "_cube_build", "_extract_1d",
"_interpolatedflat", "_s2d", "_s3d", "_x1d", "_cal"
]
# get the detector name and add it
det = fits.getval(step_input_file, "DETECTOR", 0)
#step_input_file_basename = os.path.basename(step_input_file).replace(".fits", "")
ptt_directory = os.getcwd() # directory where PTT lives
#step_files = glob.glob(os.path.join(ptt_directory, step_input_file_basename+"*.fits")) # get all fits files just created
#if len(step_files) == 0:
# step_files = glob.glob(os.path.join(ptt_directory, "*.fits"))
step_files = glob.glob(os.path.join(ptt_directory, "*.fits"))
for sf in step_files:
for stp_str in step_strings:
if stp_str in sf.lower(
): # look for the step string appearing in the name of the files
# make sure to get the right name for the MSA flagging step output file
if "msaflagopenstep" in sf:
sf = sf.replace("msaflagopenstep", "msa_flagging")
if det.lower() not in sf.lower(
): # only add the detector to the name if it is not part of the basename
new_name = os.path.basename(
sf.replace(stp_str + ".fits",
"_" + det + stp_str + ".fits"))
else:
print("Detector ", det,
" is already part of the file name ",
os.path.basename(sf), ", PTT will not add it again.")
new_name = os.path.basename(sf)
# move to the working directory
new_name = os.path.join(working_directory, new_name)
subprocess.run(["mv", sf, new_name])
def calibrateClean(self):
try:
# Calibrate each file
os.chdir(self.dirCleanStf)
print("DEBUG 2nd dec --", self.dirCleanStf)
# open writeable file for log
_log = open('calibrate.log', 'w')
cmdBase = 'calibrate_new %s ' % self.calFlags
cmdBase += '-I %s ' % self.calCooStar
cmdBase += '-N %s ' % self.calFile
cmdBase += '-M %s ' % self.calColumn
if (self.calStars != None) and (len(self.calStars) > 0):
cmdBase += '-S '
for cc in range(len(self.calStars)):
if (cc != 0):
cmdBase += ','
cmdBase += self.calStars[cc]
cmdBase += ' '
for file in self.cleanFiles:
fitsFile = '../%s.fits' % file
listFile = '%s_%3.1f_stf.lis' % (file, self.corrClean)
# Get the position angle
angle = float(fits.getval(fitsFile, 'ROTPOSN')) - 0.7
calCamera = calibrate.get_camera_type(fitsFile)
cmd = cmdBase + ('-T %.1f -c %d ' % (angle, calCamera))
cmd += listFile
# 2nd dec 2009 - changed "_out" "_log"
_log.write(cmd + '\n')
args = cmd.split()[1:]
calibrate.main(args)
_log.close() # clean up
# Copy over the calibration list.
shutil.copyfile(self.calFile, 'clean_phot_list.txt')
os.chdir(self.dirStart)
except:
os.chdir(self.dirStart)
raise
