def reprocess_wfc3ir(parallel=False, cpu_count=0):
"""
Run reprocessing script to flatten IR backgrounds
"""
if parallel:
# Might require light backend to avoid segfaults in multiprocessing
import matplotlib as mpl
mpl.rcParams['backend'] = 'agg'
import glob
import os
# https://github.com/gbrammer/wfc3
try:
from reprocess_wfc3 import reprocess_wfc3
except:
try:
from mywfc3 import reprocess_wfc3
except:
print("""
Couldn\'t `import reprocess_wfc3`.
Get it from https://github.com/gbrammer/reprocess_wfc3 """)
return False
# Fetch calibs in serial
print('\ngrizli.pipeline.reprocess: Fetch calibrations...\n')
files = glob.glob('*raw.fits')
for file in files:
reprocess_wfc3.fetch_calibs(
file,
ftpdir='https://hst-crds.stsci.edu/unchecked_get/references/hst/',
verbose=False)
# Make ramp diagnostic images
if parallel:
files = glob.glob('*raw.fits')
reprocess_wfc3.show_ramps_parallel(files, cpu_count=cpu_count)
# Reprocess all raw files
files = glob.glob('*raw.fits')
reprocess_wfc3.reprocess_parallel(files, cpu_count=cpu_count)
else:
files = glob.glob('*raw.fits')
for file in files:
if not os.path.exists(file.replace('raw.fits', 'ramp.png')):
reprocess_wfc3.show_MultiAccum_reads(raw=file,
stats_region=[[300, 700],
[300, 700]])
for file in files:
if not os.path.exists(file.replace('raw.fits', 'flt.fits')):
reprocess_wfc3.make_IMA_FLT(raw=file,
stats_region=[[300, 700],
[300, 700]])
def reprocess_wfc3ir(parallel=False, cpu_count=0):
"""
Run reprocessing script to flatten IR backgrounds
"""
if parallel:
# Might require light backend to avoid segfaults in multiprocessing
import matplotlib as mpl
mpl.rcParams['backend'] = 'agg'
import glob
import os
# https://github.com/gbrammer/wfc3
try:
from reprocess_wfc3 import reprocess_wfc3
except:
try:
from mywfc3 import reprocess_wfc3
except:
print("""
Couldn\'t `import reprocess_wfc3`.
Get it from https://github.com/gbrammer/reprocess_wfc3 """)
return False
# Fetch calibs in serial
print('\ngrizli.pipeline.reprocess: Fetch calibrations...\n')
files=glob.glob('*raw.fits')
for file in files:
reprocess_wfc3.fetch_calibs(file, ftpdir='https://hst-crds.stsci.edu/unchecked_get/references/hst/', verbose=False)
# Make ramp diagnostic images
if parallel:
files=glob.glob('*raw.fits')
reprocess_wfc3.show_ramps_parallel(files, cpu_count=cpu_count)
# Reprocess all raw files
files=glob.glob('*raw.fits')
reprocess_wfc3.reprocess_parallel(files, cpu_count=cpu_count)
else:
files=glob.glob('*raw.fits')
for file in files:
if not os.path.exists(file.replace('raw.fits','ramp.png')):
reprocess_wfc3.show_MultiAccum_reads(raw=file, stats_region=[[300,700], [300,700]])
for file in files:
if not os.path.exists(file.replace('raw.fits','flt.fits')):
reprocess_wfc3.make_IMA_FLT(raw=file, stats_region=[[300,700], [300,700]])
def inspect(root='grizli', force=False):
"""
Run the GUI inspection tool on the `ramp.png` images to flag problematic
reads with high backgrounds and/or satellite trails.
Click the right mouse button to flag a given read and go to the next
object with the 'n' key.
Type 'q' when done.
Parameters
----------
root : str
Rootname for the output inspection file:
>>> root = 'grizli'
>>> file = '{0}_inspect.fits'.format(root)
Returns
-------
Nothing returned. Makes the inspection file and runs the reprocessing.
.. note:: If the script fails puking lots of Tk-related messages, be sure
to run this script iin a fresh python session *before* importing
`~matplotlib`.
"""
import os
import glob
import matplotlib
matplotlib.use("TkAgg") ### This needs to be run first!
#import mywfc3.reprocess_wfc3
from reprocess_wfc3 import reprocess_wfc3
import astropy.io.fits as pyfits
import numpy as np
files = glob.glob('*ramp.png')
files.sort()
# Run the GUI, 'q' to quit
try:
import mywfc3.inspect
if os.path.exists('{0}_inspect.fits'.format(root)):
if force:
x = mywfc3.inspect.ImageClassifier(images=files,
logfile='{0}_inspect'.format(root))
else:
x = mywfc3.inspect.ImageClassifier(images=files,
logfile='{0}_inspect'.format(root))
except:
pass
if not os.path.exists('{0}_inspect.fits'.format(root)):
return True
#im = pyfits.open('inspect_raw.info.fits')
im = pyfits.open('{0}_inspect.fits'.format(root))
tab = im[1].data
fl = im['FLAGGED'].data
is_flagged = fl.sum(axis=1) > 0
sat_files = [file.replace('ramp.png', 'flt.fits') for file in tab['images'][is_flagged]]
read_idx = np.arange(14, dtype=int)+1
idx = np.arange(fl.shape[0])
for i in idx[is_flagged]:
pop_reads = list(read_idx[fl[i,:] > 0])
raw = tab['images'][i].replace('_ramp.png', '_raw.fits')
ramp_file = tab['images'][i].replace('_ramp.png', '_ramp.dat')
#sn_pop = mywfc3.inspect.check_background_SN(ramp_file=ramp_file, show=False)
#pop_reads = np.cast[int](np.unique(np.hstack((pop_reads, sn_pop))))
#pop_reads = list(pop_reads)
flt = raw.replace('_raw','_flt')
if os.path.exists(flt):
flt_im = pyfits.open(flt)
if 'NPOP' in flt_im[0].header:
if flt_im[0].header['NPOP'] > 0:
print('Skip %s' %(flt))
continue
print('Process %s %s' %(raw, pop_reads))
reprocess_wfc3.make_IMA_FLT(raw=raw, pop_reads=pop_reads, flatten_ramp=True)
# Remove "killed"
kill_files = [file.replace('ramp.png', 'flt.fits') for file in tab['images'][tab['kill'] > 0]]
for file in kill_files:
if os.path.exists(file):
os.remove(file)
return True
def inspect(root='grizli', force=False):
"""
Run the GUI inspection tool on the `ramp.png` images to flag problematic
reads with high backgrounds and/or satellite trails.
Click the right mouse button to flag a given read and go to the next
object with the 'n' key.
Type 'q' when done.
Parameters
----------
root : str
Rootname for the output inspection file:
>>> root = 'grizli'
>>> file = '{0}_inspect.fits'.format(root)
Returns
-------
Nothing returned. Makes the inspection file and runs the reprocessing.
.. note:: If the script fails puking lots of Tk-related messages, be sure
to run this script iin a fresh python session *before* importing
`~matplotlib`.
"""
import os
import glob
import matplotlib
matplotlib.use("TkAgg") ### This needs to be run first!
#import mywfc3.reprocess_wfc3
from reprocess_wfc3 import reprocess_wfc3
import astropy.io.fits as pyfits
import numpy as np
files = glob.glob('*ramp.png')
files.sort()
# Run the GUI, 'q' to quit
try:
import mywfc3.inspect
if os.path.exists('{0}_inspect.fits'.format(root)):
if force:
x = mywfc3.inspect.ImageClassifier(images=files,
logfile='{0}_inspect'.format(root))
else:
x = mywfc3.inspect.ImageClassifier(images=files,
logfile='{0}_inspect'.format(root))
except:
pass
if not os.path.exists('{0}_inspect.fits'.format(root)):
return True
#im = pyfits.open('inspect_raw.info.fits')
im = pyfits.open('{0}_inspect.fits'.format(root))
tab = im[1].data
fl = im['FLAGGED'].data
is_flagged = fl.sum(axis=1) > 0
sat_files = [file.replace('ramp.png', 'flt.fits') for file in tab['images'][is_flagged]]
read_idx = np.arange(14, dtype=int)+1
idx = np.arange(fl.shape[0])
for i in idx[is_flagged]:
pop_reads = list(read_idx[fl[i,:] > 0])
raw = tab['images'][i].replace('_ramp.png', '_raw.fits')
ramp_file = tab['images'][i].replace('_ramp.png', '_ramp.dat')
#sn_pop = mywfc3.inspect.check_background_SN(ramp_file=ramp_file, show=False)
#pop_reads = np.cast[int](np.unique(np.hstack((pop_reads, sn_pop))))
#pop_reads = list(pop_reads)
flt = raw.replace('_raw','_flt')
if os.path.exists(flt):
flt_im = pyfits.open(flt)
if 'NPOP' in flt_im[0].header:
if flt_im[0].header['NPOP'] > 0:
print('Skip %s' %(flt))
continue
print('Process %s %s' %(raw, pop_reads))
reprocess_wfc3.make_IMA_FLT(raw=raw, pop_reads=pop_reads, flatten_ramp=True)
# Remove "killed"
kill_files = [file.replace('ramp.png', 'flt.fits') for file in tab['images'][tab['kill'] > 0]]
for file in kill_files:
if os.path.exists(file):
os.remove(file)
return True
def reprocess_wfc3ir(parallel=False, cpu_count=0, clean_dark_refs=True):
"""
Run reprocessing script to flatten IR backgrounds
"""
if parallel:
# Might require light backend to avoid segfaults in multiprocessing
import matplotlib as mpl
mpl.rcParams['backend'] = 'agg'
import glob
import os
import astropy.io.fits as pyfits
# https://github.com/gbrammer/wfc3
try:
from reprocess_wfc3 import reprocess_wfc3
except:
try:
from mywfc3 import reprocess_wfc3
except:
print("""
Couldn\'t `import reprocess_wfc3`.
Get it from https://github.com/gbrammer/reprocess_wfc3 """)
return False
# Fetch calibs in serial
print('\ngrizli.pipeline.reprocess: Fetch calibrations...\n')
files = glob.glob('*raw.fits')
for file in files:
reprocess_wfc3.fetch_calibs(file, ftpdir='https://hst-crds.stsci.edu/unchecked_get/references/hst/', verbose=False)
# Make ramp diagnostic images
if parallel:
files = glob.glob('*raw.fits')
files.sort()
reprocess_wfc3.show_ramps_parallel(files, cpu_count=cpu_count)
# Reprocess all raw files
files = glob.glob('*raw.fits')
files.sort()
reprocess_wfc3.reprocess_parallel(files, cpu_count=cpu_count)
else:
files = glob.glob('*raw.fits')
files.sort()
for file in files:
if not os.path.exists(file.replace('raw.fits', 'ramp.png')):
reprocess_wfc3.show_MultiAccum_reads(raw=file, stats_region=[[300, 700], [300, 700]])
for file in files:
if not os.path.exists(file.replace('raw.fits', 'flt.fits')):
reprocess_wfc3.make_IMA_FLT(raw=file, stats_region=[[300, 700], [300, 700]])
# Remove DRK reference files, which now proliferate
if clean_dark_refs:
files = glob.glob('*raw.fits')
files.sort()
for file in files:
im = pyfits.open(file)
dark_file = im[0].header['DARKFILE'].replace('iref$', '')
dark_file = os.path.join(os.getenv('iref'), dark_file)
im.close()
if os.path.exists(dark_file):
print('clean_dark: {0}'.format(dark_file))
os.remove(dark_file)