def run(self):
"""The main method. See module docstrings for further details."""
logging.info('Begin logging for bias_monitor')
# Get the output directory and setup a directory to store the data
self.output_dir = os.path.join(get_config()['outputs'], 'bias_monitor')
ensure_dir_exists(os.path.join(self.output_dir, 'data'))
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in ['nircam', 'niriss', 'nirspec']:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible full-frame apertures for this instrument
siaf = Siaf(self.instrument)
possible_apertures = [
aperture for aperture in siaf.apertures
if siaf[aperture].AperType == 'FULLSCA'
]
for aperture in possible_apertures:
logging.info('Working on aperture {} in {}'.format(
aperture, instrument))
self.aperture = aperture
# Locate the record of most recent MAST search; use this time
# (plus a 30 day buffer to catch any missing files from
# previous run) as the start time in the new MAST search.
most_recent_search = self.most_recent_search()
self.query_start = most_recent_search - 30
# Query MAST for new dark files for this instrument/aperture
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
new_entries = monitor_utils.mast_query_darks(
instrument, aperture, self.query_start, self.query_end)
# Exclude ASIC tuning data
len_new_darks = len(new_entries)
new_entries = monitor_utils.exclude_asic_tuning(new_entries)
len_no_asic = len(new_entries)
num_asic = len_new_darks - len_no_asic
logging.info(
"\tFiltering out ASIC tuning files removed {} dark files.".
format(num_asic))
logging.info('\tAperture: {}, new entries: {}'.format(
self.aperture, len(new_entries)))
# Set up a directory to store the data for this aperture
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
if len(new_entries) > 0:
ensure_dir_exists(self.data_dir)
# Get any new files to process
new_files = []
for file_entry in new_entries:
output_filename = os.path.join(self.data_dir,
file_entry['filename'])
output_filename = output_filename.replace(
'_uncal.fits', '_uncal_0thgroup.fits').replace(
'_dark.fits', '_uncal_0thgroup.fits')
# Dont process files that already exist in the bias stats database
file_exists = self.file_exists_in_database(output_filename)
if file_exists:
logging.info(
'\t{} already exists in the bias database table.'.
format(output_filename))
continue
# Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem.
try:
filename = filesystem_path(file_entry['filename'])
uncal_filename = filename.replace('_dark', '_uncal')
if not os.path.isfile(uncal_filename):
logging.info(
'\t{} does not exist in JWQL filesystem, even though {} does'
.format(uncal_filename, filename))
else:
new_file = self.extract_zeroth_group(
uncal_filename)
new_files.append(new_file)
except FileNotFoundError:
logging.info(
'\t{} does not exist in JWQL filesystem'.format(
file_entry['filename']))
# Run the bias monitor on any new files
if len(new_files) > 0:
self.process(new_files)
monitor_run = True
else:
logging.info(
'\tBias monitor skipped. {} new dark files for {}, {}.'
.format(len(new_files), instrument, aperture))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'entries_found': len(new_entries),
'files_found': len(new_files),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Bias Monitor completed successfully.')
def run(self):
"""The main method. See module docstrings for further
details.
"""
logging.info('Begin logging for readnoise_monitor\n')
# Get the output directory and setup a directory to store the data
self.output_dir = os.path.join(get_config()['outputs'],
'readnoise_monitor')
ensure_dir_exists(os.path.join(self.output_dir, 'data'))
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in JWST_INSTRUMENT_NAMES:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible apertures for this instrument
siaf = Siaf(self.instrument)
possible_apertures = list(siaf.apertures)
for aperture in possible_apertures:
logging.info('\nWorking on aperture {} in {}'.format(
aperture, instrument))
self.aperture = aperture
# Locate the record of the most recent MAST search; use this time
# (plus a buffer to catch any missing files from the previous
# run) as the start time in the new MAST search.
most_recent_search = self.most_recent_search()
self.query_start = most_recent_search - 70
# Query MAST for new dark files for this instrument/aperture
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
new_entries = monitor_utils.mast_query_darks(
instrument, aperture, self.query_start, self.query_end)
# Exclude ASIC tuning data
len_new_darks = len(new_entries)
new_entries = monitor_utils.exclude_asic_tuning(new_entries)
len_no_asic = len(new_entries)
num_asic = len_new_darks - len_no_asic
logging.info(
"\tFiltering out ASIC tuning files removed {} dark files.".
format(num_asic))
logging.info('\tAperture: {}, new entries: {}'.format(
self.aperture, len(new_entries)))
# Set up a directory to store the data for this aperture
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
if len(new_entries) > 0:
ensure_dir_exists(self.data_dir)
# Get any new files to process
new_files = []
checked_files = []
for file_entry in new_entries:
output_filename = os.path.join(
self.data_dir,
file_entry['filename'].replace('_dark', '_uncal'))
# Sometimes both the dark and uncal name of a file is picked up in new_entries
if output_filename in checked_files:
logging.info(
'\t{} already checked in this run.'.format(
output_filename))
continue
checked_files.append(output_filename)
# Dont process files that already exist in the readnoise stats database
file_exists = self.file_exists_in_database(output_filename)
if file_exists:
logging.info(
'\t{} already exists in the readnoise database table.'
.format(output_filename))
continue
# Save any new uncal files with enough groups in the output directory; some dont exist in JWQL filesystem
try:
filename = filesystem_path(file_entry['filename'])
uncal_filename = filename.replace('_dark', '_uncal')
if not os.path.isfile(uncal_filename):
logging.info(
'\t{} does not exist in JWQL filesystem, even though {} does'
.format(uncal_filename, filename))
else:
num_groups = fits.getheader(
uncal_filename)['NGROUPS']
num_ints = fits.getheader(uncal_filename)['NINTS']
if instrument == 'miri':
total_cds_frames = int(
(num_groups - 6) / 2) * num_ints
else:
total_cds_frames = int(
num_groups / 2) * num_ints
# Skip processing if the file doesnt have enough groups/ints to calculate the readnoise.
# MIRI needs extra since they omit the first five and last group before calculating the readnoise.
if total_cds_frames >= 10:
shutil.copy(uncal_filename, self.data_dir)
logging.info('\tCopied {} to {}'.format(
uncal_filename, output_filename))
set_permissions(output_filename)
new_files.append(output_filename)
else:
logging.info(
'\tNot enough groups/ints to calculate readnoise in {}'
.format(uncal_filename))
except FileNotFoundError:
logging.info(
'\t{} does not exist in JWQL filesystem'.format(
file_entry['filename']))
# Run the readnoise monitor on any new files
if len(new_files) > 0:
self.process(new_files)
monitor_run = True
else:
logging.info(
'\tReadnoise monitor skipped. {} new dark files for {}, {}.'
.format(len(new_files), instrument, aperture))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'entries_found': len(new_entries),
'files_found': len(new_files),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Readnoise Monitor completed successfully.')
def run(self):
"""The main method. See module docstrings for further details.
There are 2 parts to the bad pixel monitor:
1. Bad pixels from illuminated data
2. Bad pixels from dark data
For each, we will query MAST, copy new files from the filesystem
and pass the list of copied files into the ``process()`` method.
"""
logging.info('Begin logging for bad_pixel_monitor')
# Get the output directory
self.output_dir = os.path.join(get_config()['outputs'],
'bad_pixel_monitor')
# Read in config file that defines the thresholds for the number
# of dark files that must be present in order for the monitor to run
limits = ascii.read(THRESHOLDS_FILE)
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in JWST_INSTRUMENT_NAMES:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible apertures from pysiaf
possible_apertures = self.get_possible_apertures()
for aperture in possible_apertures:
grating = None
detector_name = None
lamp = None
# NIRSpec flats use the MIRROR grating.
if self.instrument == 'nirspec':
grating = 'MIRROR'
# MIRI is unlike the other instruments. We basically treat
# the detector as the aperture name because there is no
# aperture name for a full frame MRS exposure.
if self.instrument == 'miri':
detector_name, aperture_name = aperture
self.aperture = detector_name
else:
self.aperture = aperture
aperture_name = aperture
# In flight, NIRISS plans to take darks using the LINE2 lamp
if self.instrument == 'niriss':
lamp = 'LINE2'
# What lamp is most appropriate for NIRSpec?
if self.instrument == 'nirspec':
lamp = 'LINE2'
# What lamp is most appropriate for FGS?
# if self.instrument == 'fgs':
# lamp = 'G2LAMP1'
logging.info('')
logging.info('Working on aperture {} in {}'.format(
aperture, self.instrument))
# Find the appropriate threshold for number of new files needed
match = self.aperture == limits['Aperture']
flat_file_count_threshold = limits['FlatThreshold'][
match].data[0]
dark_file_count_threshold = limits['DarkThreshold'][
match].data[0]
# Locate the record of the most recent MAST search
self.flat_query_start = self.most_recent_search(
file_type='flat')
self.dark_query_start = self.most_recent_search(
file_type='dark')
logging.info('\tFlat field query times: {} {}'.format(
self.flat_query_start, self.query_end))
logging.info('\tDark current query times: {} {}'.format(
self.dark_query_start, self.query_end))
# Query MAST using the aperture and the time of the most
# recent previous search as the starting time.
flat_templates = FLAT_EXP_TYPES[instrument]
dark_templates = DARK_EXP_TYPES[instrument]
new_flat_entries = mast_query(instrument,
flat_templates,
self.flat_query_start,
self.query_end,
aperture=aperture_name,
grating=grating,
detector=detector_name,
lamp=lamp)
new_dark_entries = mast_query(instrument,
dark_templates,
self.dark_query_start,
self.query_end,
aperture=aperture_name,
detector=detector_name)
# Filter the results
# Filtering could be different for flats vs darks.
# Kevin says we shouldn't need to worry about mixing lamps in
# the data used to create the bad pixel mask.
# In flight, data will only be taken with LINE2, LEVEL 5.
# Currently in MAST all lamps are present, but Kevin is
# not concerned about variations in flat field strucutre.
# NIRISS - results can include rate, rateints, trapsfilled
# MIRI - Jane says they now use illuminated data for dead pixel checks, just like other insts.
# NIRSpec - can be cal, x1d, rate, rateints. Can have both cal and x1d so filter repeats
# FGS - rate, rateints, trapsfilled
# NIRCam - no int flats
# The query results can contain multiple entries for files
# in different calibration states (or for different output
# products), so we need to filter the list for duplicate
# entries and for the calibration state we are interested
# in before we know how many new entries there really are.
# In the end, we need rate files as well as uncal files
# because we're going to need to create jump files.
# In order to use a given file we must have at least the
# uncal version of the file. Get the uncal and rate file
# lists to align.
if new_flat_entries:
# Exclude ASIC tuning data
len_new_flats = len(new_flat_entries)
new_flat_entries = monitor_utils.exclude_asic_tuning(
new_flat_entries)
len_no_asic = len(new_flat_entries)
num_asic = len_new_flats - len_no_asic
logging.info(
"\tFiltering out ASIC tuning files removed {} flat files."
.format(num_asic))
new_flat_entries = self.filter_query_results(
new_flat_entries, datatype='flat')
apcheck_flat_entries = pipeline_tools.aperture_size_check(
new_flat_entries, instrument, aperture)
lost_to_bad_metadata = len(new_flat_entries) - len(
apcheck_flat_entries)
logging.info(
'\t{} flat field files ignored due to inconsistency in array size and metadata.'
.format(lost_to_bad_metadata))
flat_uncal_files = locate_uncal_files(apcheck_flat_entries)
flat_uncal_files, run_flats = check_for_sufficient_files(
flat_uncal_files, instrument, aperture,
flat_file_count_threshold, 'flats')
flat_rate_files, flat_rate_files_to_copy = locate_rate_files(
flat_uncal_files)
else:
run_flats = False
flat_uncal_files, flat_rate_files, flat_rate_files_to_copy = None, None, None
if new_dark_entries:
# Exclude ASIC tuning data
len_new_darks = len(new_dark_entries)
new_dark_entries = monitor_utils.exclude_asic_tuning(
new_dark_entries)
len_no_asic = len(new_dark_entries)
num_asic = len_new_darks - len_no_asic
logging.info(
"\tFiltering out ASIC tuning files removed {} dark files."
.format(num_asic))
new_dark_entries = self.filter_query_results(
new_dark_entries, datatype='dark')
apcheck_dark_entries = pipeline_tools.aperture_size_check(
new_dark_entries, instrument, aperture)
lost_to_bad_metadata = len(new_dark_entries) - len(
apcheck_dark_entries)
logging.info(
'\t{} dark files ignored due to inconsistency in array size and metadata.'
.format(lost_to_bad_metadata))
dark_uncal_files = locate_uncal_files(apcheck_dark_entries)
dark_uncal_files, run_darks = check_for_sufficient_files(
dark_uncal_files, instrument, aperture,
dark_file_count_threshold, 'darks')
dark_rate_files, dark_rate_files_to_copy = locate_rate_files(
dark_uncal_files)
else:
run_darks = False
dark_uncal_files, dark_rate_files, dark_rate_files_to_copy = None, None, None
# Set up directories for the copied data
ensure_dir_exists(os.path.join(self.output_dir, 'data'))
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
ensure_dir_exists(self.data_dir)
# Copy files from filesystem
if run_flats:
flat_uncal_files, flat_rate_files = self.map_uncal_and_rate_file_lists(
flat_uncal_files, flat_rate_files,
flat_rate_files_to_copy, 'flat')
if run_darks:
dark_uncal_files, dark_rate_files = self.map_uncal_and_rate_file_lists(
dark_uncal_files, dark_rate_files,
dark_rate_files_to_copy, 'dark')
# Run the bad pixel monitor
if run_flats or run_darks:
self.process(flat_uncal_files, flat_rate_files,
dark_uncal_files, dark_rate_files)
# Update the query history
if dark_uncal_files is None:
num_dark_files = 0
else:
num_dark_files = len(dark_uncal_files)
if flat_uncal_files is None:
num_flat_files = 0
else:
num_flat_files = len(flat_uncal_files)
new_entry = {
'instrument': self.instrument.upper(),
'aperture': self.aperture,
'dark_start_time_mjd': self.dark_query_start,
'dark_end_time_mjd': self.query_end,
'flat_start_time_mjd': self.flat_query_start,
'flat_end_time_mjd': self.query_end,
'dark_files_found': num_dark_files,
'flat_files_found': num_flat_files,
'run_bpix_from_darks': run_darks,
'run_bpix_from_flats': run_flats,
'run_monitor': run_flats or run_darks,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Bad Pixel Monitor completed successfully.')
def run(self):
"""The main method. See module docstrings for further
details.
"""
logging.info('Begin logging for dark_monitor')
apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL']
# Get the output directory
self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor')
# Read in config file that defines the thresholds for the number
# of dark files that must be present in order for the monitor to run
limits = ascii.read(THRESHOLDS_FILE)
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in JWST_INSTRUMENT_NAMES:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible apertures from pysiaf
possible_apertures = list(Siaf(instrument).apernames)
possible_apertures = [
ap for ap in possible_apertures if ap not in apertures_to_skip
]
# Get a list of all possible readout patterns associated with the aperture
possible_readpatts = RAPID_READPATTERNS[instrument]
for aperture in possible_apertures:
logging.info('')
logging.info('Working on aperture {} in {}'.format(
aperture, instrument))
# Find appropriate threshold for the number of new files needed
match = aperture == limits['Aperture']
# If the aperture is not listed in the threshold file, we need
# a default
if not np.any(match):
file_count_threshold = 30
logging.warning((
'\tAperture {} is not present in the threshold file. Continuing '
'with the default threshold of 30 files.'.format(
aperture)))
else:
file_count_threshold = limits['Threshold'][match][0]
self.aperture = aperture
# We need a separate search for each readout pattern
for readpatt in possible_readpatts:
self.readpatt = readpatt
logging.info('\tWorking on readout pattern: {}'.format(
self.readpatt))
# Locate the record of the most recent MAST search
self.query_start = self.most_recent_search()
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
# Query MAST using the aperture and the time of the
# most recent previous search as the starting time
new_entries = monitor_utils.mast_query_darks(
instrument,
aperture,
self.query_start,
self.query_end,
readpatt=self.readpatt)
# Exclude ASIC tuning data
len_new_darks = len(new_entries)
new_entries = monitor_utils.exclude_asic_tuning(
new_entries)
len_no_asic = len(new_entries)
num_asic = len_new_darks - len_no_asic
logging.info(
"\tFiltering out ASIC tuning files removed {} dark files."
.format(num_asic))
logging.info(
'\tAperture: {}, Readpattern: {}, new entries: {}'.
format(self.aperture, self.readpatt, len(new_entries)))
# Check to see if there are enough new files to meet the
# monitor's signal-to-noise requirements
if len(new_entries) >= file_count_threshold:
logging.info(
'\tMAST query has returned sufficient new dark files for {}, {}, {} to run the dark monitor.'
.format(self.instrument, self.aperture,
self.readpatt))
# Get full paths to the files
new_filenames = []
for file_entry in new_entries:
try:
new_filenames.append(
filesystem_path(file_entry['filename']))
except FileNotFoundError:
logging.warning(
'\t\tUnable to locate {} in filesystem. Not including in processing.'
.format(file_entry['filename']))
# In some (unusual) cases, there are files in MAST with the correct aperture name
# but incorrect array sizes. Make sure that the new files all have the expected
# aperture size
temp_filenames = []
bad_size_filenames = []
expected_ap = Siaf(instrument)[aperture]
expected_xsize = expected_ap.XSciSize
expected_ysize = expected_ap.YSciSize
for new_file in new_filenames:
with fits.open(new_file) as hdulist:
xsize = hdulist[0].header['SUBSIZE1']
ysize = hdulist[0].header['SUBSIZE2']
if xsize == expected_xsize and ysize == expected_ysize:
temp_filenames.append(new_file)
else:
bad_size_filenames.append(new_file)
if len(temp_filenames) != len(new_filenames):
logging.info(
'\tSome files returned by MAST have unexpected aperture sizes. These files will be ignored: '
)
for badfile in bad_size_filenames:
logging.info('\t\t{}'.format(badfile))
new_filenames = deepcopy(temp_filenames)
# If it turns out that the monitor doesn't find enough
# of the files returned by the MAST query to meet the threshold,
# then the monitor will not be run
if len(new_filenames) < file_count_threshold:
logging.info((
"\tFilesystem search for the files identified by MAST has returned {} files. "
"This is less than the required minimum number of files ({}) necessary to run "
"the monitor. Quitting.").format(
len(new_filenames), file_count_threshold))
monitor_run = False
else:
logging.info((
"\tFilesystem search for the files identified by MAST has returned {} files."
).format(len(new_filenames)))
monitor_run = True
if monitor_run:
# Set up directories for the copied data
ensure_dir_exists(
os.path.join(self.output_dir, 'data'))
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
ensure_dir_exists(self.data_dir)
# Copy files from filesystem
dark_files, not_copied = copy_files(
new_filenames, self.data_dir)
logging.info(
'\tNew_filenames: {}'.format(new_filenames))
logging.info('\tData dir: {}'.format(
self.data_dir))
logging.info('\tCopied to working dir: {}'.format(
dark_files))
logging.info('\tNot copied: {}'.format(not_copied))
# Run the dark monitor
self.process(dark_files)
else:
logging.info((
'\tDark monitor skipped. MAST query has returned {} new dark files for '
'{}, {}, {}. {} new files are required to run dark current monitor.'
).format(len(new_entries), instrument, aperture,
self.readpatt, file_count_threshold))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'readpattern': self.readpatt,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'files_found': len(new_entries),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Dark Monitor completed successfully.')