def get_aperture(instrname, apername):
# siaf_path = os.path.join(bundle_dir, 'data', '{}_SIAF.xml'.format(instrname))
# assert os.path.exists(siaf_path), 'no SIAF for {} at {}'.format(instrname, siaf_path)
siaf = Siaf(instrument=instrname)
return siaf[apername]
def run(self):
"""The main method. See module docstrings for further
details.
"""
logging.info('Begin logging for readnoise_monitor\n')
# Get the output directory and setup a directory to store the data
self.output_dir = os.path.join(get_config()['outputs'],
'readnoise_monitor')
ensure_dir_exists(os.path.join(self.output_dir, 'data'))
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in ['nircam', 'niriss']:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible apertures for this instrument
siaf = Siaf(self.instrument)
possible_apertures = list(siaf.apertures)
for aperture in possible_apertures:
logging.info('\nWorking on aperture {} in {}'.format(
aperture, instrument))
self.aperture = aperture
# Locate the record of the most recent MAST search; use this time
# (plus a 30 day buffer to catch any missing files from the previous
# run) as the start time in the new MAST search.
most_recent_search = self.most_recent_search()
self.query_start = most_recent_search - 30
# Query MAST for new dark files for this instrument/aperture
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
new_entries = mast_query_darks(instrument, aperture,
self.query_start,
self.query_end)
logging.info('\tAperture: {}, new entries: {}'.format(
self.aperture, len(new_entries)))
# Set up a directory to store the data for this aperture
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
if len(new_entries) > 0:
ensure_dir_exists(self.data_dir)
# Get any new files to process
new_files = []
checked_files = []
for file_entry in new_entries:
output_filename = os.path.join(
self.data_dir,
file_entry['filename'].replace('_dark', '_uncal'))
# Sometimes both the dark and uncal name of a file is picked up in new_entries
if output_filename in checked_files:
logging.info(
'\t{} already checked in this run.'.format(
output_filename))
continue
checked_files.append(output_filename)
# Dont process files that already exist in the readnoise stats database
file_exists = self.file_exists_in_database(output_filename)
if file_exists:
logging.info(
'\t{} already exists in the readnoise database table.'
.format(output_filename))
continue
# Save any new uncal files with enough groups in the output directory; some dont exist in JWQL filesystem
try:
filename = filesystem_path(file_entry['filename'])
uncal_filename = filename.replace('_dark', '_uncal')
if not os.path.isfile(uncal_filename):
logging.info(
'\t{} does not exist in JWQL filesystem, even though {} does'
.format(uncal_filename, filename))
else:
num_groups = fits.getheader(
uncal_filename)['NGROUPS']
if num_groups > 1: # skip processing if the file doesnt have enough groups to calculate the readnoise; TODO change to 10 before incorporating MIRI
shutil.copy(uncal_filename, self.data_dir)
logging.info('\tCopied {} to {}'.format(
uncal_filename, output_filename))
set_permissions(output_filename)
new_files.append(output_filename)
else:
logging.info(
'\tNot enough groups to calculate readnoise in {}'
.format(uncal_filename))
except FileNotFoundError:
logging.info(
'\t{} does not exist in JWQL filesystem'.format(
file_entry['filename']))
# Run the readnoise monitor on any new files
if len(new_files) > 0:
self.process(new_files)
monitor_run = True
else:
logging.info(
'\tReadnoise monitor skipped. {} new dark files for {}, {}.'
.format(len(new_files), instrument, aperture))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'entries_found': len(new_entries),
'files_found': len(new_files),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Readnoise Monitor completed successfully.')
def compute_v2v3_offset(aperture_a, aperture_b):
'''
For the same pixel coordinates, different V2, V3 coordinates are used
depending on whether the coronagraph pupil wheel wedge is in the beam.
The offset is computed by transforming the same pixel (Det) coordinates
to V2, V3 in two different apertures and computing the difference in
the resulting Tel frame coordinates
'''
x_a, y_a = aperture_a.det_to_tel(aperture_b.XDetRef, aperture_b.YDetRef)
x_b, y_b = aperture_b.det_to_tel(aperture_b.XDetRef, aperture_b.YDetRef)
return x_a - x_b, y_a - y_b
_NIRCAM_SIAF = Siaf('NIRCam')
_MIRI_SIAF = Siaf('MIRI')
_NIRCAM_CORON_OFFSET_TEL = compute_v2v3_offset(
_NIRCAM_SIAF['NRCA5_MASKLWB'],
_NIRCAM_SIAF['NRCA5_FULL']
)
# These bad areas were defined in raw detector pixel coordinates by
# John Stansberry using a backlit image of NIRCam A5 through the
# long-wavelength bar coronagraph pupil wedge. Colin Cox translated them to
# V2, V3 for A5
NIRCAM_CORON_BAD_AREAS = np.array(
[[[ 56.525, -462.2092],
[ 56.5505, -456.9293],
def run(self):
"""The main method. See module docstrings for further details."""
logging.info('Begin logging for bias_monitor')
# Get the output directory and setup a directory to store the data
self.output_dir = os.path.join(get_config()['outputs'], 'bias_monitor')
ensure_dir_exists(os.path.join(self.output_dir, 'data'))
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in ['nircam', 'niriss', 'nirspec']:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible full-frame apertures for this instrument
siaf = Siaf(self.instrument)
possible_apertures = [
aperture for aperture in siaf.apertures
if siaf[aperture].AperType == 'FULLSCA'
]
for aperture in possible_apertures:
logging.info('Working on aperture {} in {}'.format(
aperture, instrument))
self.aperture = aperture
# Locate the record of most recent MAST search; use this time
# (plus a 30 day buffer to catch any missing files from
# previous run) as the start time in the new MAST search.
most_recent_search = self.most_recent_search()
self.query_start = most_recent_search - 30
# Query MAST for new dark files for this instrument/aperture
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
new_entries = monitor_utils.mast_query_darks(
instrument, aperture, self.query_start, self.query_end)
# Exclude ASIC tuning data
len_new_darks = len(new_entries)
new_entries = monitor_utils.exclude_asic_tuning(new_entries)
len_no_asic = len(new_entries)
num_asic = len_new_darks - len_no_asic
logging.info(
"\tFiltering out ASIC tuning files removed {} dark files.".
format(num_asic))
logging.info('\tAperture: {}, new entries: {}'.format(
self.aperture, len(new_entries)))
# Set up a directory to store the data for this aperture
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
if len(new_entries) > 0:
ensure_dir_exists(self.data_dir)
# Get any new files to process
new_files = []
for file_entry in new_entries:
output_filename = os.path.join(self.data_dir,
file_entry['filename'])
output_filename = output_filename.replace(
'_uncal.fits', '_uncal_0thgroup.fits').replace(
'_dark.fits', '_uncal_0thgroup.fits')
# Dont process files that already exist in the bias stats database
file_exists = self.file_exists_in_database(output_filename)
if file_exists:
logging.info(
'\t{} already exists in the bias database table.'.
format(output_filename))
continue
# Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem.
try:
filename = filesystem_path(file_entry['filename'])
uncal_filename = filename.replace('_dark', '_uncal')
if not os.path.isfile(uncal_filename):
logging.info(
'\t{} does not exist in JWQL filesystem, even though {} does'
.format(uncal_filename, filename))
else:
new_file = self.extract_zeroth_group(
uncal_filename)
new_files.append(new_file)
except FileNotFoundError:
logging.info(
'\t{} does not exist in JWQL filesystem'.format(
file_entry['filename']))
# Run the bias monitor on any new files
if len(new_files) > 0:
self.process(new_files)
monitor_run = True
else:
logging.info(
'\tBias monitor skipped. {} new dark files for {}, {}.'
.format(len(new_files), instrument, aperture))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'entries_found': len(new_entries),
'files_found': len(new_files),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Bias Monitor completed successfully.')
def run(self):
"""The main method. See module docstrings for further
details.
"""
logging.info('Begin logging for dark_monitor')
apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL']
# Get the output directory
self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor')
# Read in config file that defines the thresholds for the number
# of dark files that must be present in order for the monitor to run
limits = ascii.read(THRESHOLDS_FILE)
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in JWST_INSTRUMENT_NAMES:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible apertures from pysiaf
possible_apertures = list(Siaf(instrument).apernames)
possible_apertures = [
ap for ap in possible_apertures if ap not in apertures_to_skip
]
for aperture in possible_apertures:
logging.info('')
logging.info('Working on aperture {} in {}'.format(
aperture, instrument))
# Find the appropriate threshold for the number of new files needed
match = aperture == limits['Aperture']
file_count_threshold = limits['Threshold'][match]
# Locate the record of the most recent MAST search
self.aperture = aperture
self.query_start = self.most_recent_search()
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
# Query MAST using the aperture and the time of the
# most recent previous search as the starting time
new_entries = mast_query_darks(instrument, aperture,
self.query_start,
self.query_end)
logging.info('\tAperture: {}, new entries: {}'.format(
self.aperture, len(new_entries)))
# Check to see if there are enough new files to meet the
# monitor's signal-to-noise requirements
if len(new_entries) >= file_count_threshold:
logging.info(
'\tSufficient new dark files found for {}, {} to run the dark monitor.'
.format(self.instrument, self.aperture))
# Get full paths to the files
new_filenames = []
for file_entry in new_entries:
try:
new_filenames.append(
filesystem_path(file_entry['filename']))
except FileNotFoundError:
logging.warning(
'\t\tUnable to locate {} in filesystem. Not including in processing.'
.format(file_entry['filename']))
# Set up directories for the copied data
ensure_dir_exists(os.path.join(self.output_dir, 'data'))
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
ensure_dir_exists(self.data_dir)
# Copy files from filesystem
dark_files, not_copied = copy_files(
new_filenames, self.data_dir)
logging.info('\tNew_filenames: {}'.format(new_filenames))
logging.info('\tData dir: {}'.format(self.data_dir))
logging.info(
'\tCopied to working dir: {}'.format(dark_files))
logging.info('\tNot copied: {}'.format(not_copied))
# Run the dark monitor
self.process(dark_files)
monitor_run = True
else:
logging.info((
'\tDark monitor skipped. {} new dark files for {}, {}. {} new files are '
'required to run dark current monitor.').format(
len(new_entries), instrument, aperture,
file_count_threshold[0]))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'files_found': len(new_entries),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Dark Monitor completed successfully.')
def process(self, file_list):
"""The main method for processing darks. See module docstrings
for further details.
Parameters
----------
file_list : list
List of filenames (including full paths) to the dark current
files
"""
# Basic metadata that will be needed later
self.get_metadata(file_list[0])
# Determine which pipeline steps need to be executed
required_steps = pipeline_tools.get_pipeline_steps(self.instrument)
logging.info(
'\tRequired calwebb1_detector pipeline steps to have the data in the '
'correct format:')
for item in required_steps:
logging.info('\t\t{}: {}'.format(item, required_steps[item]))
# Modify the list of pipeline steps to skip those not needed for the
# preparation of dark current data
required_steps['dark_current'] = False
required_steps['persistence'] = False
# NIRSpec IR^2 readout pattern NRSIRS2 is the only one with
# nframes not a power of 2
if self.read_pattern not in pipeline_tools.GROUPSCALE_READOUT_PATTERNS:
required_steps['group_scale'] = False
# Run pipeline steps on files, generating slope files
slope_files = []
for filename in file_list:
completed_steps = pipeline_tools.completed_pipeline_steps(filename)
steps_to_run = pipeline_tools.steps_to_run(required_steps,
completed_steps)
logging.info('\tWorking on file: {}'.format(filename))
logging.info('\tPipeline steps that remain to be run:')
for item in steps_to_run:
logging.info('\t\t{}: {}'.format(item, steps_to_run[item]))
# Run any remaining required pipeline steps
if any(steps_to_run.values()) is False:
slope_files.append(filename)
else:
processed_file = filename.replace('.fits',
'_{}.fits'.format('rate'))
# If the slope file already exists, skip the pipeline call
if not os.path.isfile(processed_file):
logging.info('\tRunning pipeline on {}'.format(filename))
processed_file = pipeline_tools.run_calwebb_detector1_steps(
os.path.abspath(filename), steps_to_run)
logging.info('\tPipeline complete. Output: {}'.format(
processed_file))
else:
logging.info(
'\tSlope file {} already exists. Skipping call to pipeline.'
.format(processed_file))
pass
slope_files.append(processed_file)
# Delete the original dark ramp file to save disk space
os.remove(filename)
obs_times = []
logging.info(
'\tSlope images to use in the dark monitor for {}, {}:'.format(
self.instrument, self.aperture))
for item in slope_files:
logging.info('\t\t{}'.format(item))
# Get the observation time for each file
obstime = instrument_properties.get_obstime(item)
obs_times.append(obstime)
# Find the earliest and latest observation time, and calculate
# the mid-time.
min_time = np.min(obs_times)
max_time = np.max(obs_times)
mid_time = instrument_properties.mean_time(obs_times)
# Read in all slope images and place into a list
slope_image_stack, slope_exptimes = pipeline_tools.image_stack(
slope_files)
# Calculate a mean slope image from the inputs
slope_image, stdev_image = calculations.mean_image(slope_image_stack,
sigma_threshold=3)
mean_slope_file = self.save_mean_slope_image(slope_image, stdev_image,
slope_files)
logging.info(
'\tSigma-clipped mean of the slope images saved to: {}'.format(
mean_slope_file))
# ----- Search for new hot/dead/noisy pixels -----
# Read in baseline mean slope image and stdev image
# The baseline image is used to look for hot/dead/noisy pixels,
# but not for comparing mean dark rates. Therefore, updates to
# the baseline can be minimal.
# Limit checks for hot/dead/noisy pixels to full frame data since
# subarray data have much shorter exposure times and therefore lower
# signal-to-noise
aperture_type = Siaf(self.instrument)[self.aperture].AperType
if aperture_type == 'FULLSCA':
baseline_file = self.get_baseline_filename()
if baseline_file is None:
logging.warning((
'\tNo baseline dark current countrate image for {} {}. Setting the '
'current mean slope image to be the new baseline.'.format(
self.instrument, self.aperture)))
baseline_file = mean_slope_file
baseline_mean = deepcopy(slope_image)
baseline_stdev = deepcopy(stdev_image)
else:
logging.info('\tBaseline file is {}'.format(baseline_file))
baseline_mean, baseline_stdev = self.read_baseline_slope_image(
baseline_file)
# Check the hot/dead pixel population for changes
new_hot_pix, new_dead_pix = self.find_hot_dead_pixels(
slope_image, baseline_mean)
# Shift the coordinates to be in full frame coordinate system
new_hot_pix = self.shift_to_full_frame(new_hot_pix)
new_dead_pix = self.shift_to_full_frame(new_dead_pix)
# Exclude hot and dead pixels found previously
new_hot_pix = self.exclude_existing_badpix(new_hot_pix, 'hot')
new_dead_pix = self.exclude_existing_badpix(new_dead_pix, 'dead')
# Add new hot and dead pixels to the database
logging.info('\tFound {} new hot pixels'.format(len(
new_hot_pix[0])))
logging.info('\tFound {} new dead pixels'.format(
len(new_dead_pix[0])))
self.add_bad_pix(new_hot_pix, 'hot', file_list, mean_slope_file,
baseline_file, min_time, mid_time, max_time)
self.add_bad_pix(new_dead_pix, 'dead', file_list, mean_slope_file,
baseline_file, min_time, mid_time, max_time)
# Check for any pixels that are significantly more noisy than
# in the baseline stdev image
new_noisy_pixels = self.noise_check(stdev_image, baseline_stdev)
# Shift coordinates to be in full_frame coordinate system
new_noisy_pixels = self.shift_to_full_frame(new_noisy_pixels)
# Exclude previously found noisy pixels
new_noisy_pixels = self.exclude_existing_badpix(
new_noisy_pixels, 'noisy')
# Add new noisy pixels to the database
logging.info('\tFound {} new noisy pixels'.format(
len(new_noisy_pixels[0])))
self.add_bad_pix(new_noisy_pixels, 'noisy', file_list,
mean_slope_file, baseline_file, min_time,
mid_time, max_time)
# ----- Calculate image statistics -----
# Find amplifier boundaries so per-amp statistics can be calculated
number_of_amps, amp_bounds = instrument_properties.amplifier_info(
slope_files[0])
logging.info('\tAmplifier boundaries: {}'.format(amp_bounds))
# Calculate mean and stdev values, and fit a Gaussian to the
# histogram of the pixels in each amp
(amp_mean, amp_stdev, gauss_param, gauss_chisquared,
double_gauss_params, double_gauss_chisquared, histogram,
bins) = self.stats_by_amp(slope_image, amp_bounds)
# Construct new entry for dark database table
source_files = [os.path.basename(item) for item in file_list]
for key in amp_mean.keys():
dark_db_entry = {
'aperture': self.aperture,
'amplifier': key,
'mean': amp_mean[key],
'stdev': amp_stdev[key],
'source_files': source_files,
'obs_start_time': min_time,
'obs_mid_time': mid_time,
'obs_end_time': max_time,
'gauss_amplitude': list(gauss_param[key][0]),
'gauss_peak': list(gauss_param[key][1]),
'gauss_width': list(gauss_param[key][2]),
'gauss_chisq': gauss_chisquared[key],
'double_gauss_amplitude1': double_gauss_params[key][0],
'double_gauss_peak1': double_gauss_params[key][1],
'double_gauss_width1': double_gauss_params[key][2],
'double_gauss_amplitude2': double_gauss_params[key][3],
'double_gauss_peak2': double_gauss_params[key][4],
'double_gauss_width2': double_gauss_params[key][5],
'double_gauss_chisq': double_gauss_chisquared[key],
'mean_dark_image_file': os.path.basename(mean_slope_file),
'hist_dark_values': bins,
'hist_amplitudes': histogram,
'entry_date': datetime.datetime.now()
}
self.stats_table.__table__.insert().execute(dark_db_entry)
"""Telescope coordinate information"""
import numpy as np
import logging
_log = logging.getLogger('pynrc')
__epsilon = np.finfo(float).eps
import pysiaf
from pysiaf import JWST_PRD_VERSION, rotations, Siaf
# Create this once since it takes time to call multiple times
siaf_nrc = Siaf('NIRCam')
siaf_nrc.generate_toc()
# Functions transferred to webbpsf_ext
from webbpsf_ext.coords import dist_image
from webbpsf_ext.coords import xy_to_rtheta, rtheta_to_xy, xy_rot
from webbpsf_ext.coords import ap_radec, radec_to_v2v3, v2v3_to_pixel
from webbpsf_ext.coords import get_NRC_v2v3_limits, NIRCam_V2V3_limits
from webbpsf_ext.coords import get_NRC_v2v3_limits as get_v2v3_limits # Original name
from webbpsf_ext.coords import plotAxes
# New stuff from webbpsf_ext
from webbpsf_ext.coords import gen_sgd_offsets, get_idl_offset, radec_offset
from webbpsf_ext.coords import jwst_point
###########################################################################
#
# NIRCam Coordinate Systems
#
###########################################################################
def run(self):
"""The main method. See module docstrings for further
details.
"""
logging.info('Begin logging for dark_monitor')
apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL']
# Get the output directory
self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor')
# Read in config file that defines the thresholds for the number
# of dark files that must be present in order for the monitor to run
limits = ascii.read(THRESHOLDS_FILE)
# Use the current time as the end time for MAST query
self.query_end = Time.now().mjd
# Loop over all instruments
for instrument in JWST_INSTRUMENT_NAMES:
self.instrument = instrument
# Identify which database tables to use
self.identify_tables()
# Get a list of all possible apertures from pysiaf
possible_apertures = list(Siaf(instrument).apernames)
possible_apertures = [
ap for ap in possible_apertures if ap not in apertures_to_skip
]
# Get a list of all possible readout patterns associated with the aperture
possible_readpatts = RAPID_READPATTERNS[instrument]
for aperture in possible_apertures:
logging.info('')
logging.info('Working on aperture {} in {}'.format(
aperture, instrument))
# Find appropriate threshold for the number of new files needed
match = aperture == limits['Aperture']
# If the aperture is not listed in the threshold file, we need
# a default
if not np.any(match):
file_count_threshold = 30
logging.warning((
'\tAperture {} is not present in the threshold file. Continuing '
'with the default threshold of 30 files.'.format(
aperture)))
else:
file_count_threshold = limits['Threshold'][match][0]
self.aperture = aperture
# We need a separate search for each readout pattern
for readpatt in possible_readpatts:
self.readpatt = readpatt
logging.info('\tWorking on readout pattern: {}'.format(
self.readpatt))
# Locate the record of the most recent MAST search
self.query_start = self.most_recent_search()
logging.info('\tQuery times: {} {}'.format(
self.query_start, self.query_end))
# Query MAST using the aperture and the time of the
# most recent previous search as the starting time
new_entries = mast_query_darks(instrument,
aperture,
self.query_start,
self.query_end,
readpatt=self.readpatt)
logging.info(
'\tAperture: {}, Readpattern: {}, new entries: {}'.
format(self.aperture, self.readpatt, len(new_entries)))
# Check to see if there are enough new files to meet the
# monitor's signal-to-noise requirements
if len(new_entries) >= file_count_threshold:
logging.info(
'\tMAST query has returned sufficient new dark files for {}, {}, {} to run the dark monitor.'
.format(self.instrument, self.aperture,
self.readpatt))
# Get full paths to the files
new_filenames = []
for file_entry in new_entries:
try:
new_filenames.append(
filesystem_path(file_entry['filename']))
except FileNotFoundError:
logging.warning(
'\t\tUnable to locate {} in filesystem. Not including in processing.'
.format(file_entry['filename']))
# In some (unusual) cases, there are files in MAST with the correct aperture name
# but incorrect array sizes. Make sure that the new files all have the expected
# aperture size
temp_filenames = []
bad_size_filenames = []
expected_ap = Siaf(instrument)[aperture]
expected_xsize = expected_ap.XSciSize
expected_ysize = expected_ap.YSciSize
for new_file in new_filenames:
with fits.open(new_file) as hdulist:
xsize = hdulist[0].header['SUBSIZE1']
ysize = hdulist[0].header['SUBSIZE2']
if xsize == expected_xsize and ysize == expected_ysize:
temp_filenames.append(new_file)
else:
bad_size_filenames.append(new_file)
if len(temp_filenames) != len(new_filenames):
logging.info(
'\tSome files returned by MAST have unexpected aperture sizes. These files will be ignored: '
)
for badfile in bad_size_filenames:
logging.info('\t\t{}'.format(badfile))
new_filenames = deepcopy(temp_filenames)
# If it turns out that the monitor doesn't find enough
# of the files returned by the MAST query to meet the threshold,
# then the monitor will not be run
if len(new_filenames) < file_count_threshold:
logging.info((
"\tFilesystem search for the files identified by MAST has returned {} files. "
"This is less than the required minimum number of files ({}) necessary to run "
"the monitor. Quitting.").format(
len(new_filenames), file_count_threshold))
monitor_run = False
else:
logging.info((
"\tFilesystem search for the files identified by MAST has returned {} files."
).format(len(new_filenames)))
monitor_run = True
if monitor_run:
# Set up directories for the copied data
ensure_dir_exists(
os.path.join(self.output_dir, 'data'))
self.data_dir = os.path.join(
self.output_dir,
'data/{}_{}'.format(self.instrument.lower(),
self.aperture.lower()))
ensure_dir_exists(self.data_dir)
# Copy files from filesystem
dark_files, not_copied = copy_files(
new_filenames, self.data_dir)
logging.info(
'\tNew_filenames: {}'.format(new_filenames))
logging.info('\tData dir: {}'.format(
self.data_dir))
logging.info('\tCopied to working dir: {}'.format(
dark_files))
logging.info('\tNot copied: {}'.format(not_copied))
# Run the dark monitor
self.process(dark_files)
else:
logging.info((
'\tDark monitor skipped. MAST query has returned {} new dark files for '
'{}, {}, {}. {} new files are required to run dark current monitor.'
).format(len(new_entries), instrument, aperture,
self.readpatt, file_count_threshold))
monitor_run = False
# Update the query history
new_entry = {
'instrument': instrument,
'aperture': aperture,
'readpattern': self.readpatt,
'start_time_mjd': self.query_start,
'end_time_mjd': self.query_end,
'files_found': len(new_entries),
'run_monitor': monitor_run,
'entry_date': datetime.datetime.now()
}
self.query_table.__table__.insert().execute(new_entry)
logging.info('\tUpdated the query history table')
logging.info('Dark Monitor completed successfully.')