def get_baseline_filename(self):
"""Query the database and return the filename of the baseline
(comparison) mean dark slope image to use when searching for
new hot/dead/noisy pixels. For this we assume that the most
recent baseline file for the given detector is the one to use.
Returns
-------
filename : str
Name of fits file containing the baseline image
"""
subq = session.query(
self.pixel_table.detector,
func.max(self.pixel_table.entry_date).label('maxdate')).group_by(
self.pixel_table.detector).subquery('t2')
query = session.query(self.pixel_table).join(
subq,
and_(self.pixel_table.detector == self.detector,
self.pixel_table.entry_date == subq.c.maxdate))
count = query.count()
if not count:
filename = None
else:
filename = query.all()[0].baseline_file
# Specify the full path
filename = os.path.join(self.output_dir, 'mean_slope_images',
filename)
logging.info('Baseline filename: {}'.format(filename))
return filename
def plot_central_store_dirs():
"""Plot central store sizes (size, used, available) versus date
Returns
-------
plot : bokeh.plotting.figure.Figure object
``bokeh`` plot of total directory size versus date
"""
# Plot system stats vs. date
results = session.query(CentralStore.date, CentralStore.size,
CentralStore.available).all()
arealist = [
'logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all'
]
# Initialize plot
dates, total_sizes, availables = zip(*results)
plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
x_axis_type='datetime',
title='Central Store stats',
x_axis_label='Date',
y_axis_label='Size TB')
colors = itertools.cycle(palette)
plot.line(dates, total_sizes, legend='Total size', line_color='red')
plot.circle(dates, total_sizes, color='red')
plot.line(dates, availables, legend='Free', line_color='blue')
plot.circle(dates, availables, color='blue')
# This part of the plot should cycle through areas and plot area used values vs. date
for area, color in zip(arealist, colors):
# Query for used sizes
results = session.query(
CentralStore.date,
CentralStore.used).filter(CentralStore.area == area)
# Group by date
if results:
results_dict = defaultdict(int)
for date, value in results:
results_dict[date] += value
# Parse results so they can be easily plotted
dates = list(results_dict.keys())
values = list(results_dict.values())
# Plot the results
plot.line(dates,
values,
legend='{} files'.format(area),
line_color=color)
plot.circle(dates, values, color=color)
session.close()
return plot
def most_recent_search(self, file_type='dark'):
"""Query the query history database and return the information
on the most recent query for the given ``aperture_name`` where
the dark monitor was executed.
Parameters
----------
file_type : str
``dark`` or ``flat``. Specifies the type of file whose
previous search time is queried.
Returns
-------
query_result : float
Date (in MJD) of the ending range of the previous MAST query
where the dark monitor was run.
"""
if file_type.lower() == 'dark':
mjd_field = self.query_table.dark_end_time_mjd
elif file_type.lower() == 'flat':
mjd_field = self.query_table.flat_end_time_mjd
sub_query = session.query(self.query_table.aperture,
func.max(mjd_field).label('maxdate')
).group_by(self.query_table.aperture).subquery('t2')
# Note that "self.query_table.run_monitor == True" below is
# intentional. Switching = to "is" results in an error in the query.
query = session.query(self.query_table).join(
sub_query,
and_(
self.query_table.aperture == self.aperture,
mjd_field == sub_query.c.maxdate,
self.query_table.run_monitor == True
)
).all()
query_count = len(query)
if query_count == 0:
query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3
logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'
.format(self.aperture, query_result)))
elif query_count > 1:
raise ValueError('More than one "most recent" query?')
else:
if file_type.lower() == 'dark':
query_result = query[0].dark_end_time_mjd
elif file_type.lower() == 'flat':
query_result = query[0].flat_end_time_mjd
return query_result
def load_data(self):
"""Query the database tables to get data"""
# Determine which database tables are needed based on instrument
self.identify_tables()
# Query database for all data in NIRCamDarkDarkCurrent with a matching aperture
self.dark_table = session.query(self.stats_table) \
.filter(self.stats_table.aperture == self._aperture) \
.all()
self.pixel_table = session.query(self.pixel_table) \
.filter(self.pixel_table.detector == self.detector) \
.all()
def plot_filesystem_size():
"""Plot filesystem sizes (size, used, available) versus date
Returns
-------
plot : bokeh.plotting.figure.Figure object
``bokeh`` plot of total file counts versus date
"""
# Plot system stats vs. date
results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_size,
FilesystemGeneral.used, FilesystemGeneral.available).all()
dates, total_sizes, useds, availables = zip(*results)
plot = figure(
tools='pan,box_zoom,wheel_zoom,reset,save',
x_axis_type='datetime',
title='System stats',
x_axis_label='Date',
y_axis_label='Size TB')
plot.line(dates, total_sizes, legend='Total size', line_color='red')
plot.circle(dates, total_sizes, color='red')
plot.line(dates, useds, legend='Used bytes', line_color='green')
plot.circle(dates, useds, color='green')
plot.line(dates, availables, legend='Free bytes', line_color='blue')
plot.circle(dates, availables, color='blue')
return plot
def most_recent_search(self):
"""Query the query history database and return the information
on the most recent query for the given ``aperture_name`` where
the readnoise monitor was executed.
Returns
-------
query_result : float
Date (in MJD) of the ending range of the previous MAST query
where the readnoise monitor was run.
"""
query = session.query(self.query_table).filter(
and_(self.query_table.aperture == self.aperture,
self.query_table.run_monitor == True)).order_by(
self.query_table.end_time_mjd).all()
if len(query) == 0:
query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3
logging.info((
'\tNo query history for {}. Beginning search date will be set to {}.'
.format(self.aperture, query_result)))
else:
query_result = query[-1].end_time_mjd
return query_result
def file_exists_in_database(self, filename):
"""Checks if an entry for filename exists in the readnoise stats
database.
Parameters
----------
filename : str
The full path to the uncal filename.
Returns
-------
file_exists : bool
``True`` if filename exists in the readnoise stats database.
"""
query = session.query(self.stats_table)
results = query.filter(
self.stats_table.uncal_filename == filename).all()
if len(results) != 0:
file_exists = True
else:
file_exists = False
return file_exists
def most_recent_search(self):
"""Query the query history database and return the information
on the most recent query for the given ``aperture_name`` where
the dark monitor was executed.
Returns
-------
query_result : float
Date (in MJD) of the ending range of the previous MAST query
where the dark monitor was run.
"""
query = session.query(self.query_table).filter(self.query_table.aperture==self.aperture). \
filter(self.query_table.run_monitor==True)
dates = np.zeros(0)
for instance in query:
dates = np.append(dates, instance.end_time_mjd)
query_count = len(dates)
if query_count == 0:
query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3
logging.info((
'\tNo query history for {}. Beginning search date will be set to {}.'
.format(self.aperture, query_result)))
else:
query_result = np.max(dates)
return query_result
def load_data(self):
"""Query the database tables to get data"""
# Determine which database tables are needed based on instrument
self.identify_tables()
# Query database for all data with a matching aperture
self.bad_pixel_table = session.query(self.pixel_table) \
.filter(self.pixel_table.detector == self.detector) \
.all()
def exclude_existing_badpix(self, badpix, pixel_type):
"""Given a set of coordinates of bad pixels, determine which of
these pixels have been previously identified and remove them
from the list
Parameters
----------
badpix : tuple
Tuple of lists containing x and y pixel coordinates. (Output
of ``numpy.where`` call)
pixel_type : str
Type of bad pixel being examined. Options are ``hot``,
``dead``, and ``noisy``
Returns
-------
new_pixels_x : list
List of x coordinates of new bad pixels
new_pixels_y : list
List of y coordinates of new bad pixels
"""
if pixel_type not in ['hot', 'dead', 'noisy']:
raise ValueError(
'Unrecognized bad pixel type: {}'.format(pixel_type))
db_entries = session.query(self.pixel_table) \
.filter(self.pixel_table.type == pixel_type) \
.filter(self.pixel_table.detector == self.detector) \
.all()
already_found = []
if len(db_entries) != 0:
for _row in db_entries:
x_coords = _row.x_coord
y_coords = _row.y_coord
for x, y in zip(x_coords, y_coords):
already_found.append((x, y))
# Check to see if each pixel already appears in the database for
# the given bad pixel type
new_pixels_x = []
new_pixels_y = []
for x, y in zip(badpix[0], badpix[1]):
pixel = (x, y)
if pixel not in already_found:
new_pixels_x.append(x)
new_pixels_y.append(y)
session.close()
return (new_pixels_x, new_pixels_y)
def load_data(self):
"""Query the database tables to get all of the relevant bias data"""
# Determine which database tables are needed based on instrument
self.identify_tables()
# Query database for all data in bias stats with a matching aperture,
# and sort the data by exposure start time.
self.query_results = session.query(self.stats_table) \
.filter(self.stats_table.aperture == self._aperture) \
.order_by(self.stats_table.expstart) \
.all()
def most_recent_search(self):
"""Query the query history database and return the information
on the most recent query for the given ``aperture_name`` where
the dark monitor was executed.
Returns
-------
query_result : float
Date (in MJD) of the ending range of the previous MAST query
where the dark monitor was run.
"""
sub_query = session.query(self.query_table.aperture,
func.max(self.query_table.end_time_mjd).label('maxdate')
).group_by(self.query_table.aperture).subquery('t2')
# Note that "self.query_table.run_monitor == True" below is
# intentional. Switching = to "is" results in an error in the query.
query = session.query(self.query_table).join(
sub_query,
and_(
self.query_table.aperture == self.aperture,
self.query_table.end_time_mjd == sub_query.c.maxdate,
self.query_table.run_monitor == True
)
).all()
query_count = len(query)
if query_count == 0:
query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3
logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'
.format(self.aperture, query_result)))
elif query_count > 1:
raise ValueError('More than one "most recent" query?')
else:
query_result = query[0].end_time_mjd
return query_result
def most_recent_search(self, file_type='dark'):
"""Query the query history database and return the information
on the most recent query for the given ``aperture_name`` where
the dark monitor was executed.
Parameters
----------
file_type : str
``dark`` or ``flat``. Specifies the type of file whose
previous search time is queried.
Returns
-------
query_result : float
Date (in MJD) of the ending range of the previous MAST query
where the dark monitor was run.
"""
if file_type.lower() == 'dark':
mjd_field = self.query_table.dark_end_time_mjd
run_field = self.query_table.run_bpix_from_darks
elif file_type.lower() == 'flat':
mjd_field = self.query_table.flat_end_time_mjd
run_field = self.query_table.run_bpix_from_flats
query = session.query(self.query_table).filter(self.query_table.aperture==self.aperture). \
filter(run_field==True)
dates = np.zeros(0)
if file_type.lower() == 'dark':
for instance in query:
dates = np.append(dates, instance.dark_end_time_mjd)
elif file_type.lower() == 'flat':
for instance in query:
dates = np.append(dates, instance.flat_end_time_mjd)
query_count = len(dates)
if query_count == 0:
query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3
logging.info((
'\tNo query history for {}. Beginning search date will be set to {}.'
.format(self.aperture, query_result)))
else:
query_result = np.max(dates)
return query_result
def plot_total_file_counts():
"""Plot total file counts versus date
Returns
-------
plot : bokeh.plotting.figure.Figure object
``bokeh`` plot of total file counts versus date
"""
# Total file counts vs. date
results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_count).all()
dates, file_counts = zip(*results)
plot = figure(
tools='pan,box_zoom,reset,wheel_zoom,save',
x_axis_type='datetime',
title="Total File Counts",
x_axis_label='Date',
y_axis_label='Count')
plot.line(dates, file_counts, line_width=2, line_color='blue')
plot.circle(dates, file_counts, color='blue')
return plot
def plot_by_filetype(plot_type, instrument):
"""Plot ``count`` or ``size`` by filetype versus date for the given
instrument, or all instruments.
Parameters
----------
plot_type : str
Which data to plot. Either ``count`` or ``size``.
instrument : str
The instrument to plot for. Can be a valid JWST instrument or
``all`` to plot across all instruments.
Returns
-------
plot : bokeh.plotting.figure.Figure object
``bokeh`` plot of total file counts versus date
"""
# Determine plot title
if instrument == 'all':
title = 'Total File {} by Type'.format(plot_type.capitalize())
else:
instrument_title = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument]
title = '{} Total File {} by Type'.format(instrument_title, plot_type.capitalize())
if plot_type == 'count':
ytitle = 'Counts'
else:
ytitle = 'Size (TB)'
# Initialize plot
plot = figure(
tools='pan,box_zoom,wheel_zoom,reset,save',
x_axis_type='datetime',
title=title,
x_axis_label='Date',
y_axis_label=ytitle)
colors = itertools.cycle(palette)
for filetype, color in zip(FILE_SUFFIX_TYPES, colors):
# Query for counts
results = session.query(FilesystemInstrument.date, getattr(FilesystemInstrument, plot_type))\
.filter(FilesystemInstrument.filetype == filetype)
if instrument == 'all':
results = results.all()
else:
results = results.filter(FilesystemInstrument.instrument == instrument).all()
# Group by date
if results:
results_dict = defaultdict(int)
for date, value in results:
results_dict[date] += value
# Parse results so they can be easily plotted
dates = list(results_dict.keys())
values = list(results_dict.values())
# Plot the results
plot.line(dates, values, legend='{} files'.format(filetype), line_color=color)
plot.circle(dates, values, color=color)
return plot
