def get_baseline_filename(self): """Query the database and return the filename of the baseline (comparison) mean dark slope image to use when searching for new hot/dead/noisy pixels. For this we assume that the most recent baseline file for the given detector is the one to use. Returns ------- filename : str Name of fits file containing the baseline image """ subq = session.query( self.pixel_table.detector, func.max(self.pixel_table.entry_date).label('maxdate')).group_by( self.pixel_table.detector).subquery('t2') query = session.query(self.pixel_table).join( subq, and_(self.pixel_table.detector == self.detector, self.pixel_table.entry_date == subq.c.maxdate)) count = query.count() if not count: filename = None else: filename = query.all()[0].baseline_file # Specify the full path filename = os.path.join(self.output_dir, 'mean_slope_images', filename) logging.info('Baseline filename: {}'.format(filename)) return filename
def plot_central_store_dirs(): """Plot central store sizes (size, used, available) versus date Returns ------- plot : bokeh.plotting.figure.Figure object ``bokeh`` plot of total directory size versus date """ # Plot system stats vs. date results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all() arealist = [ 'logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all' ] # Initialize plot dates, total_sizes, availables = zip(*results) plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title='Central Store stats', x_axis_label='Date', y_axis_label='Size TB') colors = itertools.cycle(palette) plot.line(dates, total_sizes, legend='Total size', line_color='red') plot.circle(dates, total_sizes, color='red') plot.line(dates, availables, legend='Free', line_color='blue') plot.circle(dates, availables, color='blue') # This part of the plot should cycle through areas and plot area used values vs. date for area, color in zip(arealist, colors): # Query for used sizes results = session.query( CentralStore.date, CentralStore.used).filter(CentralStore.area == area) # Group by date if results: results_dict = defaultdict(int) for date, value in results: results_dict[date] += value # Parse results so they can be easily plotted dates = list(results_dict.keys()) values = list(results_dict.values()) # Plot the results plot.line(dates, values, legend='{} files'.format(area), line_color=color) plot.circle(dates, values, color=color) session.close() return plot
def most_recent_search(self, file_type='dark'): """Query the query history database and return the information on the most recent query for the given ``aperture_name`` where the dark monitor was executed. Parameters ---------- file_type : str ``dark`` or ``flat``. Specifies the type of file whose previous search time is queried. Returns ------- query_result : float Date (in MJD) of the ending range of the previous MAST query where the dark monitor was run. """ if file_type.lower() == 'dark': mjd_field = self.query_table.dark_end_time_mjd elif file_type.lower() == 'flat': mjd_field = self.query_table.flat_end_time_mjd sub_query = session.query(self.query_table.aperture, func.max(mjd_field).label('maxdate') ).group_by(self.query_table.aperture).subquery('t2') # Note that "self.query_table.run_monitor == True" below is # intentional. Switching = to "is" results in an error in the query. query = session.query(self.query_table).join( sub_query, and_( self.query_table.aperture == self.aperture, mjd_field == sub_query.c.maxdate, self.query_table.run_monitor == True ) ).all() query_count = len(query) if query_count == 0: query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) elif query_count > 1: raise ValueError('More than one "most recent" query?') else: if file_type.lower() == 'dark': query_result = query[0].dark_end_time_mjd elif file_type.lower() == 'flat': query_result = query[0].flat_end_time_mjd return query_result
def load_data(self): """Query the database tables to get data""" # Determine which database tables are needed based on instrument self.identify_tables() # Query database for all data in NIRCamDarkDarkCurrent with a matching aperture self.dark_table = session.query(self.stats_table) \ .filter(self.stats_table.aperture == self._aperture) \ .all() self.pixel_table = session.query(self.pixel_table) \ .filter(self.pixel_table.detector == self.detector) \ .all()
def plot_filesystem_size(): """Plot filesystem sizes (size, used, available) versus date Returns ------- plot : bokeh.plotting.figure.Figure object ``bokeh`` plot of total file counts versus date """ # Plot system stats vs. date results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_size, FilesystemGeneral.used, FilesystemGeneral.available).all() dates, total_sizes, useds, availables = zip(*results) plot = figure( tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title='System stats', x_axis_label='Date', y_axis_label='Size TB') plot.line(dates, total_sizes, legend='Total size', line_color='red') plot.circle(dates, total_sizes, color='red') plot.line(dates, useds, legend='Used bytes', line_color='green') plot.circle(dates, useds, color='green') plot.line(dates, availables, legend='Free bytes', line_color='blue') plot.circle(dates, availables, color='blue') return plot
def most_recent_search(self): """Query the query history database and return the information on the most recent query for the given ``aperture_name`` where the readnoise monitor was executed. Returns ------- query_result : float Date (in MJD) of the ending range of the previous MAST query where the readnoise monitor was run. """ query = session.query(self.query_table).filter( and_(self.query_table.aperture == self.aperture, self.query_table.run_monitor == True)).order_by( self.query_table.end_time_mjd).all() if len(query) == 0: query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(( '\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) else: query_result = query[-1].end_time_mjd return query_result
def file_exists_in_database(self, filename): """Checks if an entry for filename exists in the readnoise stats database. Parameters ---------- filename : str The full path to the uncal filename. Returns ------- file_exists : bool ``True`` if filename exists in the readnoise stats database. """ query = session.query(self.stats_table) results = query.filter( self.stats_table.uncal_filename == filename).all() if len(results) != 0: file_exists = True else: file_exists = False return file_exists
def most_recent_search(self): """Query the query history database and return the information on the most recent query for the given ``aperture_name`` where the dark monitor was executed. Returns ------- query_result : float Date (in MJD) of the ending range of the previous MAST query where the dark monitor was run. """ query = session.query(self.query_table).filter(self.query_table.aperture==self.aperture). \ filter(self.query_table.run_monitor==True) dates = np.zeros(0) for instance in query: dates = np.append(dates, instance.end_time_mjd) query_count = len(dates) if query_count == 0: query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(( '\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) else: query_result = np.max(dates) return query_result
def load_data(self): """Query the database tables to get data""" # Determine which database tables are needed based on instrument self.identify_tables() # Query database for all data with a matching aperture self.bad_pixel_table = session.query(self.pixel_table) \ .filter(self.pixel_table.detector == self.detector) \ .all()
def exclude_existing_badpix(self, badpix, pixel_type): """Given a set of coordinates of bad pixels, determine which of these pixels have been previously identified and remove them from the list Parameters ---------- badpix : tuple Tuple of lists containing x and y pixel coordinates. (Output of ``numpy.where`` call) pixel_type : str Type of bad pixel being examined. Options are ``hot``, ``dead``, and ``noisy`` Returns ------- new_pixels_x : list List of x coordinates of new bad pixels new_pixels_y : list List of y coordinates of new bad pixels """ if pixel_type not in ['hot', 'dead', 'noisy']: raise ValueError( 'Unrecognized bad pixel type: {}'.format(pixel_type)) db_entries = session.query(self.pixel_table) \ .filter(self.pixel_table.type == pixel_type) \ .filter(self.pixel_table.detector == self.detector) \ .all() already_found = [] if len(db_entries) != 0: for _row in db_entries: x_coords = _row.x_coord y_coords = _row.y_coord for x, y in zip(x_coords, y_coords): already_found.append((x, y)) # Check to see if each pixel already appears in the database for # the given bad pixel type new_pixels_x = [] new_pixels_y = [] for x, y in zip(badpix[0], badpix[1]): pixel = (x, y) if pixel not in already_found: new_pixels_x.append(x) new_pixels_y.append(y) session.close() return (new_pixels_x, new_pixels_y)
def load_data(self): """Query the database tables to get all of the relevant bias data""" # Determine which database tables are needed based on instrument self.identify_tables() # Query database for all data in bias stats with a matching aperture, # and sort the data by exposure start time. self.query_results = session.query(self.stats_table) \ .filter(self.stats_table.aperture == self._aperture) \ .order_by(self.stats_table.expstart) \ .all()
def most_recent_search(self): """Query the query history database and return the information on the most recent query for the given ``aperture_name`` where the dark monitor was executed. Returns ------- query_result : float Date (in MJD) of the ending range of the previous MAST query where the dark monitor was run. """ sub_query = session.query(self.query_table.aperture, func.max(self.query_table.end_time_mjd).label('maxdate') ).group_by(self.query_table.aperture).subquery('t2') # Note that "self.query_table.run_monitor == True" below is # intentional. Switching = to "is" results in an error in the query. query = session.query(self.query_table).join( sub_query, and_( self.query_table.aperture == self.aperture, self.query_table.end_time_mjd == sub_query.c.maxdate, self.query_table.run_monitor == True ) ).all() query_count = len(query) if query_count == 0: query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(('\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) elif query_count > 1: raise ValueError('More than one "most recent" query?') else: query_result = query[0].end_time_mjd return query_result
def most_recent_search(self, file_type='dark'): """Query the query history database and return the information on the most recent query for the given ``aperture_name`` where the dark monitor was executed. Parameters ---------- file_type : str ``dark`` or ``flat``. Specifies the type of file whose previous search time is queried. Returns ------- query_result : float Date (in MJD) of the ending range of the previous MAST query where the dark monitor was run. """ if file_type.lower() == 'dark': mjd_field = self.query_table.dark_end_time_mjd run_field = self.query_table.run_bpix_from_darks elif file_type.lower() == 'flat': mjd_field = self.query_table.flat_end_time_mjd run_field = self.query_table.run_bpix_from_flats query = session.query(self.query_table).filter(self.query_table.aperture==self.aperture). \ filter(run_field==True) dates = np.zeros(0) if file_type.lower() == 'dark': for instance in query: dates = np.append(dates, instance.dark_end_time_mjd) elif file_type.lower() == 'flat': for instance in query: dates = np.append(dates, instance.flat_end_time_mjd) query_count = len(dates) if query_count == 0: query_result = 57357.0 # a.k.a. Dec 1, 2015 == CV3 logging.info(( '\tNo query history for {}. Beginning search date will be set to {}.' .format(self.aperture, query_result))) else: query_result = np.max(dates) return query_result
def plot_total_file_counts(): """Plot total file counts versus date Returns ------- plot : bokeh.plotting.figure.Figure object ``bokeh`` plot of total file counts versus date """ # Total file counts vs. date results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_count).all() dates, file_counts = zip(*results) plot = figure( tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_type='datetime', title="Total File Counts", x_axis_label='Date', y_axis_label='Count') plot.line(dates, file_counts, line_width=2, line_color='blue') plot.circle(dates, file_counts, color='blue') return plot
def plot_by_filetype(plot_type, instrument): """Plot ``count`` or ``size`` by filetype versus date for the given instrument, or all instruments. Parameters ---------- plot_type : str Which data to plot. Either ``count`` or ``size``. instrument : str The instrument to plot for. Can be a valid JWST instrument or ``all`` to plot across all instruments. Returns ------- plot : bokeh.plotting.figure.Figure object ``bokeh`` plot of total file counts versus date """ # Determine plot title if instrument == 'all': title = 'Total File {} by Type'.format(plot_type.capitalize()) else: instrument_title = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument] title = '{} Total File {} by Type'.format(instrument_title, plot_type.capitalize()) if plot_type == 'count': ytitle = 'Counts' else: ytitle = 'Size (TB)' # Initialize plot plot = figure( tools='pan,box_zoom,wheel_zoom,reset,save', x_axis_type='datetime', title=title, x_axis_label='Date', y_axis_label=ytitle) colors = itertools.cycle(palette) for filetype, color in zip(FILE_SUFFIX_TYPES, colors): # Query for counts results = session.query(FilesystemInstrument.date, getattr(FilesystemInstrument, plot_type))\ .filter(FilesystemInstrument.filetype == filetype) if instrument == 'all': results = results.all() else: results = results.filter(FilesystemInstrument.instrument == instrument).all() # Group by date if results: results_dict = defaultdict(int) for date, value in results: results_dict[date] += value # Parse results so they can be easily plotted dates = list(results_dict.keys()) values = list(results_dict.values()) # Plot the results plot.line(dates, values, legend='{} files'.format(filetype), line_color=color) plot.circle(dates, values, color=color) return plot