def jwqldb_table_viewer(request, tablename_param=None): """Generate the JWQL Table Viewer view. Parameters ---------- request : HttpRequest object Incoming request from the webpage tablename_param : str Table name parameter from URL Returns ------- HttpResponse object Outgoing response sent to the webpage """ if tablename_param is None: table_meta, tablename = get_jwqldb_table_view_components(request) else: table_meta = build_table(tablename_param) tablename = tablename_param _, _, engine, _ = load_connection(get_config()['connection_string']) all_jwql_tables = engine.table_names() if 'django_migrations' in all_jwql_tables: all_jwql_tables.remove( 'django_migrations') # No necessary information. jwql_tables_by_instrument = {} instruments = ['nircam', 'nirspec', 'niriss', 'miri', 'fgs'] # Sort tables by instrument for instrument in instruments: jwql_tables_by_instrument[instrument] = [ tablename for tablename in all_jwql_tables if instrument in tablename ] # Don't forget tables that dont contain instrument specific instrument information. jwql_tables_by_instrument['general'] = [ table for table in all_jwql_tables if not any(instrument in table for instrument in instruments) ] template = 'jwqldb_table_viewer.html' # If value of table_meta is None (when coming from home page) if table_meta is None: context = {'inst': '', 'all_jwql_tables': jwql_tables_by_instrument} # If table_meta is empty, just render table with no data. elif table_meta.empty: context = { 'inst': '', 'all_jwql_tables': jwql_tables_by_instrument, 'table_columns': table_meta.columns.values, 'table_name': tablename } # Else, everything is good to go, render the table. else: context = { 'inst': '', 'all_jwql_tables': jwql_tables_by_instrument, 'table_columns': table_meta.columns.values, 'table_rows': table_meta.values, 'table_name': tablename } return render(request, template, context)
def create_table(status_dict): """Create interactive ``bokeh`` table containing the logfile status results. Parameters ---------- status_dict : dict Nested dictionary with status results from all logfiles """ # Rearrange the nested dictionary into a non-nested dict for the table filenames = [] dates = [] missings = [] results = [] for key in status_dict: filenames.append(status_dict[key]['logname']) dates.append(datetime.fromtimestamp(status_dict[key]['latest_time'])) missings.append(str(status_dict[key]['missing_file'])) results.append(status_dict[key]['status']) # div to color the boxes in the status column success_template = """ <div style="background:<%= (function colorfromstr(){ if(value == "success"){ return("green")} else{return("red")} }()) %>; color: white"> <%= value %></div> """ # div to color the boxes in the column for possibly late logfiles missing_template = """ <div style="background:<%= (function colorfrombool(){ if(value == "True"){ return("orange")} else{return("green")} }()) %>; color: white"> <%= value %></div> """ success_formatter = HTMLTemplateFormatter(template=success_template) missing_formatter = HTMLTemplateFormatter(template=missing_template) data = dict(name=list(status_dict.keys()), filename=filenames, date=dates, missing=missings, result=results) source = ColumnDataSource(data) datefmt = DateFormatter(format="RFC-2822") columns = [ TableColumn(field="name", title="Monitor Name", width=200), TableColumn(field="filename", title="Most Recent File", width=350), TableColumn(field="date", title="Most Recent Time", width=200, formatter=datefmt), TableColumn(field="missing", title="Possible Missing File", width=200, formatter=missing_formatter), TableColumn(field="result", title="Status", width=100, formatter=success_formatter), ] data_table = DataTable(source=source, columns=columns, width=800, height=280, index_position=None) # Get output directory for saving the table files output_dir = get_config()['outputs'] output_filename = 'cron_status_table' # Save full html html_outfile = os.path.join(output_dir, 'monitor_cron_jobs', '{}.html'.format(output_filename)) output_file(html_outfile) save(data_table) try: set_permissions(html_outfile) except PermissionError: logging.warning('Unable to set permissions for {}'.format(html_outfile)) logging.info('Saved Bokeh full HTML file: {}'.format(html_outfile))
from .data_containers import get_edb_components from .data_containers import get_filenames_by_instrument from .data_containers import get_header_info from .data_containers import get_image_info from .data_containers import get_proposal_info from .data_containers import get_thumbnails_all_instruments from .data_containers import nirspec_trending from .data_containers import random_404_page from .data_containers import get_jwqldb_table_view_components from .data_containers import thumbnails_ajax from .data_containers import thumbnails_query_ajax from .forms import InstrumentAnomalySubmitForm from .forms import AnomalyQueryForm from .forms import FileSearchForm FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') def anomaly_query(request): """The anomaly query form page""" form = AnomalyQueryForm(request.POST or None) if request.method == 'POST': if form.is_valid(): query_configs = {} for instrument in ['miri', 'nirspec', 'niriss', 'nircam', 'fgs']: query_configs[instrument] = {} query_configs[instrument]['filters'] = [ query_unformat(i) for i in form.cleaned_data['{}_filt'.format(instrument)]
def get_dashboard_components(): """Build and return dictionaries containing components and html needed for the dashboard. Returns ------- dashboard_components : dict A dictionary containing components needed for the dashboard. dashboard_html : dict A dictionary containing full HTML needed for the dashboard. """ output_dir = get_config()['outputs'] name_dict = { '': '', 'monitor_mast': 'Database Monitor', 'monitor_filesystem': 'Filesystem Monitor' } # Run the cron job monitor to produce an updated table monitor_cron_jobs.status(production_mode=True) # Build dictionary of Bokeh components from files in the output directory dashboard_components = {} for dir_name, _, file_list in os.walk(output_dir): monitor_name = os.path.basename(dir_name) # Only continue if the dashboard knows how to build that monitor if monitor_name in name_dict.keys(): formatted_monitor_name = name_dict[monitor_name] dashboard_components[formatted_monitor_name] = {} for fname in file_list: if 'component' in fname: full_fname = '{}/{}'.format(monitor_name, fname) plot_name = fname.split('_component')[0] # Generate formatted plot name formatted_plot_name = plot_name.title().replace('_', ' ') for lowercase, mixed_case in JWST_INSTRUMENT_NAMES_MIXEDCASE.items( ): formatted_plot_name = formatted_plot_name.replace( lowercase.capitalize(), mixed_case) formatted_plot_name = formatted_plot_name.replace( 'Jwst', 'JWST') formatted_plot_name = formatted_plot_name.replace( 'Caom', 'CAOM') # Get the div html_file = full_fname.split('.')[0] + '.html' with open(os.path.join(output_dir, html_file), 'r') as f: div = f.read() # Get the script js_file = full_fname.split('.')[0] + '.js' with open(os.path.join(output_dir, js_file), 'r') as f: script = f.read() # Save to dictionary dashboard_components[formatted_monitor_name][ formatted_plot_name] = [div, script] # Add HTML that cannot be saved as components to the dictionary with open( os.path.join(output_dir, 'monitor_cron_jobs', 'cron_status_table.html'), 'r') as f: cron_status_table_html = f.read() dashboard_html = {} dashboard_html['Cron Job Monitor'] = cron_status_table_html return dashboard_components, dashboard_html
def run(self): """The main method. See module docstrings for further details. There are 2 parts to the bad pixel monitor: 1. Bad pixels from illuminated data 2. Bad pixels from dark data For each, we will query MAST, copy new files from the filesystem and pass the list of copied files into the ``process()`` method. """ logging.info('Begin logging for bad_pixel_monitor') # Get the output directory self.output_dir = os.path.join(get_config()['outputs'], 'bad_pixel_monitor') # Read in config file that defines the thresholds for the number # of dark files that must be present in order for the monitor to run limits = ascii.read(THRESHOLDS_FILE) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures from pysiaf possible_apertures = self.get_possible_apertures() for aperture in possible_apertures: grating = None detector_name = None lamp = None # NIRSpec flats use the MIRROR grating. if self.instrument == 'nirspec': grating = 'MIRROR' # MIRI is unlike the other instruments. We basically treat # the detector as the aperture name because there is no # aperture name for a full frame MRS exposure. if self.instrument == 'miri': detector_name, aperture_name = aperture self.aperture = detector_name else: self.aperture = aperture aperture_name = aperture # In flight, NIRISS plans to take darks using the LINE2 lamp if self.instrument == 'niriss': lamp = 'LINE2' # What lamp is most appropriate for NIRSpec? if self.instrument == 'nirspec': lamp = 'LINE2' # What lamp is most appropriate for FGS? # if self.instrument == 'fgs': # lamp = 'G2LAMP1' logging.info('') logging.info('Working on aperture {} in {}'.format( aperture, self.instrument)) # Find the appropriate threshold for number of new files needed match = self.aperture == limits['Aperture'] flat_file_count_threshold = limits['FlatThreshold'][ match].data[0] dark_file_count_threshold = limits['DarkThreshold'][ match].data[0] # Locate the record of the most recent MAST search self.flat_query_start = self.most_recent_search( file_type='flat') self.dark_query_start = self.most_recent_search( file_type='dark') logging.info('\tFlat field query times: {} {}'.format( self.flat_query_start, self.query_end)) logging.info('\tDark current query times: {} {}'.format( self.dark_query_start, self.query_end)) # Query MAST using the aperture and the time of the most # recent previous search as the starting time. flat_templates = FLAT_EXP_TYPES[instrument] dark_templates = DARK_EXP_TYPES[instrument] new_flat_entries = mast_query(instrument, flat_templates, self.flat_query_start, self.query_end, aperture=aperture_name, grating=grating, detector=detector_name, lamp=lamp) new_dark_entries = mast_query(instrument, dark_templates, self.dark_query_start, self.query_end, aperture=aperture_name, detector=detector_name) # Filter the results # Filtering could be different for flats vs darks. # Kevin says we shouldn't need to worry about mixing lamps in # the data used to create the bad pixel mask. # In flight, data will only be taken with LINE2, LEVEL 5. # Currently in MAST all lamps are present, but Kevin is # not concerned about variations in flat field strucutre. # NIRISS - results can include rate, rateints, trapsfilled # MIRI - Jane says they now use illuminated data for dead pixel checks, just like other insts. # NIRSpec - can be cal, x1d, rate, rateints. Can have both cal and x1d so filter repeats # FGS - rate, rateints, trapsfilled # NIRCam - no int flats # The query results can contain multiple entries for files # in different calibration states (or for different output # products), so we need to filter the list for duplicate # entries and for the calibration state we are interested # in before we know how many new entries there really are. # In the end, we need rate files as well as uncal files # because we're going to need to create jump files. # In order to use a given file we must have at least the # uncal version of the file. Get the uncal and rate file # lists to align. if new_flat_entries: # Exclude ASIC tuning data len_new_flats = len(new_flat_entries) new_flat_entries = monitor_utils.exclude_asic_tuning( new_flat_entries) len_no_asic = len(new_flat_entries) num_asic = len_new_flats - len_no_asic logging.info( "\tFiltering out ASIC tuning files removed {} flat files." .format(num_asic)) new_flat_entries = self.filter_query_results( new_flat_entries, datatype='flat') apcheck_flat_entries = pipeline_tools.aperture_size_check( new_flat_entries, instrument, aperture) lost_to_bad_metadata = len(new_flat_entries) - len( apcheck_flat_entries) logging.info( '\t{} flat field files ignored due to inconsistency in array size and metadata.' .format(lost_to_bad_metadata)) flat_uncal_files = locate_uncal_files(apcheck_flat_entries) flat_uncal_files, run_flats = check_for_sufficient_files( flat_uncal_files, instrument, aperture, flat_file_count_threshold, 'flats') flat_rate_files, flat_rate_files_to_copy = locate_rate_files( flat_uncal_files) else: run_flats = False flat_uncal_files, flat_rate_files, flat_rate_files_to_copy = None, None, None if new_dark_entries: # Exclude ASIC tuning data len_new_darks = len(new_dark_entries) new_dark_entries = monitor_utils.exclude_asic_tuning( new_dark_entries) len_no_asic = len(new_dark_entries) num_asic = len_new_darks - len_no_asic logging.info( "\tFiltering out ASIC tuning files removed {} dark files." .format(num_asic)) new_dark_entries = self.filter_query_results( new_dark_entries, datatype='dark') apcheck_dark_entries = pipeline_tools.aperture_size_check( new_dark_entries, instrument, aperture) lost_to_bad_metadata = len(new_dark_entries) - len( apcheck_dark_entries) logging.info( '\t{} dark files ignored due to inconsistency in array size and metadata.' .format(lost_to_bad_metadata)) dark_uncal_files = locate_uncal_files(apcheck_dark_entries) dark_uncal_files, run_darks = check_for_sufficient_files( dark_uncal_files, instrument, aperture, dark_file_count_threshold, 'darks') dark_rate_files, dark_rate_files_to_copy = locate_rate_files( dark_uncal_files) else: run_darks = False dark_uncal_files, dark_rate_files, dark_rate_files_to_copy = None, None, None # Set up directories for the copied data ensure_dir_exists(os.path.join(self.output_dir, 'data')) self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) ensure_dir_exists(self.data_dir) # Copy files from filesystem if run_flats: flat_uncal_files, flat_rate_files = self.map_uncal_and_rate_file_lists( flat_uncal_files, flat_rate_files, flat_rate_files_to_copy, 'flat') if run_darks: dark_uncal_files, dark_rate_files = self.map_uncal_and_rate_file_lists( dark_uncal_files, dark_rate_files, dark_rate_files_to_copy, 'dark') # Run the bad pixel monitor if run_flats or run_darks: self.process(flat_uncal_files, flat_rate_files, dark_uncal_files, dark_rate_files) # Update the query history if dark_uncal_files is None: num_dark_files = 0 else: num_dark_files = len(dark_uncal_files) if flat_uncal_files is None: num_flat_files = 0 else: num_flat_files = len(flat_uncal_files) new_entry = { 'instrument': self.instrument.upper(), 'aperture': self.aperture, 'dark_start_time_mjd': self.dark_query_start, 'dark_end_time_mjd': self.query_end, 'flat_start_time_mjd': self.flat_query_start, 'flat_end_time_mjd': self.query_end, 'dark_files_found': num_dark_files, 'flat_files_found': num_flat_files, 'run_bpix_from_darks': run_darks, 'run_bpix_from_flats': run_flats, 'run_monitor': run_flats or run_darks, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Bad Pixel Monitor completed successfully.')
def run(self): """The main method. See module docstrings for further details. """ logging.info('Begin logging for dark_monitor') apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL'] # Get the output directory self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor') # Read in config file that defines the thresholds for the number # of dark files that must be present in order for the monitor to run limits = ascii.read(THRESHOLDS_FILE) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures from pysiaf possible_apertures = list(Siaf(instrument).apernames) possible_apertures = [ap for ap in possible_apertures if ap not in apertures_to_skip] for aperture in possible_apertures: logging.info('') logging.info('Working on aperture {} in {}'.format(aperture, instrument)) # Find the appropriate threshold for the number of new files needed match = aperture == limits['Aperture'] file_count_threshold = limits['Threshold'][match] # Locate the record of the most recent MAST search self.aperture = aperture self.query_start = self.most_recent_search() logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end)) # Query MAST using the aperture and the time of the # most recent previous search as the starting time new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end) logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries))) # Check to see if there are enough new files to meet the # monitor's signal-to-noise requirements if len(new_entries) >= file_count_threshold: logging.info('\tSufficient new dark files found for {}, {} to run the dark monitor.' .format(self.instrument, self.aperture)) # Get full paths to the files new_filenames = [] for file_entry in new_entries: try: new_filenames.append(filesystem_path(file_entry['filename'])) except FileNotFoundError: logging.warning('\t\tUnable to locate {} in filesystem. Not including in processing.' .format(file_entry['filename'])) # Set up directories for the copied data ensure_dir_exists(os.path.join(self.output_dir, 'data')) self.data_dir = os.path.join(self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) ensure_dir_exists(self.data_dir) # Copy files from filesystem dark_files, not_copied = copy_files(new_filenames, self.data_dir) logging.info('\tNew_filenames: {}'.format(new_filenames)) logging.info('\tData dir: {}'.format(self.data_dir)) logging.info('\tCopied to working dir: {}'.format(dark_files)) logging.info('\tNot copied: {}'.format(not_copied)) # Run the dark monitor self.process(dark_files) monitor_run = True else: logging.info(('\tDark monitor skipped. {} new dark files for {}, {}. {} new files are ' 'required to run dark current monitor.').format( len(new_entries), instrument, aperture, file_count_threshold[0])) monitor_run = False # Update the query history new_entry = {'instrument': instrument, 'aperture': aperture, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'files_found': len(new_entries), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now()} self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Dark Monitor completed successfully.')
def test_get_config(): """Assert that the ``get_config`` function successfully creates a dictionary. """ settings = get_config() assert isinstance(settings, dict)
import os import glob import jwql.instrument_monitors.nirspec_monitors.data_trending.utils.mnemonics as mn import jwql.instrument_monitors.nirspec_monitors.data_trending.utils.sql_interface as sql import jwql.instrument_monitors.nirspec_monitors.data_trending.utils.csv_to_AstropyTable as apt from jwql.utils.utils import get_config, filename_parser from astropy.table import Table, Column from jwql.instrument_monitors.nirspec_monitors.data_trending.utils.process_data import once_a_day_routine __location__ = os.path.realpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) #point to the directory where your files are located! directory = os.path.join(get_config()['outputs'], 'nirspec_data_trending', 'nirspec_new_15min', '*.CSV') #here some some files contain the same data but they are all incomplete #in order to generate a full database we have to import all of them filenames = glob.glob(directory) def process_file(conn, path): '''Parse CSV file, process data within and put to DB Parameters ---------- conn : DBobject Connection object to temporary database path : str defines path to the files
def run(self): """The main method. See module docstrings for further details. """ logging.info('Begin logging for readnoise_monitor\n') # Get the output directory and setup a directory to store the data self.output_dir = os.path.join(get_config()['outputs'], 'readnoise_monitor') ensure_dir_exists(os.path.join(self.output_dir, 'data')) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures for this instrument siaf = Siaf(self.instrument) possible_apertures = list(siaf.apertures) for aperture in possible_apertures: logging.info('\nWorking on aperture {} in {}'.format( aperture, instrument)) self.aperture = aperture # Locate the record of the most recent MAST search; use this time # (plus a buffer to catch any missing files from the previous # run) as the start time in the new MAST search. most_recent_search = self.most_recent_search() self.query_start = most_recent_search - 70 # Query MAST for new dark files for this instrument/aperture logging.info('\tQuery times: {} {}'.format( self.query_start, self.query_end)) new_entries = monitor_utils.mast_query_darks( instrument, aperture, self.query_start, self.query_end) # Exclude ASIC tuning data len_new_darks = len(new_entries) new_entries = monitor_utils.exclude_asic_tuning(new_entries) len_no_asic = len(new_entries) num_asic = len_new_darks - len_no_asic logging.info( "\tFiltering out ASIC tuning files removed {} dark files.". format(num_asic)) logging.info('\tAperture: {}, new entries: {}'.format( self.aperture, len(new_entries))) # Set up a directory to store the data for this aperture self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) if len(new_entries) > 0: ensure_dir_exists(self.data_dir) # Get any new files to process new_files = [] checked_files = [] for file_entry in new_entries: output_filename = os.path.join( self.data_dir, file_entry['filename'].replace('_dark', '_uncal')) # Sometimes both the dark and uncal name of a file is picked up in new_entries if output_filename in checked_files: logging.info( '\t{} already checked in this run.'.format( output_filename)) continue checked_files.append(output_filename) # Dont process files that already exist in the readnoise stats database file_exists = self.file_exists_in_database(output_filename) if file_exists: logging.info( '\t{} already exists in the readnoise database table.' .format(output_filename)) continue # Save any new uncal files with enough groups in the output directory; some dont exist in JWQL filesystem try: filename = filesystem_path(file_entry['filename']) uncal_filename = filename.replace('_dark', '_uncal') if not os.path.isfile(uncal_filename): logging.info( '\t{} does not exist in JWQL filesystem, even though {} does' .format(uncal_filename, filename)) else: num_groups = fits.getheader( uncal_filename)['NGROUPS'] num_ints = fits.getheader(uncal_filename)['NINTS'] if instrument == 'miri': total_cds_frames = int( (num_groups - 6) / 2) * num_ints else: total_cds_frames = int( num_groups / 2) * num_ints # Skip processing if the file doesnt have enough groups/ints to calculate the readnoise. # MIRI needs extra since they omit the first five and last group before calculating the readnoise. if total_cds_frames >= 10: shutil.copy(uncal_filename, self.data_dir) logging.info('\tCopied {} to {}'.format( uncal_filename, output_filename)) set_permissions(output_filename) new_files.append(output_filename) else: logging.info( '\tNot enough groups/ints to calculate readnoise in {}' .format(uncal_filename)) except FileNotFoundError: logging.info( '\t{} does not exist in JWQL filesystem'.format( file_entry['filename'])) # Run the readnoise monitor on any new files if len(new_files) > 0: self.process(new_files) monitor_run = True else: logging.info( '\tReadnoise monitor skipped. {} new dark files for {}, {}.' .format(len(new_files), instrument, aperture)) monitor_run = False # Update the query history new_entry = { 'instrument': instrument, 'aperture': aperture, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'entries_found': len(new_entries), 'files_found': len(new_files), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Readnoise Monitor completed successfully.')
from bokeh.models.tickers import LogTicker from datetime import datetime import numpy as np from jwql.database.database_interface import session from jwql.database.database_interface import NIRCamDarkQueryHistory, NIRCamDarkPixelStats, NIRCamDarkDarkCurrent from jwql.database.database_interface import NIRISSDarkQueryHistory, NIRISSDarkPixelStats, NIRISSDarkDarkCurrent from jwql.database.database_interface import MIRIDarkQueryHistory, MIRIDarkPixelStats, MIRIDarkDarkCurrent from jwql.database.database_interface import NIRSpecDarkQueryHistory, NIRSpecDarkPixelStats, NIRSpecDarkDarkCurrent from jwql.database.database_interface import FGSDarkQueryHistory, FGSDarkPixelStats, FGSDarkDarkCurrent from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import get_config from jwql.bokeh_templating import BokehTemplate SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) OUTPUTS_DIR = get_config()['outputs'] class DarkMonitor(BokehTemplate): # Combine instrument and aperture into a single property because we # do not want to invoke the setter unless both are updated @property def aperture_info(self): return (self._instrument, self._aperture) @aperture_info.setter def aperture_info(self, info): self._instrument, self._aperture = info self.pre_init() self.post_init()
def run(self): """The main method. See module docstrings for further details. """ logging.info('Begin logging for dark_monitor') apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL'] # Get the output directory self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor') # Read in config file that defines the thresholds for the number # of dark files that must be present in order for the monitor to run limits = ascii.read(THRESHOLDS_FILE) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures from pysiaf possible_apertures = list(Siaf(instrument).apernames) possible_apertures = [ ap for ap in possible_apertures if ap not in apertures_to_skip ] # Get a list of all possible readout patterns associated with the aperture possible_readpatts = RAPID_READPATTERNS[instrument] for aperture in possible_apertures: logging.info('') logging.info('Working on aperture {} in {}'.format( aperture, instrument)) # Find appropriate threshold for the number of new files needed match = aperture == limits['Aperture'] # If the aperture is not listed in the threshold file, we need # a default if not np.any(match): file_count_threshold = 30 logging.warning(( '\tAperture {} is not present in the threshold file. Continuing ' 'with the default threshold of 30 files.'.format( aperture))) else: file_count_threshold = limits['Threshold'][match][0] self.aperture = aperture # We need a separate search for each readout pattern for readpatt in possible_readpatts: self.readpatt = readpatt logging.info('\tWorking on readout pattern: {}'.format( self.readpatt)) # Locate the record of the most recent MAST search self.query_start = self.most_recent_search() logging.info('\tQuery times: {} {}'.format( self.query_start, self.query_end)) # Query MAST using the aperture and the time of the # most recent previous search as the starting time new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt) logging.info( '\tAperture: {}, Readpattern: {}, new entries: {}'. format(self.aperture, self.readpatt, len(new_entries))) # Check to see if there are enough new files to meet the # monitor's signal-to-noise requirements if len(new_entries) >= file_count_threshold: logging.info( '\tMAST query has returned sufficient new dark files for {}, {}, {} to run the dark monitor.' .format(self.instrument, self.aperture, self.readpatt)) # Get full paths to the files new_filenames = [] for file_entry in new_entries: try: new_filenames.append( filesystem_path(file_entry['filename'])) except FileNotFoundError: logging.warning( '\t\tUnable to locate {} in filesystem. Not including in processing.' .format(file_entry['filename'])) # In some (unusual) cases, there are files in MAST with the correct aperture name # but incorrect array sizes. Make sure that the new files all have the expected # aperture size temp_filenames = [] bad_size_filenames = [] expected_ap = Siaf(instrument)[aperture] expected_xsize = expected_ap.XSciSize expected_ysize = expected_ap.YSciSize for new_file in new_filenames: with fits.open(new_file) as hdulist: xsize = hdulist[0].header['SUBSIZE1'] ysize = hdulist[0].header['SUBSIZE2'] if xsize == expected_xsize and ysize == expected_ysize: temp_filenames.append(new_file) else: bad_size_filenames.append(new_file) if len(temp_filenames) != len(new_filenames): logging.info( '\tSome files returned by MAST have unexpected aperture sizes. These files will be ignored: ' ) for badfile in bad_size_filenames: logging.info('\t\t{}'.format(badfile)) new_filenames = deepcopy(temp_filenames) # If it turns out that the monitor doesn't find enough # of the files returned by the MAST query to meet the threshold, # then the monitor will not be run if len(new_filenames) < file_count_threshold: logging.info(( "\tFilesystem search for the files identified by MAST has returned {} files. " "This is less than the required minimum number of files ({}) necessary to run " "the monitor. Quitting.").format( len(new_filenames), file_count_threshold)) monitor_run = False else: logging.info(( "\tFilesystem search for the files identified by MAST has returned {} files." ).format(len(new_filenames))) monitor_run = True if monitor_run: # Set up directories for the copied data ensure_dir_exists( os.path.join(self.output_dir, 'data')) self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) ensure_dir_exists(self.data_dir) # Copy files from filesystem dark_files, not_copied = copy_files( new_filenames, self.data_dir) logging.info( '\tNew_filenames: {}'.format(new_filenames)) logging.info('\tData dir: {}'.format( self.data_dir)) logging.info('\tCopied to working dir: {}'.format( dark_files)) logging.info('\tNot copied: {}'.format(not_copied)) # Run the dark monitor self.process(dark_files) else: logging.info(( '\tDark monitor skipped. MAST query has returned {} new dark files for ' '{}, {}, {}. {} new files are required to run dark current monitor.' ).format(len(new_entries), instrument, aperture, self.readpatt, file_count_threshold)) monitor_run = False # Update the query history new_entry = { 'instrument': instrument, 'aperture': aperture, 'readpattern': self.readpatt, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'files_found': len(new_entries), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Dark Monitor completed successfully.')
def clean_search(self): """Validate the "search" field. Check that the input is either a proposal or fileroot, and one that matches files in the filesystem. Returns ------- str The cleaned data input into the "search" field """ # Get the cleaned search data search = self.cleaned_data['search'] # Make sure the search is either a proposal or fileroot if search.isnumeric() and 1 < int(search) < 99999: self.search_type = 'proposal' elif self._search_is_fileroot(search): self.search_type = 'fileroot' else: raise forms.ValidationError('Invalid search term {}. Please provide proposal number ' 'or file root.'.format(search)) # If they searched for a proposal... if self.search_type == 'proposal': # See if there are any matching proposals and, if so, what # instrument they are for proposal_string = '{:05d}'.format(int(search)) search_string_public = os.path.join(get_config()['filesystem'], 'public', 'jw{}'.format(proposal_string), '*', '*{}*.fits'.format(proposal_string)) search_string_proprietary = os.path.join(get_config()['filesystem'], 'proprietary', 'jw{}'.format(proposal_string), '*', '*{}*.fits'.format(proposal_string)) all_files = glob.glob(search_string_public) all_files.extend(glob.glob(search_string_proprietary)) # Ignore "original" files all_files = [filename for filename in all_files if 'original' not in filename] if len(all_files) > 0: all_instruments = [] for file in all_files: instrument = filename_parser(file)['instrument'] all_instruments.append(instrument) if len(set(all_instruments)) > 1: raise forms.ValidationError('Cannot return result for proposal with multiple instruments ({}).'.format(', '.join(set(all_instruments)))) self.instrument = all_instruments[0] else: raise forms.ValidationError('Proposal {} not in the filesystem.'.format(search)) # If they searched for a fileroot... elif self.search_type == 'fileroot': # See if there are any matching fileroots and, if so, what instrument they are for search_string_public = os.path.join(get_config()['filesystem'], 'public', search[:7], search[:13], '{}*.fits'.format(search)) search_string_proprietary = os.path.join(get_config()['filesystem'], 'proprietary', search[:7], search[:13], '{}*.fits'.format(search)) all_files = glob.glob(search_string_public) all_files.extend(glob.glob(search_string_proprietary)) # Ignore "original" files all_files = [filename for filename in all_files if 'original' not in filename] if len(all_files) == 0: raise forms.ValidationError('Fileroot {} not in the filesystem.'.format(search)) instrument = search.split('_')[-1][:3] self.instrument = JWST_INSTRUMENT_NAMES_SHORTHAND[instrument] return self.cleaned_data['search']
def get_dashboard_components(): """Build and return dictionaries containing components and html needed for the dashboard. Returns ------- dashboard_components : dict A dictionary containing components needed for the dashboard. dashboard_html : dict A dictionary containing full HTML needed for the dashboard. """ output_dir = get_config()['outputs'] name_dict = { '': '', 'monitor_mast': 'Database Monitor', 'database_monitor_jwst': 'JWST', 'database_monitor_caom': 'JWST (CAOM)', 'monitor_filesystem': 'Filesystem Monitor', 'filecount_type': 'Total File Counts by Type', 'size_type': 'Total File Sizes by Type', 'filecount': 'Total File Counts', 'system_stats': 'System Statistics' } # Exclude monitors that can't be saved as components exclude_list = ['monitor_cron_jobs'] # Run the cron job monitor to produce an updated table monitor_cron_jobs.status(production_mode=True) # Build dictionary of components dashboard_components = {} for dir_name, subdir_list, file_list in os.walk(output_dir): monitor_name = os.path.basename(dir_name) if monitor_name not in exclude_list: dashboard_components[name_dict[monitor_name]] = {} for fname in file_list: if 'component' in fname: full_fname = '{}/{}'.format(monitor_name, fname) plot_name = fname.split('_component')[0] # Get the div html_file = full_fname.split('.')[0] + '.html' with open(os.path.join(output_dir, html_file)) as f: div = f.read() # Get the script js_file = full_fname.split('.')[0] + '.js' with open(os.path.join(output_dir, js_file)) as f: script = f.read() dashboard_components[name_dict[monitor_name]][ name_dict[plot_name]] = [div, script] # Add HTML that cannot be saved as components to the dictionary with open( os.path.join(output_dir, 'monitor_cron_jobs', 'cron_status_table.html')) as f: cron_status_table_html = f.read() dashboard_html = {} dashboard_html['Cron Job Monitor'] = cron_status_table_html return dashboard_components, dashboard_html
import os import pkg_resources from jwql.utils import utils module_path = pkg_resources.resource_filename('jwql', '') setup_path = os.path.normpath(os.path.join(module_path, '../setup.py')) try: with open(setup_path) as f: data = f.readlines() for line in data: if 'VERSION =' in line: __version__ = line.split(' ')[-1].replace("'", "").strip() config_version = utils.get_config()['jwql_version'] if __version__ != config_version: print( "Warning: config file JWQL version is {}, while JWQL is using {}". format(config_version, __version__)) except FileNotFoundError: print('Could not determine jwql version') __version__ = '0.0.0'
from bokeh.embed import components from bokeh.layouts import gridplot from bokeh.palettes import Category20_20 as palette from bokeh.plotting import figure, output_file, save from jwql.database.database_interface import engine from jwql.database.database_interface import session from jwql.database.database_interface import FilesystemGeneral from jwql.database.database_interface import FilesystemInstrument from jwql.utils.logging_functions import configure_logging, log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config FILESYSTEM = get_config()['filesystem'] def gather_statistics(general_results_dict, instrument_results_dict): """Walks the filesytem to gather various statistics to eventually store in the database Parameters ---------- general_results_dict : dict A dictionary for the ``filesystem_general`` database table instrument_results_dict : dict A dictionary for the ``filesystem_instrument`` database table Returns -------
def monitor_template_main(): """ The main function of the ``monitor_template`` module.""" # Example of logging my_variable = 'foo' logging.info('Some useful information: {}'.format(my_variable)) # Example of querying for a dataset via MAST API service = "Mast.Jwst.Filtered.Niriss" params = { "columns": "filename", "filters": [{ "paramName": "filter", "values": ['F430M'] }] } response = Mast.service_request_async(service, params) result = response[0].json()['data'] filename_of_interest = result[0][ 'filename'] # jw00304002001_02102_00001_nis_uncal.fits # Example of parsing a filename filename_dict = filename_parser(filename_of_interest) # Contents of filename_dict: # {'program_id': '00304', # 'observation': '002', # 'visit': '001', # 'visit_group': '02', # 'parallel_seq_id': '1', # 'activity': '02', # 'exposure_id': '00001', # 'detector': 'nis', # 'suffix': 'uncal'} # Example of locating a dataset in the filesystem filesystem = get_config()['filesystem'] dataset = os.path.join(filesystem, 'jw{}'.format(filename_dict['program_id']), filename_of_interest) # Example of reading in dataset using jwst.datamodels im = datamodels.open(dataset) # Now have access to: # im.data # Data array # im.err # ERR array # im.meta # Metadata such as header keywords # Example of saving a file and setting permissions im.save('some_filename.fits') set_permissions('some_filename.fits') # Example of creating and exporting a Bokeh plot plt = Donut(im.data, plot_width=600, plot_height=600) plt.sizing_mode = 'stretch_both' # Necessary for responsive sizing on web app script, div = components(plt) plot_output_dir = get_config()['outputs'] div_outfile = os.path.join(plot_output_dir, 'monitor_name', filename_of_interest + "_component.html") script_outfile = os.path.join(plot_output_dir, 'monitor_name', filename_of_interest + "_component.js") for outfile, component in zip([div_outfile, script_outfile], [div, script]): with open(outfile, 'w') as f: f.write(component) f.close() set_permissions(outfile) # Perform any other necessary code well_named_variable = "Function does something." result_of_second_function = second_function(well_named_variable)
def generate_preview_images(): """The main function of the ``generate_preview_image`` module.""" # Begin logging logging.info("Beginning the script run") filesystem = get_config()['filesystem'] preview_image_filesystem = get_config()['preview_image_filesystem'] thumbnail_filesystem = get_config()['thumbnail_filesystem'] filenames = glob(os.path.join(filesystem, '*/*.fits')) grouped_filenames = group_filenames(filenames) logging.info(f"Found {len(filenames)} filenames") for file_list in grouped_filenames: filename = file_list[0] # Determine the save location try: identifier = 'jw{}'.format(filename_parser(filename)['program_id']) except ValueError as error: identifier = os.path.basename(filename).split('.fits')[0] preview_output_directory = os.path.join(preview_image_filesystem, identifier) thumbnail_output_directory = os.path.join(thumbnail_filesystem, identifier) # Check to see if the preview images already exist and skip # if they do file_exists = check_existence(file_list, preview_output_directory) if file_exists: logging.info( "JPG already exists for {}, skipping.".format(filename)) continue # Create the output directories if necessary if not os.path.exists(preview_output_directory): os.makedirs(preview_output_directory) permissions.set_permissions(preview_output_directory) logging.info(f'Created directory {preview_output_directory}') if not os.path.exists(thumbnail_output_directory): os.makedirs(thumbnail_output_directory) permissions.set_permissions(thumbnail_output_directory) logging.info(f'Created directory {thumbnail_output_directory}') # If the exposure contains more than one file (because more # than one detector was used), then create a mosaic max_size = 8 numfiles = len(file_list) if numfiles != 1: try: mosaic_image, mosaic_dq = create_mosaic(file_list) logging.info('Created mosiac for:') for item in file_list: logging.info(f'\t{item}') except (ValueError, FileNotFoundError) as error: logging.error(error) dummy_file = create_dummy_filename(file_list) if numfiles in [2, 4]: max_size = 16 elif numfiles in [8]: max_size = 32 # Create the nominal preview image and thumbnail try: im = PreviewImage(filename, "SCI") im.clip_percent = 0.01 im.scaling = 'log' im.cmap = 'viridis' im.output_format = 'jpg' im.preview_output_directory = preview_output_directory im.thumbnail_output_directory = thumbnail_output_directory # If a mosaic was made from more than one file # insert it and it's associated DQ array into the # instance of PreviewImage. Also set the input # filename to indicate that we have mosaicked data if numfiles != 1: im.data = mosaic_image im.dq = mosaic_dq im.file = dummy_file im.make_image(max_img_size=max_size) except ValueError as error: logging.warning(error) # Complete logging: logging.info("Completed.")
Dependencies ------------ Users must have a ``config.json`` configuration file with a proper ``connection_string`` key that points to the ``jwqldb`` database. The ``connection_string`` format is ``postgresql+psycopg2://user:password@host:port/database``. """ from jwql.database.database_interface import base, set_read_permissions from jwql.utils.utils import get_config if __name__ == '__main__': connection_string = get_config()['connection_string'] server_type = connection_string.split('@')[-1][0] assert server_type != 'p', 'Cannot reset production database!' prompt = ('About to reset all tables for database instance {}. Do you ' 'wish to proceed? (y/n)\n'.format(connection_string)) response = input(prompt) if response.lower() == 'y': base.metadata.drop_all() base.metadata.create_all() set_read_permissions() print('\nDatabase instance {} has been reset'.format(connection_string))
import jwql.instrument_monitors.miri_monitors.data_trending.utils.mnemonics as mn import jwql.instrument_monitors.miri_monitors.data_trending.utils.sql_interface as sql import jwql.instrument_monitors.miri_monitors.data_trending.utils.csv_to_AstropyTable as apt from jwql.instrument_monitors.miri_monitors.data_trending.utils.process_data import whole_day_routine, wheelpos_routine from jwql.utils.utils import get_config, filename_parser import os import glob import statistics import sqlite3 #set _location_ variable __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) #files with data to initially fill the database directory = os.path.join(get_config()['outputs'], 'miri_data_trending', 'trainings_data_day', '*.CSV') paths = glob.glob(directory) def process_file(conn, path): '''Parse CSV file, process data within and put to DB Parameters ---------- conn : DBobject Connection object to temporary database path : str defines file to read ''' m_raw_data = apt.mnemonics(path)
from bokeh.palettes import Category20_20 as palette from bokeh.plotting import figure, output_file, save from jwql.database.database_interface import engine from jwql.database.database_interface import session from jwql.database.database_interface import FilesystemGeneral from jwql.database.database_interface import FilesystemInstrument from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import configure_logging, log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table SETTINGS = get_config() FILESYSTEM = SETTINGS['filesystem'] PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary') PUBLIC_FILESYSTEM = os.path.join(FILESYSTEM, 'public') CENTRAL = SETTINGS['jwql_dir'] OUTPUTS = SETTINGS['outputs'] def gather_statistics(general_results_dict, instrument_results_dict): """Walks the filesytem to gather various statistics to eventually store in the database Parameters ---------- general_results_dict : dict A dictionary for the ``filesystem_general`` database table
import os import re import tempfile from astropy.io import fits from astropy.table import Table from astropy.time import Time from django.conf import settings import numpy as np from operator import itemgetter # astroquery.mast import that depends on value of auth_mast # this import has to be made before any other import of astroquery.mast from jwql.utils.utils import get_config, filename_parser, check_config_for_key check_config_for_key('auth_mast') auth_mast = get_config()['auth_mast'] mast_flavour = '.'.join(auth_mast.split('.')[1:]) from astropy import config conf = config.get_config('astroquery') conf['mast'] = {'server': 'https://{}'.format(mast_flavour)} from astroquery.mast import Mast from jwedb.edb_interface import mnemonic_inventory from jwql.database import database_interface as di from jwql.database.database_interface import load_connection from jwql.edb.engineering_database import get_mnemonic, get_mnemonic_info from jwql.instrument_monitors.miri_monitors.data_trending import dashboard as miri_dash from jwql.instrument_monitors.nirspec_monitors.data_trending import dashboard as nirspec_dash from jwql.jwql_monitors import monitor_cron_jobs from jwql.utils.utils import ensure_dir_exists from jwql.utils.constants import MONITORS, JWST_INSTRUMENT_NAMES_MIXEDCASE
def status(production_mode=True): """Main function: determine the status of the instrument montiors by examining log files. Parameters ---------- production_mode : bool If ``True``, look in the main log directory. If ``False``, look in the ``dev`` log file directory. Returns ------- logfile_status : dict Nested dictionary containing the status for all monitors. Top level keys include all monitors. Within a given monitor, the value is a dictionary containing 'missing_file' and 'status' keys. 'missing_file' is a boolean describing whether or not there is a suspected missing log file based on the timestamps of the existing files. 'status' is a string that is either 'success' or 'failure'. """ # Begin logging logging.info("Beginning cron job status monitor") # Get main logfile path log_path = get_config()['log_dir'] # If we are in development mode, the log files are in a slightly # different location than in production mode if production_mode: log_path = os.path.join(log_path, 'prod') else: log_path = os.path.join(log_path, 'dev') # Set up a dictionary to keep track of results logfile_status = {} # Get a list of the directories under the main logging directory. generator = os.walk(log_path, topdown=True) # Loop over monitors for subdir, subsubdir, filenames in generator: # When running in production mode, skip the 'dev' subdirectory, # as it contains the development version of the monitor logs if production_mode: subsubdir[:] = [dirname for dirname in subsubdir if dirname != 'dev'] if len(filenames) > 0: monitor_name = subdir.split('/')[-1] # Avoid monitor_cron_jobs itseft if monitor_name != 'monitor_cron_jobs': log_file_list = [os.path.join(subdir, filename) for filename in filenames] # Find the cadence of the monitor delta_time, stdev_time = get_cadence(log_file_list) # Identify the most recent log file latest_log, latest_log_time = find_latest(log_file_list) # Check to see if we expect a file more recent than the latest missing_file = missing_file_check(delta_time, stdev_time, latest_log) if missing_file: logging.warning('Expected a more recent {} logfile than {}' .format(monitor_name, os.path.basename(latest_log))) # Check the file for success/failure result = success_check(latest_log) logging.info('{}: Latest log file indicates {}'.format(monitor_name, result)) # Add results to the dictionary logfile_status[monitor_name] = {'logname': os.path.basename(latest_log), 'latest_time': latest_log_time, 'missing_file': missing_file, 'status': result} # Create table of results using Bokeh create_table(logfile_status) logging.info('Cron job status monitor completed successfully.')
def get_image_info(file_root, rewrite): """Build and return a dictionary containing information for a given ``file_root``. Parameters ---------- file_root : str The rootname of the file of interest (e.g. ``jw86600008001_02101_00007_guider2``). rewrite : bool ``True`` if the corresponding JPEG needs to be rewritten, ``False`` if not. Returns ------- image_info : dict A dictionary containing various information for the given ``file_root``. """ # Initialize dictionary to store information image_info = {} image_info['all_jpegs'] = [] image_info['suffixes'] = [] image_info['num_ints'] = {} preview_dir = os.path.join(get_config()['jwql_dir'], 'preview_images') # Find all of the matching files dirname = file_root[:7] search_filepath = os.path.join(FILESYSTEM_DIR, dirname, file_root + '*.fits') image_info['all_files'] = glob.glob(search_filepath) for file in image_info['all_files']: # Get suffix information suffix = os.path.basename(file).split('_')[4].split('.')[0] image_info['suffixes'].append(suffix) # Determine JPEG file location jpg_dir = os.path.join(preview_dir, dirname) jpg_filename = os.path.basename( os.path.splitext(file)[0] + '_integ0.jpg') jpg_filepath = os.path.join(jpg_dir, jpg_filename) # Check that a jpg does not already exist. If it does (and rewrite=False), # just call the existing jpg file if os.path.exists(jpg_filepath) and not rewrite: pass # If it doesn't, make it using the preview_image module else: if not os.path.exists(jpg_dir): os.makedirs(jpg_dir) im = PreviewImage(file, 'SCI') im.output_directory = jpg_dir im.make_image() # Record how many integrations there are per filetype search_jpgs = os.path.join(preview_dir, dirname, file_root + '_{}_integ*.jpg'.format(suffix)) num_jpgs = len(glob.glob(search_jpgs)) image_info['num_ints'][suffix] = num_jpgs image_info['all_jpegs'].append(jpg_filepath) return image_info
from bokeh.palettes import Category20_20 as palette from bokeh.plotting import figure, output_file, save from jwql.database.database_interface import engine from jwql.database.database_interface import session from jwql.database.database_interface import FilesystemGeneral from jwql.database.database_interface import FilesystemInstrument from jwql.database.database_interface import CentralStore from jwql.utils.logging_functions import configure_logging, log_info, log_fail from jwql.utils.permissions import set_permissions from jwql.utils.constants import FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.utils import filename_parser from jwql.utils.utils import get_config from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table FILESYSTEM = get_config()['filesystem'] PROPRIETARY_FILESYSTEM = os.path.join(FILESYSTEM, 'proprietary') PUBLIC_FILESYSTEM = os.path.join(FILESYSTEM, 'public') CENTRAL = get_config()['jwql_dir'] def gather_statistics(general_results_dict, instrument_results_dict): """Walks the filesytem to gather various statistics to eventually store in the database Parameters ---------- general_results_dict : dict A dictionary for the ``filesystem_general`` database table instrument_results_dict : dict A dictionary for the ``filesystem_instrument`` database table