def test_directory(test_dir=TEST_DIRECTORY): """Create a test directory for preview image. Parameters ---------- test_dir : str Path to directory used for testing Yields ------- test_dir : str Path to directory used for testing """ # Set up local test directory ensure_dir_exists(test_dir) yield test_dir # Tear down local test directory and any files within if os.path.isdir(test_dir): shutil.rmtree(test_dir) # Empty test directory on central storage jpgs = glob.glob(os.path.join(get_config()['test_dir'], '*.jpg')) thumbs = glob.glob(os.path.join(get_config()['test_dir'], '*.thumbs')) for file in jpgs + thumbs: os.remove(file)
def make_log_file(module): """Create the log file name based on the module name. The name of the ``log_file`` is a combination of the name of the module being logged and the current datetime. Parameters ---------- module : str The name of the module being logged. production_mode : bool Whether or not the output should be written to the production environment. path : str Where to write the log if user-supplied path; default to working dir. Returns ------- log_file : str The full path to where the log file will be written to. """ # Build filename timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M') filename = '{0}_{1}.log'.format(module, timestamp) # Determine save location user = pwd.getpwuid(os.getuid()).pw_name admin_account = get_config()['admin_account'] log_path = get_config()['log_dir'] # For production if user == admin_account and socket.gethostname()[0] == 'p': log_file = os.path.join(log_path, 'prod', module, filename) # For test elif user == admin_account and socket.gethostname()[0] == 't': log_file = os.path.join(log_path, 'test', module, filename) # For dev elif user == admin_account and socket.gethostname()[0] == 'd': log_file = os.path.join(log_path, 'dev', module, filename) # For local (also write to dev) else: log_file = os.path.join(log_path, 'dev', module, filename) # Make sure parent directory exists ensure_dir_exists(os.path.dirname(log_file)) return log_file
def save_mean_slope_image(self, slope_img, stdev_img, files): """Save the mean slope image and associated stdev image to a file Parameters ---------- slope_img : numpy.ndarray 2D array containing the mean slope image stdev_img : numpy.ndarray 2D array containing the stdev image associated with the mean slope image. files : list List of input files used to construct the mean slope image Returns ------- output_filename : str Name of fits file to save mean and stdev images within """ output_filename = '{}_{}_{}_to_{}_mean_slope_image.fits'.format( self.instrument.lower(), self.aperture.lower(), self.query_start, self.query_end) mean_slope_dir = os.path.join(get_config()['outputs'], 'dark_monitor', 'mean_slope_images') ensure_dir_exists(mean_slope_dir) output_filename = os.path.join(mean_slope_dir, output_filename) logging.info("Name of mean slope image: {}".format(output_filename)) primary_hdu = fits.PrimaryHDU() primary_hdu.header['INSTRUME'] = (self.instrument, 'JWST instrument') primary_hdu.header['APERTURE'] = (self.aperture, 'Aperture name') primary_hdu.header['QRY_STRT'] = (self.query_start, 'MAST Query start time (MJD)') primary_hdu.header['QRY_END'] = (self.query_end, 'MAST Query end time (MJD)') files_string = 'FILES USED: ' for filename in files: files_string += '{}, '.format(filename) primary_hdu.header.add_history(files_string) mean_img_hdu = fits.ImageHDU(slope_img, name='MEAN') stdev_img_hdu = fits.ImageHDU(stdev_img, name='STDEV') hdu_list = fits.HDUList([primary_hdu, mean_img_hdu, stdev_img_hdu]) hdu_list.writeto(output_filename, overwrite=True) set_permissions(output_filename) return output_filename
def path_check(): """Check that the ``CRDS_PATH`` environment variable is set. This will be the location to which CRDS reference files are downloaded. If the env variable is not set, default to use ``$HOME/crds_cache/`` Returns ------- crds_path : str Full path to the location of the CRDS reference files """ crds_path = os.environ.get('CRDS_PATH') if crds_path is None: reffile_dir = '{}/crds_cache'.format(os.environ.get('HOME')) os.environ["CRDS_PATH"] = reffile_dir ensure_dir_exists(reffile_dir) print('CRDS_PATH environment variable not set. Setting to {}'.format(reffile_dir)) return reffile_dir else: return crds_path
def make_log_file(module, production_mode=True, path='./'): """Create the log file name based on the module name. The name of the ``log_file`` is a combination of the name of the module being logged and the current datetime. Parameters ---------- module : str The name of the module being logged. production_mode : bool Whether or not the output should be written to the production environment. path : str Where to write the log if user-supplied path; default to working dir. Returns ------- log_file : str The full path to where the log file will be written to. """ timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M') filename = '{0}_{1}.log'.format(module, timestamp) user = pwd.getpwuid(os.getuid()).pw_name settings = get_config() admin_account = settings['admin_account'] log_path = settings['log_dir'] exempt_modules = [] if user != admin_account and module not in exempt_modules and production_mode: module = os.path.join('dev', module) if production_mode: log_file = os.path.join(log_path, module, filename) else: log_file = os.path.join(path, filename) ensure_dir_exists(os.path.dirname(log_file)) return log_file
def run(self): """The main method. See module docstrings for further details. """ logging.info('Begin logging for readnoise_monitor\n') # Get the output directory and setup a directory to store the data self.output_dir = os.path.join(get_config()['outputs'], 'readnoise_monitor') ensure_dir_exists(os.path.join(self.output_dir, 'data')) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in ['nircam', 'niriss']: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures for this instrument siaf = Siaf(self.instrument) possible_apertures = list(siaf.apertures) for aperture in possible_apertures: logging.info('\nWorking on aperture {} in {}'.format( aperture, instrument)) self.aperture = aperture # Locate the record of the most recent MAST search; use this time # (plus a 30 day buffer to catch any missing files from the previous # run) as the start time in the new MAST search. most_recent_search = self.most_recent_search() self.query_start = most_recent_search - 30 # Query MAST for new dark files for this instrument/aperture logging.info('\tQuery times: {} {}'.format( self.query_start, self.query_end)) new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end) logging.info('\tAperture: {}, new entries: {}'.format( self.aperture, len(new_entries))) # Set up a directory to store the data for this aperture self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) if len(new_entries) > 0: ensure_dir_exists(self.data_dir) # Get any new files to process new_files = [] checked_files = [] for file_entry in new_entries: output_filename = os.path.join( self.data_dir, file_entry['filename'].replace('_dark', '_uncal')) # Sometimes both the dark and uncal name of a file is picked up in new_entries if output_filename in checked_files: logging.info( '\t{} already checked in this run.'.format( output_filename)) continue checked_files.append(output_filename) # Dont process files that already exist in the readnoise stats database file_exists = self.file_exists_in_database(output_filename) if file_exists: logging.info( '\t{} already exists in the readnoise database table.' .format(output_filename)) continue # Save any new uncal files with enough groups in the output directory; some dont exist in JWQL filesystem try: filename = filesystem_path(file_entry['filename']) uncal_filename = filename.replace('_dark', '_uncal') if not os.path.isfile(uncal_filename): logging.info( '\t{} does not exist in JWQL filesystem, even though {} does' .format(uncal_filename, filename)) else: num_groups = fits.getheader( uncal_filename)['NGROUPS'] if num_groups > 1: # skip processing if the file doesnt have enough groups to calculate the readnoise; TODO change to 10 before incorporating MIRI shutil.copy(uncal_filename, self.data_dir) logging.info('\tCopied {} to {}'.format( uncal_filename, output_filename)) set_permissions(output_filename) new_files.append(output_filename) else: logging.info( '\tNot enough groups to calculate readnoise in {}' .format(uncal_filename)) except FileNotFoundError: logging.info( '\t{} does not exist in JWQL filesystem'.format( file_entry['filename'])) # Run the readnoise monitor on any new files if len(new_files) > 0: self.process(new_files) monitor_run = True else: logging.info( '\tReadnoise monitor skipped. {} new dark files for {}, {}.' .format(len(new_files), instrument, aperture)) monitor_run = False # Update the query history new_entry = { 'instrument': instrument, 'aperture': aperture, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'entries_found': len(new_entries), 'files_found': len(new_files), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Readnoise Monitor completed successfully.')
def run(self): """The main method. See module docstrings for further details.""" logging.info('Begin logging for bias_monitor') # Get the output directory and setup a directory to store the data self.output_dir = os.path.join(get_config()['outputs'], 'bias_monitor') ensure_dir_exists(os.path.join(self.output_dir, 'data')) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in ['nircam', 'niriss', 'nirspec']: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible full-frame apertures for this instrument siaf = Siaf(self.instrument) possible_apertures = [ aperture for aperture in siaf.apertures if siaf[aperture].AperType == 'FULLSCA' ] for aperture in possible_apertures: logging.info('Working on aperture {} in {}'.format( aperture, instrument)) self.aperture = aperture # Locate the record of most recent MAST search; use this time # (plus a 30 day buffer to catch any missing files from # previous run) as the start time in the new MAST search. most_recent_search = self.most_recent_search() self.query_start = most_recent_search - 30 # Query MAST for new dark files for this instrument/aperture logging.info('\tQuery times: {} {}'.format( self.query_start, self.query_end)) new_entries = monitor_utils.mast_query_darks( instrument, aperture, self.query_start, self.query_end) # Exclude ASIC tuning data len_new_darks = len(new_entries) new_entries = monitor_utils.exclude_asic_tuning(new_entries) len_no_asic = len(new_entries) num_asic = len_new_darks - len_no_asic logging.info( "\tFiltering out ASIC tuning files removed {} dark files.". format(num_asic)) logging.info('\tAperture: {}, new entries: {}'.format( self.aperture, len(new_entries))) # Set up a directory to store the data for this aperture self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) if len(new_entries) > 0: ensure_dir_exists(self.data_dir) # Get any new files to process new_files = [] for file_entry in new_entries: output_filename = os.path.join(self.data_dir, file_entry['filename']) output_filename = output_filename.replace( '_uncal.fits', '_uncal_0thgroup.fits').replace( '_dark.fits', '_uncal_0thgroup.fits') # Dont process files that already exist in the bias stats database file_exists = self.file_exists_in_database(output_filename) if file_exists: logging.info( '\t{} already exists in the bias database table.'. format(output_filename)) continue # Save the 0th group image from each new file in the output directory; some dont exist in JWQL filesystem. try: filename = filesystem_path(file_entry['filename']) uncal_filename = filename.replace('_dark', '_uncal') if not os.path.isfile(uncal_filename): logging.info( '\t{} does not exist in JWQL filesystem, even though {} does' .format(uncal_filename, filename)) else: new_file = self.extract_zeroth_group( uncal_filename) new_files.append(new_file) except FileNotFoundError: logging.info( '\t{} does not exist in JWQL filesystem'.format( file_entry['filename'])) # Run the bias monitor on any new files if len(new_files) > 0: self.process(new_files) monitor_run = True else: logging.info( '\tBias monitor skipped. {} new dark files for {}, {}.' .format(len(new_files), instrument, aperture)) monitor_run = False # Update the query history new_entry = { 'instrument': instrument, 'aperture': aperture, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'entries_found': len(new_entries), 'files_found': len(new_files), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Bias Monitor completed successfully.')
def get_edb_components(request): """Return dictionary with content needed for the EDB page. Parameters ---------- request : HttpRequest object Incoming request from the webpage Returns ------- edb_components : dict Dictionary with the required components """ mnemonic_name_search_result = {} mnemonic_query_result = {} mnemonic_query_result_plot = None mnemonic_exploration_result = None # If this is a POST request, we need to process the form data if request.method == 'POST': if 'mnemonic_name_search' in request.POST.keys(): # authenticate with astroquery.mast if necessary logged_in = log_into_mast(request) mnemonic_name_search_form = MnemonicSearchForm( request.POST, logged_in=logged_in, prefix='mnemonic_name_search') if mnemonic_name_search_form.is_valid(): mnemonic_identifier = mnemonic_name_search_form[ 'search'].value() if mnemonic_identifier is not None: mnemonic_name_search_result = get_mnemonic_info( mnemonic_identifier) # create forms for search fields not clicked mnemonic_query_form = MnemonicQueryForm(prefix='mnemonic_query') mnemonic_exploration_form = MnemonicExplorationForm( prefix='mnemonic_exploration') elif 'mnemonic_query' in request.POST.keys(): # authenticate with astroquery.mast if necessary logged_in = log_into_mast(request) mnemonic_query_form = MnemonicQueryForm(request.POST, logged_in=logged_in, prefix='mnemonic_query') # proceed only if entries make sense if mnemonic_query_form.is_valid(): mnemonic_identifier = mnemonic_query_form['search'].value() start_time = Time(mnemonic_query_form['start_time'].value(), format='iso') end_time = Time(mnemonic_query_form['end_time'].value(), format='iso') if mnemonic_identifier is not None: mnemonic_query_result = get_mnemonic( mnemonic_identifier, start_time, end_time) mnemonic_query_result_plot = mnemonic_query_result.bokeh_plot( ) # generate table download in web app result_table = mnemonic_query_result.data # save file locally to be available for download static_dir = os.path.join(settings.BASE_DIR, 'static') ensure_dir_exists(static_dir) file_name_root = 'mnemonic_query_result_table' file_for_download = '{}.csv'.format(file_name_root) path_for_download = os.path.join(static_dir, file_for_download) # add meta data to saved table comments = [] comments.append( 'DMS EDB query of {}:'.format(mnemonic_identifier)) for key, value in mnemonic_query_result.info.items(): comments.append('{} = {}'.format(key, str(value))) result_table.meta['comments'] = comments comments.append(' ') comments.append('Start time {}'.format(start_time.isot)) comments.append('End time {}'.format(end_time.isot)) comments.append('Number of rows {}'.format( len(result_table))) comments.append(' ') result_table.write(path_for_download, format='ascii.fixed_width', overwrite=True, delimiter=',', bookend=False) mnemonic_query_result.file_for_download = file_for_download # create forms for search fields not clicked mnemonic_name_search_form = MnemonicSearchForm( prefix='mnemonic_name_search') mnemonic_exploration_form = MnemonicExplorationForm( prefix='mnemonic_exploration') elif 'mnemonic_exploration' in request.POST.keys(): mnemonic_exploration_form = MnemonicExplorationForm( request.POST, prefix='mnemonic_exploration') if mnemonic_exploration_form.is_valid(): mnemonic_exploration_result, meta = mnemonic_inventory() # loop over filled fields and implement simple AND logic for field in mnemonic_exploration_form.fields: field_value = mnemonic_exploration_form[field].value() if field_value != '': column_name = mnemonic_exploration_form[field].label # matching indices in table (case-insensitive) index = [ i for i, item in enumerate( mnemonic_exploration_result[column_name]) if re.search(field_value, item, re.IGNORECASE) ] mnemonic_exploration_result = mnemonic_exploration_result[ index] mnemonic_exploration_result.n_rows = len( mnemonic_exploration_result) # generate tables for display and download in web app display_table = copy.deepcopy(mnemonic_exploration_result) # temporary html file, # see http://docs.astropy.org/en/stable/_modules/astropy/table/ tmpdir = tempfile.mkdtemp() file_name_root = 'mnemonic_exploration_result_table' path_for_html = os.path.join(tmpdir, '{}.html'.format(file_name_root)) with open(path_for_html, 'w') as tmp: display_table.write(tmp, format='jsviewer') mnemonic_exploration_result.html_file_content = open( path_for_html, 'r').read() # pass on meta data to have access to total number of mnemonics mnemonic_exploration_result.meta = meta # save file locally to be available for download static_dir = os.path.join(settings.BASE_DIR, 'static') ensure_dir_exists(static_dir) file_for_download = '{}.csv'.format(file_name_root) path_for_download = os.path.join(static_dir, file_for_download) display_table.write(path_for_download, format='ascii.fixed_width', overwrite=True, delimiter=',', bookend=False) mnemonic_exploration_result.file_for_download = file_for_download if mnemonic_exploration_result.n_rows == 0: mnemonic_exploration_result = 'empty' # create forms for search fields not clicked mnemonic_name_search_form = MnemonicSearchForm( prefix='mnemonic_name_search') mnemonic_query_form = MnemonicQueryForm(prefix='mnemonic_query') else: mnemonic_name_search_form = MnemonicSearchForm( prefix='mnemonic_name_search') mnemonic_query_form = MnemonicQueryForm(prefix='mnemonic_query') mnemonic_exploration_form = MnemonicExplorationForm( prefix='mnemonic_exploration') edb_components = { 'mnemonic_query_form': mnemonic_query_form, 'mnemonic_query_result': mnemonic_query_result, 'mnemonic_query_result_plot': mnemonic_query_result_plot, 'mnemonic_name_search_form': mnemonic_name_search_form, 'mnemonic_name_search_result': mnemonic_name_search_result, 'mnemonic_exploration_form': mnemonic_exploration_form, 'mnemonic_exploration_result': mnemonic_exploration_result } return edb_components
def run(self): """The main method. See module docstrings for further details. """ logging.info('Begin logging for dark_monitor') apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL'] # Get the output directory self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor') # Read in config file that defines the thresholds for the number # of dark files that must be present in order for the monitor to run limits = ascii.read(THRESHOLDS_FILE) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures from pysiaf possible_apertures = list(Siaf(instrument).apernames) possible_apertures = [ ap for ap in possible_apertures if ap not in apertures_to_skip ] for aperture in possible_apertures: logging.info('') logging.info('Working on aperture {} in {}'.format( aperture, instrument)) # Find the appropriate threshold for the number of new files needed match = aperture == limits['Aperture'] file_count_threshold = limits['Threshold'][match] # Locate the record of the most recent MAST search self.aperture = aperture self.query_start = self.most_recent_search() logging.info('\tQuery times: {} {}'.format( self.query_start, self.query_end)) # Query MAST using the aperture and the time of the # most recent previous search as the starting time new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end) logging.info('\tAperture: {}, new entries: {}'.format( self.aperture, len(new_entries))) # Check to see if there are enough new files to meet the # monitor's signal-to-noise requirements if len(new_entries) >= file_count_threshold: logging.info( '\tSufficient new dark files found for {}, {} to run the dark monitor.' .format(self.instrument, self.aperture)) # Get full paths to the files new_filenames = [] for file_entry in new_entries: try: new_filenames.append( filesystem_path(file_entry['filename'])) except FileNotFoundError: logging.warning( '\t\tUnable to locate {} in filesystem. Not including in processing.' .format(file_entry['filename'])) # Set up directories for the copied data ensure_dir_exists(os.path.join(self.output_dir, 'data')) self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) ensure_dir_exists(self.data_dir) # Copy files from filesystem dark_files, not_copied = copy_files( new_filenames, self.data_dir) logging.info('\tNew_filenames: {}'.format(new_filenames)) logging.info('\tData dir: {}'.format(self.data_dir)) logging.info( '\tCopied to working dir: {}'.format(dark_files)) logging.info('\tNot copied: {}'.format(not_copied)) # Run the dark monitor self.process(dark_files) monitor_run = True else: logging.info(( '\tDark monitor skipped. {} new dark files for {}, {}. {} new files are ' 'required to run dark current monitor.').format( len(new_entries), instrument, aperture, file_count_threshold[0])) monitor_run = False # Update the query history new_entry = { 'instrument': instrument, 'aperture': aperture, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'files_found': len(new_entries), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Dark Monitor completed successfully.')
def run(self): """The main method. See module docstrings for further details. There are 2 parts to the bad pixel monitor: 1. Bad pixels from illuminated data 2. Bad pixels from dark data For each, we will query MAST, copy new files from the filesystem and pass the list of copied files into the ``process()`` method. """ logging.info('Begin logging for bad_pixel_monitor') # Get the output directory self.output_dir = os.path.join(get_config()['outputs'], 'bad_pixel_monitor') # Read in config file that defines the thresholds for the number # of dark files that must be present in order for the monitor to run limits = ascii.read(THRESHOLDS_FILE) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures from pysiaf possible_apertures = self.get_possible_apertures() for aperture in possible_apertures: grating = None detector_name = None lamp = None # NIRSpec flats use the MIRROR grating. if self.instrument == 'nirspec': grating = 'MIRROR' # MIRI is unlike the other instruments. We basically treat # the detector as the aperture name because there is no # aperture name for a full frame MRS exposure. if self.instrument == 'miri': detector_name, aperture_name = aperture self.aperture = detector_name else: self.aperture = aperture aperture_name = aperture # In flight, NIRISS plans to take darks using the LINE2 lamp if self.instrument == 'niriss': lamp = 'LINE2' # What lamp is most appropriate for NIRSpec? if self.instrument == 'nirspec': lamp = 'LINE2' # What lamp is most appropriate for FGS? #if self.instrument == 'fgs': # lamp = 'G2LAMP1' logging.info('') logging.info('Working on aperture {} in {}'.format(aperture, self.instrument)) # Find the appropriate threshold for the number of new files needed match = self.aperture == limits['Aperture'] flat_file_count_threshold = limits['FlatThreshold'][match].data[0] dark_file_count_threshold = limits['DarkThreshold'][match].data[0] # Locate the record of the most recent MAST search self.flat_query_start = self.most_recent_search(file_type='flat') self.dark_query_start = self.most_recent_search(file_type='dark') logging.info('\tFlat field query times: {} {}'.format(self.flat_query_start, self.query_end)) logging.info('\tDark current query times: {} {}'.format(self.dark_query_start, self.query_end)) # Query MAST using the aperture and the time of the most # recent previous search as the starting time. flat_templates = FLAT_EXP_TYPES[instrument] dark_templates = DARK_EXP_TYPES[instrument] new_flat_entries = mast_query(instrument, flat_templates, self.flat_query_start, self.query_end, aperture=aperture_name, grating=grating, detector=detector_name, lamp=lamp) new_dark_entries = mast_query(instrument, dark_templates, self.dark_query_start, self.query_end, aperture=aperture_name, detector=detector_name) # Filter the results # Filtering could be different for flats vs darks. # Kevin says we shouldn't need to worry about mixing lamps in the data used to create the bad pixel # mask. In flight, data will only be taken with LINE2, LEVEL 5. Currently in MAST all lamps are # present, but Kevin is not concerned about variations in flat field strucutre. # NIRISS - results can include rate, rateints, trapsfilled # MIRI - Jane says they now use illuminated data for dead pixel checks, just like other insts. # NIRSpec - can be cal, x1d, rate, rateints. Can have both cal and x1d so filter repeats # FGS - rate, rateints, trapsfilled # NIRCam - no int flats # The query results can contain multiple entries for files # in different calibration states (or for different output # products), so we need to filter the list for duplicate # entries and for the calibration state we are interested # in before we know how many new entries there really are. # In the end, we need rate files as well as uncal files # because we're going to need to create jump files. # In order to use a given file we must have at least the # uncal version of the file. Get the uncal and rate file # lists to align. if new_flat_entries: new_flat_entries = self.filter_query_results(new_flat_entries, datatype='flat') flat_uncal_files = locate_uncal_files(new_flat_entries) flat_uncal_files, run_flats = check_for_sufficient_files(flat_uncal_files, instrument, aperture, flat_file_count_threshold, 'flats') flat_rate_files, flat_rate_files_to_copy = locate_rate_files(flat_uncal_files) else: run_flats = False flat_uncal_files, flat_rate_files, flat_rate_files_to_copy = None, None, None if new_dark_entries: new_dark_entries = self.filter_query_results(new_dark_entries, datatype='dark') dark_uncal_files = locate_uncal_files(new_dark_entries) dark_uncal_files, run_darks = check_for_sufficient_files(dark_uncal_files, instrument, aperture, dark_file_count_threshold, 'darks') dark_rate_files, dark_rate_files_to_copy = locate_rate_files(dark_uncal_files) else: run_darks = False dark_uncal_files, dark_rate_files, dark_rate_files_to_copy = None, None, None # Set up directories for the copied data ensure_dir_exists(os.path.join(self.output_dir, 'data')) self.data_dir = os.path.join(self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) ensure_dir_exists(self.data_dir) # Copy files from filesystem if run_flats: flat_uncal_files, flat_rate_files = self.map_uncal_and_rate_file_lists(flat_uncal_files, flat_rate_files, flat_rate_files_to_copy, 'flat') if run_darks: dark_uncal_files, dark_rate_files = self.map_uncal_and_rate_file_lists(dark_uncal_files, dark_rate_files, dark_rate_files_to_copy, 'dark') # Run the bad pixel monitor if run_flats or run_darks: self.process(flat_uncal_files, flat_rate_files, dark_uncal_files, dark_rate_files) # Update the query history if dark_uncal_files is None: num_dark_files = 0 else: num_dark_files = len(dark_uncal_files) if flat_uncal_files is None: num_flat_files = 0 else: num_flat_files = len(flat_uncal_files) new_entry = {'instrument': self.instrument.upper(), 'aperture': self.aperture, 'dark_start_time_mjd': self.dark_query_start, 'dark_end_time_mjd': self.query_end, 'flat_start_time_mjd': self.flat_query_start, 'flat_end_time_mjd': self.query_end, 'dark_files_found': num_dark_files, 'flat_files_found': num_flat_files, 'run_bpix_from_darks': run_darks, 'run_bpix_from_flats': run_flats, 'run_monitor': run_flats or run_darks, 'entry_date': datetime.datetime.now()} self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Bad Pixel Monitor completed successfully.')
def create_table(status_dict): """Create interactive ``bokeh`` table containing the logfile status results. Parameters ---------- status_dict : dict Nested dictionary with status results from all logfiles """ # Rearrange the nested dictionary into a non-nested dict for the table filenames = [] dates = [] missings = [] results = [] for key in status_dict: filenames.append(status_dict[key]['logname']) dates.append(datetime.fromtimestamp(status_dict[key]['latest_time'])) missings.append(str(status_dict[key]['missing_file'])) results.append(status_dict[key]['status']) # div to color the boxes in the status column success_template = """ <div style="background:<%= (function colorfromstr(){ if(value == "success"){ return("green")} else{return("red")} }()) %>; color: white"> <%= value %></div> """ # div to color the boxes in the column for possibly late logfiles missing_template = """ <div style="background:<%= (function colorfrombool(){ if(value == "True"){ return("orange")} else{return("green")} }()) %>; color: white"> <%= value %></div> """ success_formatter = HTMLTemplateFormatter(template=success_template) missing_formatter = HTMLTemplateFormatter(template=missing_template) data = dict(name=list(status_dict.keys()), filename=filenames, date=dates, missing=missings, result=results) source = ColumnDataSource(data) datefmt = DateFormatter(format="RFC-2822") columns = [ TableColumn(field="name", title="Monitor Name", width=200), TableColumn(field="filename", title="Most Recent File", width=350), TableColumn(field="date", title="Most Recent Time", width=200, formatter=datefmt), TableColumn(field="missing", title="Possible Missing File", width=200, formatter=missing_formatter), TableColumn(field="result", title="Status", width=100, formatter=success_formatter), ] data_table = DataTable(source=source, columns=columns, width=800, height=280, index_position=None) # Get output directory for saving the table files output_dir = SETTINGS['outputs'] output_filename = 'cron_status_table' # verify/create output sub-directory output_dir = os.path.join(output_dir, 'monitor_cron_jobs') ensure_dir_exists(output_dir) # Save full html html_outfile = os.path.join(output_dir, '{}.html'.format(output_filename)) output_file(html_outfile) save(data_table) try: set_permissions(html_outfile) except PermissionError: logging.warning( 'Unable to set permissions for {}'.format(html_outfile)) logging.info('Saved Bokeh full HTML file: {}'.format(html_outfile))
def run(self): """The main method. See module docstrings for further details. """ logging.info('Begin logging for dark_monitor') apertures_to_skip = ['NRCALL_FULL', 'NRCAS_FULL', 'NRCBS_FULL'] # Get the output directory self.output_dir = os.path.join(get_config()['outputs'], 'dark_monitor') # Read in config file that defines the thresholds for the number # of dark files that must be present in order for the monitor to run limits = ascii.read(THRESHOLDS_FILE) # Use the current time as the end time for MAST query self.query_end = Time.now().mjd # Loop over all instruments for instrument in JWST_INSTRUMENT_NAMES: self.instrument = instrument # Identify which database tables to use self.identify_tables() # Get a list of all possible apertures from pysiaf possible_apertures = list(Siaf(instrument).apernames) possible_apertures = [ ap for ap in possible_apertures if ap not in apertures_to_skip ] # Get a list of all possible readout patterns associated with the aperture possible_readpatts = RAPID_READPATTERNS[instrument] for aperture in possible_apertures: logging.info('') logging.info('Working on aperture {} in {}'.format( aperture, instrument)) # Find appropriate threshold for the number of new files needed match = aperture == limits['Aperture'] # If the aperture is not listed in the threshold file, we need # a default if not np.any(match): file_count_threshold = 30 logging.warning(( '\tAperture {} is not present in the threshold file. Continuing ' 'with the default threshold of 30 files.'.format( aperture))) else: file_count_threshold = limits['Threshold'][match][0] self.aperture = aperture # We need a separate search for each readout pattern for readpatt in possible_readpatts: self.readpatt = readpatt logging.info('\tWorking on readout pattern: {}'.format( self.readpatt)) # Locate the record of the most recent MAST search self.query_start = self.most_recent_search() logging.info('\tQuery times: {} {}'.format( self.query_start, self.query_end)) # Query MAST using the aperture and the time of the # most recent previous search as the starting time new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt) logging.info( '\tAperture: {}, Readpattern: {}, new entries: {}'. format(self.aperture, self.readpatt, len(new_entries))) # Check to see if there are enough new files to meet the # monitor's signal-to-noise requirements if len(new_entries) >= file_count_threshold: logging.info( '\tMAST query has returned sufficient new dark files for {}, {}, {} to run the dark monitor.' .format(self.instrument, self.aperture, self.readpatt)) # Get full paths to the files new_filenames = [] for file_entry in new_entries: try: new_filenames.append( filesystem_path(file_entry['filename'])) except FileNotFoundError: logging.warning( '\t\tUnable to locate {} in filesystem. Not including in processing.' .format(file_entry['filename'])) # In some (unusual) cases, there are files in MAST with the correct aperture name # but incorrect array sizes. Make sure that the new files all have the expected # aperture size temp_filenames = [] bad_size_filenames = [] expected_ap = Siaf(instrument)[aperture] expected_xsize = expected_ap.XSciSize expected_ysize = expected_ap.YSciSize for new_file in new_filenames: with fits.open(new_file) as hdulist: xsize = hdulist[0].header['SUBSIZE1'] ysize = hdulist[0].header['SUBSIZE2'] if xsize == expected_xsize and ysize == expected_ysize: temp_filenames.append(new_file) else: bad_size_filenames.append(new_file) if len(temp_filenames) != len(new_filenames): logging.info( '\tSome files returned by MAST have unexpected aperture sizes. These files will be ignored: ' ) for badfile in bad_size_filenames: logging.info('\t\t{}'.format(badfile)) new_filenames = deepcopy(temp_filenames) # If it turns out that the monitor doesn't find enough # of the files returned by the MAST query to meet the threshold, # then the monitor will not be run if len(new_filenames) < file_count_threshold: logging.info(( "\tFilesystem search for the files identified by MAST has returned {} files. " "This is less than the required minimum number of files ({}) necessary to run " "the monitor. Quitting.").format( len(new_filenames), file_count_threshold)) monitor_run = False else: logging.info(( "\tFilesystem search for the files identified by MAST has returned {} files." ).format(len(new_filenames))) monitor_run = True if monitor_run: # Set up directories for the copied data ensure_dir_exists( os.path.join(self.output_dir, 'data')) self.data_dir = os.path.join( self.output_dir, 'data/{}_{}'.format(self.instrument.lower(), self.aperture.lower())) ensure_dir_exists(self.data_dir) # Copy files from filesystem dark_files, not_copied = copy_files( new_filenames, self.data_dir) logging.info( '\tNew_filenames: {}'.format(new_filenames)) logging.info('\tData dir: {}'.format( self.data_dir)) logging.info('\tCopied to working dir: {}'.format( dark_files)) logging.info('\tNot copied: {}'.format(not_copied)) # Run the dark monitor self.process(dark_files) else: logging.info(( '\tDark monitor skipped. MAST query has returned {} new dark files for ' '{}, {}, {}. {} new files are required to run dark current monitor.' ).format(len(new_entries), instrument, aperture, self.readpatt, file_count_threshold)) monitor_run = False # Update the query history new_entry = { 'instrument': instrument, 'aperture': aperture, 'readpattern': self.readpatt, 'start_time_mjd': self.query_start, 'end_time_mjd': self.query_end, 'files_found': len(new_entries), 'run_monitor': monitor_run, 'entry_date': datetime.datetime.now() } self.query_table.__table__.insert().execute(new_entry) logging.info('\tUpdated the query history table') logging.info('Dark Monitor completed successfully.')