def mast_authenticate(token=None): """Verify MAST authentication status, login if needed.""" if Mast.authenticated() is False: if token is None: raise ValueError( 'You are not authenticated in MAST. Please provide a valid token.' ) else: Mast.login(token=token)
def test_invalid_query(): """Test that the mnemonic query for an unauthorized user fails.""" Mast.logout() mnemonic_identifier = 'IMIR_HK_ICE_SEC_VOLT4' start_time = Time('2019-01-16 00:00:00.000', format='iso') end_time = Time('2019-01-16 00:01:00.000', format='iso') try: query_single_mnemonic(mnemonic_identifier, start_time, end_time, token='1234') except RuntimeError: pass
def test_invalid_query(): """Test that the mnemonic query for an unauthorized user fails.""" Mast.logout() mnemonic_identifier = 'IMIR_HK_ICE_SEC_VOLT4' start_time = Time('2019-01-16 00:00:00.000', format='iso') end_time = Time('2019-01-16 00:01:00.000', format='iso') with pytest.raises((ValueError, RuntimeError)) as error_message: query_single_mnemonic(mnemonic_identifier, start_time, end_time, token='1234') assert 'You are not authenticated in MAST' in str(error_message.value) or \ 'Must be logged in as an authorized user' in str(error_message.value)
def get_mast_token(request=None): """Return MAST token from either Astroquery.Mast, webpage cookies, the JWQL configuration file, or an environment variable. Parameters ---------- request : HttpRequest object Incoming request from the webpage Returns ------- token : str or None User-specific MAST token string, if available """ if Mast.authenticated(): print('Authenticated with Astroquery MAST magic') return None else: try: # check if token is available via config file check_config_for_key('mast_token') token = get_config()['mast_token'] print('Authenticated with config.json MAST token.') return token except (KeyError, ValueError): # check if token is available via environment variable # see https://auth.mast.stsci.edu/info try: token = os.environ['MAST_API_TOKEN'] print('Authenticated with MAST token environment variable.') return token except KeyError: return None
def mast_request(version): """Access MAST and get JSON version of the keyword dictionary. Parameters ---------- version : string Keyword dictionary version Returns ------- kwd_dict : json object Keyword dictionary as a JSON nested dictionary object """ service = "Mast.Jwst.Keywords" params = { "filename": "all.json", "version": str(version), "cacheBreaker": time() } resp = Mast.service_request_async(service, params) kwd_dict = resp[0].json() return kwd_dict
def test_astroquery_mast(): """Test if the astroquery.mast service can complete a request""" service = 'Mast.Caom.Filtered' params = {'columns': 'COUNT_BIG(*)', 'filters': [], 'pagesize': 1, 'page': 1} response = Mast.service_request_async(service, params) result = response[0].json() assert result['status'] == 'COMPLETE'
def query_single_mnemonic(mnemonic_identifier, start_time, end_time, token=None): """Query DMS EDB to get the mnemonic readings in a time interval. The underlying MAST service returns data that include the datapoint preceding the requested start time and the datapoint that follows the requested end time. Parameters ---------- mnemonic_identifier : str Telemetry mnemonic identifier, e.g. 'SA_ZFGOUTFOV' start_time : astropy.time.Time instance Start time end_time : astropy.time.Time instance End time token : str MAST token Returns ------- data, meta, info : tuple Table and two dictionaries with the results of the query """ mast_authenticate(token=token) if not is_valid_mnemonic(mnemonic_identifier): raise RuntimeError('Mnemonic identifier is invalid!') if not isinstance(start_time, Time): raise RuntimeError( 'Please specify a valid start time (instance of astropy.time.core.Time)' ) if not isinstance(end_time, Time): raise RuntimeError( 'Please specify a valid end time (instance of astropy.time.core.Time)' ) parameters = { 'mnemonic': mnemonic_identifier, 'start': start_time.iso, 'end': end_time.iso } result = Mast.service_request_async(mast_edb_timeseries_service, parameters) data, meta = process_mast_service_request_result(result) # get auxiliary information (description, subsystem, ...) info = query_mnemonic_info(mnemonic_identifier) return data, meta, info
def log_into_mast(request): """Login via astroquery.mast if user authenticated in web app. Parameters ---------- request : HttpRequest object Incoming request from the webpage """ if Mast.authenticated(): return True # get the MAST access token if present access_token = str(get_mast_token(request)) # authenticate with astroquery.mast if necessary if access_token != 'None': Mast.login(token=access_token) return Mast.authenticated() else: return False
def mnemonic_inventory(): """Return all mnemonics in the DMS engineering database. Returns ------- data : astropy.table.Table Table representation of the mnemonic inventory. meta : dict Additional information returned by the query. """ out = Mast.service_request_async(mast_edb_mnemonic_service, {}) data, meta = process_mast_service_request_result(out) return data, meta
def get_thumbnails_all_instruments(instruments): """Return a list of thumbnails available in the filesystem for all instruments given requested parameters. Returns ------- thumbnails : list A list of thumbnails available in the filesystem for the given instrument. """ # Make sure instruments are of the proper format (e.g. "Nircam") thumbnail_list = [] for inst in instruments: # JWST_INSTRUMENT_NAMES: instrument = inst[0].upper() + inst[1:].lower() ### adjust query based on request # Query MAST for all rootnames for the instrument service = "Mast.Jwst.Filtered.{}".format(instrument) params = { "columns": "filename, expstart, filter, readpatt, date_beg, date_end, apername, exp_type", "filters": [{ "paramName": "expstart", "values": [{ "min": 57404.04, "max": 57404.07 }], }] } response = Mast.service_request_async(service, params) results = response[0].json()['data'] # Parse the results to get the rootnames filenames = [result['filename'].split('.')[0] for result in results] # Get list of all thumbnails thumbnails = glob.glob( os.path.join(THUMBNAIL_FILESYSTEM, '*', '*.thumb')) thumbnail_list.extend(thumbnails) # Get subset of preview images that match the filenames thumbnails = [ os.path.basename(item) for item in thumbnail_list if os.path.basename(item).split('_integ')[0] in filenames ] return thumbnails
def query_single_mnemonic(mnemonic_identifier, start_time, end_time): """Query DMS EDB to get the mnemonic readings in a time interval. Parameters ---------- mnemonic_identifier : str Telemetry mnemonic identifier, e.g. 'SA_ZFGOUTFOV' start_time : astropy.time.Time instance Start time end_time : astropy.time.Time instance End time Returns ------- mnemonic : instance of EdbMnemonic Object that contains the returned data """ if not isinstance(start_time, astropy.time.core.Time): raise RuntimeError( 'Please specify a valid start time (instance of astropy.time.core.Time)' ) if not isinstance(end_time, astropy.time.core.Time): raise RuntimeError( 'Please specify a valid end time (instance of astropy.time.core.Time)' ) if not isinstance(mnemonic_identifier, str): raise RuntimeError('Please provide mnemonic_identifier of type string') parameters = { 'mnemonic': mnemonic_identifier, 'start': start_time.iso, 'end': end_time.iso } result = Mast.service_request_async(mast_edb_timeseries_service, parameters) data, meta = process_mast_service_request_result(result) # get auxiliary information (description, subsystem, ...) info = query_mnemonic_info(mnemonic_identifier) # create and return instance mnemonic = EdbMnemonic(mnemonic_identifier, start_time, end_time, data, meta, info) return mnemonic
def get_filenames_by_instrument(instrument, restriction='all'): """Returns a list of filenames that match the given ``instrument``. Parameters ---------- instrument : str The instrument of interest (e.g. `FGS`). restriction : str If ``all``, all filenames will be returned. If ``public``, only publicly-available filenames will be returned. If ``proprietary``, only proprietary filenames will be returned. Returns ------- filenames : list A list of files that match the given instrument. """ service = INSTRUMENT_SERVICE_MATCH[instrument] # Query for filenames params = {"columns": "filename, isRestricted", "filters": []} response = Mast.service_request_async(service, params) result = response[0].json() # Determine filenames to return based on restriction parameter if restriction == 'all': filenames = [item['filename'] for item in result['data']] elif restriction == 'public': filenames = [ item['filename'] for item in result['data'] if item['isRestricted'] is False ] elif restriction == 'proprietary': filenames = [ item['filename'] for item in result['data'] if item['isRestricted'] is True ] else: raise KeyError( '{} is not a valid restriction level. Use "all", "public", or "proprietary".' .format(restriction)) return filenames
def get_files_for_instrument(self, instrument): """Given an instrument, query the database for all filenames and paths associated with said instrument Parameters ---------- instrument : str Name of JWST instrument Returns ------- filepaths: list List of all filepaths in database for the provided instrument filenames: list List of all filenames in database for the provided instrument """ instrument = instrument.upper() if self.db_type == 'SQL': results = self.session.query(self.ObservationWebtest)\ .filter(self.ObservationWebtest.instrument == instrument) elif self.db_type == 'MAST': params = {"columns": "*", "filters": []} response = Mast.service_request_async(self.service, params) results = response[0].json()['data'] filepaths = [] filenames = [] for i in results: if self.db_type == 'SQL': filename = i.filename elif self.db_type == 'MAST': filename = i['filename'] prog_id = filename[2:7] file_path = os.path.join('jw' + prog_id, filename) filepaths.append(file_path) filenames.append(filename) session.close() return filepaths, filenames
def query_mnemonic_info(mnemonic_identifier): """Query the EDB to return the mnemonic description. Parameters ---------- mnemonic_identifier : str Telemetry mnemonic identifier, e.g. ``SA_ZFGOUTFOV`` Returns ------- info : dict Object that contains the returned data """ parameters = {"mnemonic": "{}".format(mnemonic_identifier)} result = Mast.service_request_async(mast_edb_dictionary_service, parameters) info = process_mast_service_request_result(result, data_as_table=False)[0] return info
def get_preview_images_by_instrument(inst): """Return a list of preview images available in the filesystem for the given instrument. Parameters ---------- inst : str The instrument of interest (e.g. ``NIRCam``). Returns ------- preview_images : list A list of preview images available in the filesystem for the given instrument. """ # Make sure the instrument is of the proper format (e.g. "Nircam") instrument = inst[0].upper() + inst[1:].lower() # Query MAST for all rootnames for the instrument service = "Mast.Jwst.Filtered.{}".format(instrument) params = {"columns": "filename", "filters": []} response = Mast.service_request_async(service, params) results = response[0].json()['data'] # Parse the results to get the rootnames filenames = [result['filename'].split('.')[0] for result in results] # Get list of all preview_images preview_images = glob.glob( os.path.join(PREVIEW_IMAGE_FILESYSTEM, '*', '*.jpg')) # Get subset of preview images that match the filenames preview_images = [ os.path.basename(item) for item in preview_images if os.path.basename(item).split('_integ')[0] in filenames ] # Return only return preview_images
def mnemonic_inventory(): """Return all mnemonics in the DMS engineering database. No authentication is required, this information is public. Since this is a rather large and quasi-static table (~15000 rows), it is cached using functools. Returns ------- data : astropy.table.Table Table representation of the mnemonic inventory. meta : dict Additional information returned by the query. """ out = Mast.service_request_async(MAST_EDB_MNEMONIC_SERVICE, {}) data, meta = process_mast_service_request_result(out) # convert numerical ID to str for homogenity (all columns are str) data['tlmIdentifier'] = data['tlmIdentifier'].astype(str) return data, meta
def get_instrument_proposals(instrument): """Return a list of proposals for the given instrument Parameters ---------- instrument : str Name of the JWST instrument Returns ------- proposals : list List of proposals for the given instrument """ service = "Mast.Jwst.Filtered.{}".format(instrument) params = {"columns": "program", "filters": []} response = Mast.service_request_async(service, params) results = response[0].json()['data'] proposals = list(set(result['program'] for result in results)) return proposals
def get_expstart(instrument, rootname): """Return the exposure start time (``expstart``) for the given ``rootname``. The ``expstart`` is gathered from a query to the ``astroquery.mast`` service. Parameters ---------- instrument : str The instrument of interest (e.g. `FGS`). rootname : str The rootname of the observation of interest (e.g. ``jw86700006001_02101_00006_guider1``). Returns ------- expstart : float The exposure start time of the observation (in MJD). """ file_set_name = '_'.join(rootname.split('_')[:-1]) service = INSTRUMENT_SERVICE_MATCH[instrument] params = { 'columns': 'filename, expstart', 'filters': [{ 'paramName': 'fileSetName', 'values': [file_set_name] }] } response = Mast.service_request_async(service, params) result = response[0].json() if result['data'] == []: expstart = 0 print("WARNING: no data") else: expstart = min([item['expstart'] for item in result['data']]) return expstart
def query_mnemonic_info(mnemonic_identifier, token=None): """Query the EDB to return the mnemonic description. Parameters ---------- mnemonic_identifier : str Telemetry mnemonic identifier, e.g. ``SA_ZFGOUTFOV`` token : str MAST token Returns ------- info : dict Object that contains the returned data """ parameters = {"mnemonic": "{}".format(mnemonic_identifier)} result = Mast.service_request_async(MAST_EDB_DICTIONARY_SERVICE, parameters) info = process_mast_service_request_result(result, data_as_table=False)[0] return info
def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters=None, add_requests=None, caom=False, return_data=False): """Get the counts for a given instrument and data product Parameters ---------- instrument: str The instrument name, i.e. one of ['niriss','nircam','nirspec', 'miri','fgs'] dataproduct: sequence, str The type of data product to search add_filters: dict The ('paramName':'values') pairs to include in the 'filters' argument of the request e.g. add_filters = {'filter':'GR150R'} add_requests: dict The ('request':'value') pairs to include in the request e.g. add_requests = {'pagesize':1, 'page':1} caom: bool Query CAOM service return_data: bool Return the actual data instead of counts only Returns ------- int, dict The number of database records that satisfy the search criteria or a dictionary of the data if `return_data=True` """ filters = [] # Make sure the dataproduct is a list if isinstance(dataproduct, str): dataproduct = [dataproduct] # Make sure the instrument is supported if instrument.lower() not in [ ins.lower() for ins in JWST_INSTRUMENT_NAMES ]: raise TypeError('Supported instruments include:', JWST_INSTRUMENT_NAMES) # CAOM service if caom: # Declare the service service = 'Mast.Caom.Filtered' # Set the filters filters += [{ 'paramName': 'obs_collection', 'values': ['JWST'] }, { 'paramName': 'instrument_name', 'values': [instrument] }, { 'paramName': 'dataproduct_type', 'values': dataproduct }] # Instruent filtered service else: # Declare the service service = 'Mast.Jwst.Filtered.{}'.format(instrument.title()) # Include additonal filters if isinstance(add_filters, dict): filters += [{ "paramName": name, "values": [val] } for name, val in add_filters.items()] # Assemble the request params = { 'columns': 'COUNT_BIG(*)', 'filters': filters, 'removenullcolumns': True } # Just get the counts if return_data: params['columns'] = '*' # Add requests if isinstance(add_requests, dict): params.update(add_requests) response = Mast.service_request_async(service, params) result = response[0].json() # Return all the data if return_data: return result # Or just the counts else: return result['data'][0]['Column1']
def monitor_template_main(): """ The main function of the ``monitor_template`` module.""" # Example of logging my_variable = 'foo' logging.info('Some useful information: {}'.format(my_variable)) # Example of querying for a dataset via MAST API service = "Mast.Jwst.Filtered.Niriss" params = { "columns": "filename", "filters": [{ "paramName": "filter", "values": ['F430M'] }] } response = Mast.service_request_async(service, params) result = response[0].json()['data'] filename_of_interest = result[0][ 'filename'] # jw00304002001_02102_00001_nis_uncal.fits # Example of parsing a filename filename_dict = filename_parser(filename_of_interest) # Contents of filename_dict: # {'program_id': '00304', # 'observation': '002', # 'visit': '001', # 'visit_group': '02', # 'parallel_seq_id': '1', # 'activity': '02', # 'exposure_id': '00001', # 'detector': 'nis', # 'suffix': 'uncal'} # Example of locating a dataset in the filesystem filesystem = get_config()['filesystem'] dataset = os.path.join(filesystem, 'jw{}'.format(filename_dict['program_id']), filename_of_interest) # Example of reading in dataset using jwst.datamodels im = datamodels.open(dataset) # Now have access to: # im.data # Data array # im.err # ERR array # im.meta # Metadata such as header keywords # Example of saving a file and setting permissions im.save('some_filename.fits') set_permissions('some_filename.fits') # Example of creating and exporting a Bokeh plot plt = Donut(im.data, plot_width=600, plot_height=600) plt.sizing_mode = 'stretch_both' # Necessary for responsive sizing on web app script, div = components(plt) plot_output_dir = get_config()['outputs'] div_outfile = os.path.join(plot_output_dir, 'monitor_name', filename_of_interest + "_component.html") script_outfile = os.path.join(plot_output_dir, 'monitor_name', filename_of_interest + "_component.js") for outfile, component in zip([div_outfile, script_outfile], [div, script]): with open(outfile, 'w') as f: f.write(component) f.close() set_permissions(outfile) # Perform any other necessary code well_named_variable = "Function does something." result_of_second_function = second_function(well_named_variable)
def get_thumbnails_all_instruments(parameters): """Return a list of thumbnails available in the filesystem for all instruments given requested MAST parameters and queried anomalies. Parameters ---------- parameters: dict A dictionary containing the following keys, some of which are dictionaries: instruments apertures filters detector effexptm_min effexptm_max anomalies Returns ------- thumbnails : list A list of thumbnails available in the filesystem for the given instrument. """ anomalies = parameters['anomalies'] filenames = [] for inst in parameters['instruments']: # Make sure instruments are of the proper format (e.g. "Nircam") instrument = inst[0].upper() + inst[1:].lower() # Query MAST for all rootnames for the instrument service = "Mast.Jwst.Filtered.{}".format(instrument) if (parameters['apertures'][inst.lower()] == []) and (parameters['detectors'][inst.lower()] == []) \ and (parameters['filters'][inst.lower()] == []) and (parameters['exposure_types'][inst.lower()] == []) \ and (parameters['read_patterns'][inst.lower()] == []): params = {"columns": "*", "filters": []} else: query_filters = [] if (parameters['apertures'][inst.lower()] != []): if instrument != "Nircam": query_filters.append({ "paramName": "pps_aper", "values": parameters['apertures'][inst.lower()] }) if instrument == "Nircam": query_filters.append({ "paramName": "apername", "values": parameters['apertures'][inst.lower()] }) if (parameters['detectors'][inst.lower()] != []): query_filters.append({ "paramName": "detector", "values": parameters['detectors'][inst.lower()] }) if (parameters['filters'][inst.lower()] != []): query_filters.append({ "paramName": "filter", "values": parameters['filters'][inst.lower()] }) if (parameters['exposure_types'][inst.lower()] != []): query_filters.append({ "paramName": "exp_type", "values": parameters['exposure_types'][inst.lower()] }) if (parameters['read_patterns'][inst.lower()] != []): query_filters.append({ "paramName": "readpatt", "values": parameters['read_patterns'][inst.lower()] }) params = {"columns": "*", "filters": query_filters} response = Mast.service_request_async(service, params) results = response[0].json()['data'] inst_filenames = [ result['filename'].split('.')[0] for result in results ] filenames.extend(inst_filenames) # Get list of all thumbnails thumbnail_list = glob.glob( os.path.join(THUMBNAIL_FILESYSTEM, '*', '*.thumb')) # Get subset of preview images that match the filenames thumbnails_subset = [ os.path.basename(item) for item in thumbnail_list if os.path.basename(item).split('_integ')[0] in filenames ] # Eliminate any duplicates thumbnails_subset = list(set(thumbnails_subset)) # Determine whether or not queried anomalies are flagged final_subset = [] if anomalies != { 'miri': [], 'nirspec': [], 'niriss': [], 'nircam': [], 'fgs': [] }: for thumbnail in thumbnails_subset: components = thumbnail.split('_') rootname = ''.join((components[0], '_', components[1], '_', components[2], '_', components[3])) try: instrument = filename_parser(thumbnail)['instrument'] thumbnail_anomalies = get_current_flagged_anomalies( rootname, instrument) if thumbnail_anomalies: for anomaly in anomalies[instrument.lower()]: if anomaly.lower() in thumbnail_anomalies: # thumbnail contains an anomaly selected in the query final_subset.append(thumbnail) except KeyError: print("Error with thumbnail: ", thumbnail) else: # if no anomalies are flagged, return all thumbnails from query final_subset = thumbnails_subset return list(set(final_subset))
def get_thumbnails_all_instruments(parameters): """Return a list of thumbnails available in the filesystem for all instruments given requested MAST parameters and queried anomalies. Parameters ---------- parameters: dict A dictionary containing the following keys, some of which are dictionaries: instruments apertures filters detector effexptm_min effexptm_max anomalies Returns ------- thumbnails : list A list of thumbnails available in the filesystem for the given instrument. """ anomalies = parameters['anomalies'] thumbnail_list = [] filenames = [] if parameters['instruments'] is None: thumbnails = [] for inst in parameters['instruments']: print("Retrieving thumbnails for", inst) # Make sure instruments are of the proper format (e.g. "Nircam") instrument = inst[0].upper() + inst[1:].lower() # Query MAST for all rootnames for the instrument service = "Mast.Jwst.Filtered.{}".format(instrument) params = { "columns": "*", "filters": [{ "paramName": "apername", "values": parameters['apertures'][inst.lower()] }, { "paramName": "detector", "values": parameters['detectors'][inst.lower()] }, { "paramName": "filter", "values": parameters['filters'][inst.lower()] }, { "paramName": "exp_type", "values": parameters['exposure_types'][inst.lower()] }, { "paramName": "readpatt", "values": parameters['read_patterns'][inst.lower()] }] } response = Mast.service_request_async(service, params) results = response[0].json()['data'] for result in results: filename = result['filename'].split('.')[0] filenames.append(filename) # Get list of all thumbnails thumbnails = glob.glob( os.path.join(THUMBNAIL_FILESYSTEM, '*', '*.thumb')) thumbnail_list.extend(thumbnails) # Get subset of preview images that match the filenames thumbnails_subset = [ os.path.basename(item) for item in thumbnail_list if os.path.basename(item).split('_integ')[0] in filenames ] # Eliminate any duplicates thumbnails_subset = list(set(thumbnails_subset)) # Determine whether or not queried anomalies are flagged final_subset = [] for thumbnail in thumbnails_subset: components = thumbnail.split('_') rootname = '{}_{}_{}_{}'.format(components[0], components[1], components[2], components[3]) try: instrument = JWST_INSTRUMENT_NAMES_SHORTHAND[thumbnail.split("_") [3][:3]] thumbnail_anomalies = get_current_flagged_anomalies( rootname, instrument) if thumbnail_anomalies: for anomaly in anomalies[instrument.lower()]: if anomaly.lower() in thumbnail_anomalies: print(thumbnail, "contains an anomaly selected in the query") final_subset.append(thumbnail) except KeyError: try: instrument = JWST_INSTRUMENT_NAMES_SHORTHAND[thumbnail.split( "_")[2][:3]] thumbnail_anomalies = get_current_flagged_anomalies( rootname, instrument) if thumbnail_anomalies: for anomaly in anomalies[instrument.lower()]: if anomaly.lower() in thumbnail_anomalies: print(thumbnail, "contains an anomaly selected in the query") final_subset.append(thumbnail) except KeyError: print("Error with thumbnail: ", thumbnail) if not final_subset: print("No images matched anomaly selection") final_subset = thumbnails_subset if not final_subset: final_subset = thumbnails[:10] return list(set(final_subset))
""" import astropy from astropy.table import Table from astropy.time import Time from astroquery.mast import Mast from bokeh.embed import components from bokeh.plotting import figure from .utils import get_config # should use oauth.register_oauth()? settings = get_config() mast_token = settings['mast_token'] Mast.login(token=mast_token) # could eventually be moved to constants.py mast_edb_timeseries_service = 'Mast.JwstEdb.GetTimeseries.All' mast_edb_dictionary_service = 'Mast.JwstEdb.Dictionary' mast_edb_mnemonic_service = 'Mast.JwstEdb.Mnemonics' class EdbMnemonic: """Class to hold and manipulate results of DMS EngDB queries.""" def __init__(self, mnemonic_identifier, start_time, end_time, data, meta, info): """Populate attributes. Parameters ----------