def test_filenames(temporary_directory): """Exercise mirage input generation from APT files (.xml and .pointing).""" for instrument in ['misc']: apt_dir = os.path.join(TEST_DATA_DIR, instrument) if instrument == 'misc': apt_file_seeds = [ '12345/12345_nircam_imaging_prime_niriss_wfss_parallel.xml' ] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') catalogs = {} for instrument_name in 'fgs nircam niriss miri nirspec'.split(): if instrument_name.lower() == 'nircam': catalogs[instrument_name.lower()] = {} catalogs[instrument_name.lower()]['sw'] = source_list_file_name catalogs[instrument_name.lower()]['lw'] = source_list_file_name else: catalogs[instrument_name.lower()] = source_list_file_name for i, apt_file_seed in enumerate(apt_file_seeds): print('\n\n' + '=' * 100 + '\n') obs_yaml_files = glob.glob( os.path.join(temporary_directory, 'jw*.yaml')) for file in obs_yaml_files: os.remove(file) if '.xml' in apt_file_seed: apt_file_xml = os.path.join(apt_dir, apt_file_seed) apt_file_pointing = os.path.join( apt_dir, apt_file_seed.replace('.xml', '.pointing')) observation_list_file = os.path.join( temporary_directory, '{}_observation_list.yaml'.format( apt_file_seed.replace('/', '_').split('.')[0])) print('Processing program {}'.format(apt_file_xml)) yam = yaml_generator.SimInput( input_xml=apt_file_xml, pointing_file=apt_file_pointing, observation_list_file=observation_list_file, verbose=True, output_dir=temporary_directory, simdata_output_dir=temporary_directory, offline=True) try: yam.create_inputs() except RuntimeError as e: print('\nERROR detected. Skipping {} because of error\n{}\n\n'. format(apt_file_xml, e)) continue Table([yam.info['yamlfile']]).pprint(max_lines=-1) exposure_numbers = np.array( [int(s.split('_')[2]) for s in yam.info['yamlfile']]) assert np.max(exposure_numbers) <= 18
def RunAllAPTTemplates(instrument): '''Parse the given APT files and create a set of .yamls for a given instrument ''' # Define .pointing and .xml file locations pointing_file = os.path.join(TESTS_DIR, 'test_data', instrument, instrument + 'Test.pointing') xml_file = os.path.join(TESTS_DIR, 'test_data', instrument, instrument + 'Test.xml') # Open XML file, get element tree of the APT proposal to determine how # many observations there are with open(xml_file) as f: tree = etree.parse(f) observation_data = tree.find(APT_NAMESPACE + 'DataRequests') obs_results = observation_data.findall('.//' + APT_NAMESPACE + 'Observation') n_obs = len(obs_results) # Locate catalogs for target(s) sw_cats = [os.path.join(TESTS_DIR, 'test_data', '2MASS_RA273.09deg_Dec65.60deg.list')] * n_obs lw_cats = [os.path.join(TESTS_DIR, 'test_data', 'WISE_RA273.09deg_Dec65.60deg.list')] * n_obs # Point to appropriate output directory out_dir = os.path.join(TESTS_DIR, 'test_data', instrument, 'APT_{}_out'.format(instrument)) if os.path.exists(out_dir): shutil.rmtree(out_dir) os.makedirs(out_dir) else: os.makedirs(out_dir) # Write observationlist.yaml observationlist_file = os.path.join(out_dir, instrument + '_observationlist.yaml') write_observationlist.write_yaml(xml_file, pointing_file, observationlist_file, ps_cat_sw=sw_cats, ps_cat_lw=lw_cats) # Create a series of data simulator input yaml files yam = yaml_generator.SimInput() yam.input_xml = xml_file yam.pointing_file = pointing_file yam.output_dir = out_dir yam.simdata_output_dir = out_dir yam.observation_table = observationlist_file yam.use_JWST_pipeline = False yam.use_linearized_darks = True yam.datatype = 'linear' yam.reffile_setup() yam.create_inputs() # Ensure that some of the expected files have been created assert os.path.exists(os.path.join(out_dir, 'Observation_table_for_' + instrument + 'Test.xml_with_yaml_parameters.csv')), \ 'Observation table not created.' number_of_yaml_files = len(glob.glob(os.path.join(out_dir, 'jw{:05d}*.yaml'.format(int(PROPOSAL_ID))))) print('PROPOSAL_ID: {}'.format(PROPOSAL_ID)) print('number of observations: {}'.format(n_obs)) print('number of files written: {}'.format(number_of_yaml_files)) assert n_obs == 17 assert number_of_yaml_files == 150 assert number_of_yaml_files >= n_obs, 'Fewer yaml files created than observations'
def create_yaml_files(): """Create a series of data simulator input yaml files from APT files """ xml = 'APT_files/NCam019.xml' pointing_file = 'APT_files/NCam019.pointing' output_dir = 'yaml_files/' simdata_output_dir = 'Mirage_Output' catalogs = { 'LMC-ASTROMETRIC-FIELD': { 'point_source': 'Input_Catalogs/car19_ptsrc.cat' }, 'LMC-ASTROM-CENTER-JMAG14-STAR': { 'point_source': 'Input_Catalogs/car19_ptsrc.cat' } } background = 'low' yam = yaml_generator.SimInput(xml, pointing_file, output_dir=output_dir, simdata_output_dir=simdata_output_dir, catalogs=catalogs, datatype='raw', dateobs_for_background=False, reffile_defaults='crds', background=background) yam.use_linearized_darks = True yam.create_inputs() return yam
'HD-37093': { 'point_source': os.path.join(ami_example_dir, 'stars_field19_20_combined_allfilters.list') } } pav3 = 275 dates = '2020-09-20' reffile_defaults = 'crds' datatype = 'linear, raw' yam = yaml_generator.SimInput(input_xml=xml_name, pointing_file=pointing_name, catalogs=catalogues, roll_angle=pav3, dates=dates, reffile_defaults=reffile_defaults, verbose=True, output_dir=output_directory, simdata_output_dir=simdata_output_directory, datatype=datatype) yam.create_inputs() # Create all files yaml_files = glob.glob(os.path.join(output_directory, 'jw*.yaml')) print(yaml_files) for file in yaml_files: # set astrometric reference file to None to use pysiaf with open(file, 'r') as infile:
# # In order to fully process the Mirage output with the default steps used by the pipeline, it would be best to use the 'raw' output and run the entire calibration pipeline. # In[16]: datatype = 'linear, raw' # In[ ]: # Run the yaml generator yam = yaml_generator.SimInput(input_xml=xml_file, pointing_file=pointing_file, catalogs=cat_dict, cosmic_rays=cosmic_rays, background=background, roll_angle=pav3, dates=dates, reffile_defaults=reffile_defaults, verbose=True, output_dir=output_dir, simdata_output_dir=simulation_dir, datatype=datatype) yam.create_inputs() # In[ ]: yfiles = glob(os.path.join(output_dir, '*.yaml')) print(yfiles) # --- # <a id='run_steps_together'></a> # # Create simulated data
def test_complete_input_generation(temporary_directory): """Exercise mirage input generation from APT files (.xml and .pointing).""" for instrument in ['NIRCam', 'NIRISS', 'NIRSpec', 'MIRI', 'misc', 'FGS']: apt_dir = os.path.join(TEST_DATA_DIR, instrument) if instrument == 'NIRISS': apt_file_seeds = [ 'com1093', '1087_minimal', '1088', '1087', 'm31_field_test_observation' ] # apt_file_seeds = ['com1093'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'NIRCam': apt_file_seeds = ['1069', '1144-OTE-10', 'NIRCamTest'] source_list_file_name = os.path.join( apt_dir, 'seed_im_from_catalog_test_ptsrc_catalog.list') elif instrument == 'NIRSpec': apt_file_seeds = ['1164'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'MIRI': apt_file_seeds = ['1171'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'FGS': apt_file_seeds = ['MIRAGE-test'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'misc': apt_xml_files = glob.glob(os.path.join(apt_dir, '*/*.xml')) apt_file_seeds = [f.split(apt_dir)[1] for f in apt_xml_files] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') catalogs = {} for instrument_name in 'fgs nircam niriss miri nirspec'.split(): if instrument_name.lower() == 'nircam': catalogs[instrument_name.lower()] = {} catalogs[instrument_name.lower()]['sw'] = source_list_file_name catalogs[instrument_name.lower()]['lw'] = source_list_file_name else: catalogs[instrument_name.lower()] = source_list_file_name for i, apt_file_seed in enumerate(apt_file_seeds): print('\n\n' + '=' * 100 + '\n') # For the moment, skip tests that contain NirissImaging observations # or that take too long for Travis skip_for_now = ['DeepField', 'NCam010', '1071'] skip_bool = any([ True if prop in apt_file_seed else False for prop in skip_for_now ]) if skip_bool: continue obs_yaml_files = glob.glob( os.path.join(temporary_directory, 'jw*.yaml')) for file in obs_yaml_files: os.remove(file) if '.xml' in apt_file_seed: apt_file_xml = os.path.join(apt_dir, apt_file_seed[1:]) apt_file_pointing = os.path.join( apt_dir, apt_file_seed[1:].replace('.xml', '.pointing')) observation_list_file = os.path.join( temporary_directory, '{}_observation_list.yaml'.format( apt_file_seed.replace('/', '_').split('.')[0])) else: observation_list_file = os.path.join( temporary_directory, '{}_observation_list.yaml'.format(apt_file_seed)) apt_file_xml = os.path.join(apt_dir, '{}.xml'.format(apt_file_seed)) apt_file_pointing = os.path.join( apt_dir, '{}.pointing'.format(apt_file_seed)) print('Processing program {}'.format(apt_file_xml)) yam = yaml_generator.SimInput( input_xml=apt_file_xml, pointing_file=apt_file_pointing, catalogs=catalogs, observation_list_file=observation_list_file, verbose=True, output_dir=temporary_directory, simdata_output_dir=temporary_directory, offline=True) try: yam.create_inputs() except RuntimeError as e: print('\nERROR detected. Skipping {} because of error\n{}\n\n'. format(apt_file_xml, e)) continue yfiles = glob.glob( os.path.join(yam.output_dir, 'jw*{}*.yaml'.format(yam.info['ProposalID'][0]))) valid_instrument_list = [ s for s in yam.info['Instrument'] if s.lower() in 'fgs nircam niriss'.split() ] assert len(valid_instrument_list) == len(yfiles) if os.path.basename(apt_file_xml) in [ '54321_niriss_wfss_prime_nircam_imaging_parallel.xml', '12345_nircam_imaging_prime_niriss_wfss_parallel.xml' ]: prog = os.path.basename(apt_file_xml).split('_')[0] expected_nircam_files = { '54321': [60, 75], '12345': [40, 90, 40, 20] } expected_niriss_files = { '54321': [12, 15], '12345': [8, 18, 8, 0] } nircam_files_from_list = np.array([ True if s.lower() == 'nircam' else False for s in valid_instrument_list ]) num_nircam_files = np.sum(nircam_files_from_list) num_niriss_files = len(yfiles) - num_nircam_files assert num_nircam_files == np.sum( np.array(expected_nircam_files[prog])) assert num_niriss_files == np.sum( np.array(expected_niriss_files[prog])) num_obs = len(expected_nircam_files[prog]) for i in range(1, num_obs + 1): obs_num = str(i).zfill(3) obs_yfiles_nircam = glob.glob( os.path.join(yam.output_dir, 'jw{}{}*nrc*.yaml'.format(prog, obs_num))) obs_yfiles_niriss = glob.glob( os.path.join(yam.output_dir, 'jw{}{}*nis*.yaml'.format(prog, obs_num))) assert len( obs_yfiles_nircam) == expected_nircam_files[prog][i - 1] assert len( obs_yfiles_niriss) == expected_niriss_files[prog][i - 1]
catalogues = { 'niriss': os.path.join(ami_example_dir, 'stars_field19_20_combined_allfilters.list') } parameter_defaults = {'PAV3': 275., 'DATE': '2020-09-20'} observation_file = os.path.join(output_directory, '793_observation_list.yaml') generate_observationlist.get_observation_dict(xml_name, observation_file, catalogs=catalogues) yam = yaml_generator.SimInput(input_xml=xml_name, pointing_file=pointing_name, catalogs=catalogues, observation_list_file=observation_file, verbose=True, output_dir=output_directory, simdata_output_dir=simdata_output_directory, use_JWST_pipeline=True, offline=False, parameter_defaults=parameter_defaults) datatype = 'linear, raw' yam.datatype = datatype yam.create_inputs() yaml_files = glob.glob(os.path.join(output_directory, 'jw*.yaml')) for file in yaml_files: # set astrometric reference file to None to use pysiaf with open(file, 'r') as infile:
catalogs = { 'ROSS-905': { 'point_source': None, 'tso_imaging_catalog': imaging_tso_catalog, 'tso_grism_catalog': grism_tso_catalog, } } background = 'medium' pav3 = 0. yam = yaml_generator.SimInput(xml_file, pointing_file, catalogs=catalogs, verbose=True, output_dir=output_yaml_dir, simdata_output_dir=output_data_dir, background=background, roll_angle=pav3, dates=None, datatype='linear, raw', dateobs_for_background=False, reffile_defaults='crds') yam.use_linearized_darks = True yam.create_inputs() # ---------- SIMULATE F322W2 GRISMR TSO ---------- gr_tso_yaml_file = os.path.join(output_yaml_dir, 'jw00042001001_01101_00002_nrca5.yaml') gr_f322w2 = GrismTSO(gr_tso_yaml_file, SED_file=sed_file,
def test_complete_input_generation(): """Exercise mirage input generation from APT files (.xml and .pointing).""" # generate output directory temporary_directory() for instrument in ['NIRCam', 'NIRISS', 'NIRSpec', 'MIRI', 'misc', 'FGS']: # for instrument in ['NIRISS', 'NIRSpec', 'MIRI', 'FGS', 'NIRCam']: # for instrument in ['NIRISS', 'NIRSpec', 'MIRI', 'FGS']: # for instrument in ['NIRISS']: # for instrument in ['misc']: # for instrument in ['NIRSpec']: # for instrument in ['MIRI']: # for instrument in ['FGS']: # for instrument in ['NIRCam']: apt_dir = os.path.join(TEST_DATA_DIR, instrument) if instrument == 'NIRISS': apt_file_seeds = [ '1087_minimal', '1088', '1087', 'm31_field_test_observation' ] # apt_file_seeds = ['1087'] # apt_file_seeds = ['1087_minimal'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'NIRCam': apt_file_seeds = ['1069', '1144-OTE-10', 'NIRCamTest'] # apt_file_seeds = ['NIRCamTest'] # apt_file_seeds = ['1069'] # apt_file_seeds = ['1144-OTE-10'] source_list_file_name = os.path.join( apt_dir, 'seed_im_from_catalog_test_ptsrc_catalog.list') elif instrument == 'NIRSpec': apt_file_seeds = ['1164'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'MIRI': apt_file_seeds = ['1171'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'FGS': apt_file_seeds = ['MIRAGE-test'] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') elif instrument == 'misc': apt_xml_files = glob.glob(os.path.join(apt_dir, '*/*.xml')) apt_file_seeds = [f.split(apt_dir)[1] for f in apt_xml_files] source_list_file_name = os.path.join(apt_dir, 'niriss_point_sources.list') catalogs = {} for instrument_name in 'fgs nircam niriss miri nirspec'.split(): if instrument_name.lower() == 'nircam': catalogs[instrument_name.lower()] = {} catalogs[instrument_name.lower()]['sw'] = source_list_file_name catalogs[instrument_name.lower()]['lw'] = source_list_file_name else: catalogs[instrument_name.lower()] = source_list_file_name for i, apt_file_seed in enumerate(apt_file_seeds): print('\n\n' + '=' * 100 + '\n') if 'DeepField' in apt_file_seed: continue obs_yaml_files = glob.glob(os.path.join(TEMPORARY_DIR, 'jw*.yaml')) for file in obs_yaml_files: os.remove(file) if '.xml' in apt_file_seed: apt_file_xml = os.path.join(apt_dir, apt_file_seed[1:]) apt_file_pointing = os.path.join( apt_dir, apt_file_seed[1:].replace('.xml', '.pointing')) observation_list_file = os.path.join( TEMPORARY_DIR, '{}_observation_list.yaml'.format( apt_file_seed.replace('/', '_').split('.')[0])) else: observation_list_file = os.path.join( TEMPORARY_DIR, '{}_observation_list.yaml'.format(apt_file_seed)) apt_file_xml = os.path.join(apt_dir, '{}.xml'.format(apt_file_seed)) apt_file_pointing = os.path.join( apt_dir, '{}.pointing'.format(apt_file_seed)) print('Processing program {}'.format(apt_file_xml)) yam = yaml_generator.SimInput( input_xml=apt_file_xml, pointing_file=apt_file_pointing, catalogs=catalogs, observation_list_file=observation_list_file, verbose=True, output_dir=TEMPORARY_DIR, simdata_output_dir=TEMPORARY_DIR, offline=True) try: yam.create_inputs() except RuntimeError as e: print('\nERROR detected. Skipping {} because of error\n{}\n\n'. format(apt_file_xml, e)) continue yfiles = glob.glob( os.path.join(yam.output_dir, 'jw*{}*.yaml'.format(yam.info['ProposalID'][0]))) valid_instrument_list = [ s for s in yam.info['Instrument'] if s.lower() in 'fgs nircam niriss'.split() ] assert len(valid_instrument_list) == len(yfiles)
def RunAllAPTTemplates(instrument): '''Parse the given APT files and create a set of .yamls for a given instrument ''' # Define .pointing and .xml file locations pointing_file = os.path.join(TESTS_DIR, 'test_data', instrument, instrument + 'Test.pointing') xml_file = os.path.join(TESTS_DIR, 'test_data', instrument, instrument + 'Test.xml') # Open XML file, get element tree of the APT proposal to determine how # many observations there are with open(xml_file) as f: tree = etree.parse(f) observation_data = tree.find(APT_NAMESPACE + 'DataRequests') obs_results = observation_data.findall('.//' + APT_NAMESPACE + 'Observation') n_obs = len(obs_results) # Locate catalogs for target(s) sw_cats = [ os.path.join(TESTS_DIR, 'test_data', '2MASS_RA273.09deg_Dec65.60deg.list') ] * n_obs lw_cats = [ os.path.join(TESTS_DIR, 'test_data', 'WISE_RA273.09deg_Dec65.60deg.list') ] * n_obs # Point to appropriate output directory out_dir = os.path.join(TESTS_DIR, 'test_data', instrument, 'APT_{}_out'.format(instrument)) if not os.path.exists(out_dir): os.makedirs(out_dir) # Write observationlist.yaml observationlist_file = os.path.join(out_dir, instrument + '_observationlist.yaml') write_observationlist.write_yaml(xml_file, pointing_file, observationlist_file, ps_cat_sw=sw_cats, ps_cat_lw=lw_cats) # Create a series of data simulator input yaml files yam = yaml_generator.SimInput() yam.input_xml = xml_file yam.pointing_file = pointing_file yam.siaf = os.path.expandvars( '$MIRAGE_DATA/nircam/reference_files/SIAF/NIRCam_SIAF_2018-01-08.csv') yam.output_dir = out_dir yam.simdata_output_dir = out_dir yam.observation_table = observationlist_file yam.use_JWST_pipeline = False yam.use_linearized_darks = True yam.datatype = 'linear' yam.reffile_setup() yam.create_inputs() # Ensure that some of the expected files have been created assert os.path.exists(os.path.join(out_dir, 'Observation_table_for_' + instrument + 'Test.xml_with_yaml_parameters.csv')), \ 'Observation table not created.' assert len(glob.glob(os.path.join(out_dir, 'V' + PROPOSAL_ID + '*.yaml'))) \ >= n_obs, 'Fewer yaml files created than observations' # If a reference observationlist.yaml file exists, ensure that the # file that was just created matches it reference_yaml = os.path.join( out_dir, 'REFERENCE_' + instrument + '_observationlist.yaml') if os.path.exists(reference_yaml): assert yaml.load(reference_yaml) == yaml.load(observationlist_file),\ 'The created observationlist.yaml file does not match the reference' +\ 'observationlist.yaml file. Either the APT parser is malfunctioning,' +\ 'or the reference yaml is out of date.'