def test_filenames(temporary_directory):
"""Exercise mirage input generation from APT files (.xml and .pointing)."""
for instrument in ['misc']:
apt_dir = os.path.join(TEST_DATA_DIR, instrument)
if instrument == 'misc':
apt_file_seeds = [
'12345/12345_nircam_imaging_prime_niriss_wfss_parallel.xml'
]
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
catalogs = {}
for instrument_name in 'fgs nircam niriss miri nirspec'.split():
if instrument_name.lower() == 'nircam':
catalogs[instrument_name.lower()] = {}
catalogs[instrument_name.lower()]['sw'] = source_list_file_name
catalogs[instrument_name.lower()]['lw'] = source_list_file_name
else:
catalogs[instrument_name.lower()] = source_list_file_name
for i, apt_file_seed in enumerate(apt_file_seeds):
print('\n\n' + '=' * 100 + '\n')
obs_yaml_files = glob.glob(
os.path.join(temporary_directory, 'jw*.yaml'))
for file in obs_yaml_files:
os.remove(file)
if '.xml' in apt_file_seed:
apt_file_xml = os.path.join(apt_dir, apt_file_seed)
apt_file_pointing = os.path.join(
apt_dir, apt_file_seed.replace('.xml', '.pointing'))
observation_list_file = os.path.join(
temporary_directory, '{}_observation_list.yaml'.format(
apt_file_seed.replace('/', '_').split('.')[0]))
print('Processing program {}'.format(apt_file_xml))
yam = yaml_generator.SimInput(
input_xml=apt_file_xml,
pointing_file=apt_file_pointing,
observation_list_file=observation_list_file,
verbose=True,
output_dir=temporary_directory,
simdata_output_dir=temporary_directory,
offline=True)
try:
yam.create_inputs()
except RuntimeError as e:
print('\nERROR detected. Skipping {} because of error\n{}\n\n'.
format(apt_file_xml, e))
continue
Table([yam.info['yamlfile']]).pprint(max_lines=-1)
exposure_numbers = np.array(
[int(s.split('_')[2]) for s in yam.info['yamlfile']])
assert np.max(exposure_numbers) <= 18
def RunAllAPTTemplates(instrument):
'''Parse the given APT files and create a set of .yamls for a given
instrument
'''
# Define .pointing and .xml file locations
pointing_file = os.path.join(TESTS_DIR, 'test_data', instrument, instrument + 'Test.pointing')
xml_file = os.path.join(TESTS_DIR, 'test_data', instrument, instrument + 'Test.xml')
# Open XML file, get element tree of the APT proposal to determine how
# many observations there are
with open(xml_file) as f:
tree = etree.parse(f)
observation_data = tree.find(APT_NAMESPACE + 'DataRequests')
obs_results = observation_data.findall('.//' + APT_NAMESPACE + 'Observation')
n_obs = len(obs_results)
# Locate catalogs for target(s)
sw_cats = [os.path.join(TESTS_DIR, 'test_data', '2MASS_RA273.09deg_Dec65.60deg.list')] * n_obs
lw_cats = [os.path.join(TESTS_DIR, 'test_data', 'WISE_RA273.09deg_Dec65.60deg.list')] * n_obs
# Point to appropriate output directory
out_dir = os.path.join(TESTS_DIR, 'test_data', instrument, 'APT_{}_out'.format(instrument))
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
else:
os.makedirs(out_dir)
# Write observationlist.yaml
observationlist_file = os.path.join(out_dir, instrument + '_observationlist.yaml')
write_observationlist.write_yaml(xml_file, pointing_file, observationlist_file,
ps_cat_sw=sw_cats, ps_cat_lw=lw_cats)
# Create a series of data simulator input yaml files
yam = yaml_generator.SimInput()
yam.input_xml = xml_file
yam.pointing_file = pointing_file
yam.output_dir = out_dir
yam.simdata_output_dir = out_dir
yam.observation_table = observationlist_file
yam.use_JWST_pipeline = False
yam.use_linearized_darks = True
yam.datatype = 'linear'
yam.reffile_setup()
yam.create_inputs()
# Ensure that some of the expected files have been created
assert os.path.exists(os.path.join(out_dir, 'Observation_table_for_' +
instrument +
'Test.xml_with_yaml_parameters.csv')), \
'Observation table not created.'
number_of_yaml_files = len(glob.glob(os.path.join(out_dir, 'jw{:05d}*.yaml'.format(int(PROPOSAL_ID)))))
print('PROPOSAL_ID: {}'.format(PROPOSAL_ID))
print('number of observations: {}'.format(n_obs))
print('number of files written: {}'.format(number_of_yaml_files))
assert n_obs == 17
assert number_of_yaml_files == 150
assert number_of_yaml_files >= n_obs, 'Fewer yaml files created than observations'
def create_yaml_files():
"""Create a series of data simulator input yaml files
from APT files
"""
xml = 'APT_files/NCam019.xml'
pointing_file = 'APT_files/NCam019.pointing'
output_dir = 'yaml_files/'
simdata_output_dir = 'Mirage_Output'
catalogs = {
'LMC-ASTROMETRIC-FIELD': {
'point_source': 'Input_Catalogs/car19_ptsrc.cat'
},
'LMC-ASTROM-CENTER-JMAG14-STAR': {
'point_source': 'Input_Catalogs/car19_ptsrc.cat'
}
}
background = 'low'
yam = yaml_generator.SimInput(xml,
pointing_file,
output_dir=output_dir,
simdata_output_dir=simdata_output_dir,
catalogs=catalogs,
datatype='raw',
dateobs_for_background=False,
reffile_defaults='crds',
background=background)
yam.use_linearized_darks = True
yam.create_inputs()
return yam
'HD-37093': {
'point_source':
os.path.join(ami_example_dir,
'stars_field19_20_combined_allfilters.list')
}
}
pav3 = 275
dates = '2020-09-20'
reffile_defaults = 'crds'
datatype = 'linear, raw'
yam = yaml_generator.SimInput(input_xml=xml_name,
pointing_file=pointing_name,
catalogs=catalogues,
roll_angle=pav3,
dates=dates,
reffile_defaults=reffile_defaults,
verbose=True,
output_dir=output_directory,
simdata_output_dir=simdata_output_directory,
datatype=datatype)
yam.create_inputs()
# Create all files
yaml_files = glob.glob(os.path.join(output_directory, 'jw*.yaml'))
print(yaml_files)
for file in yaml_files:
# set astrometric reference file to None to use pysiaf
with open(file, 'r') as infile:
#
# In order to fully process the Mirage output with the default steps used by the pipeline, it would be best to use the 'raw' output and run the entire calibration pipeline.
# In[16]:
datatype = 'linear, raw'
# In[ ]:
# Run the yaml generator
yam = yaml_generator.SimInput(input_xml=xml_file,
pointing_file=pointing_file,
catalogs=cat_dict,
cosmic_rays=cosmic_rays,
background=background,
roll_angle=pav3,
dates=dates,
reffile_defaults=reffile_defaults,
verbose=True,
output_dir=output_dir,
simdata_output_dir=simulation_dir,
datatype=datatype)
yam.create_inputs()
# In[ ]:
yfiles = glob(os.path.join(output_dir, '*.yaml'))
print(yfiles)
# ---
# <a id='run_steps_together'></a>
# # Create simulated data
def test_complete_input_generation(temporary_directory):
"""Exercise mirage input generation from APT files (.xml and .pointing)."""
for instrument in ['NIRCam', 'NIRISS', 'NIRSpec', 'MIRI', 'misc', 'FGS']:
apt_dir = os.path.join(TEST_DATA_DIR, instrument)
if instrument == 'NIRISS':
apt_file_seeds = [
'com1093', '1087_minimal', '1088', '1087',
'm31_field_test_observation'
]
# apt_file_seeds = ['com1093']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'NIRCam':
apt_file_seeds = ['1069', '1144-OTE-10', 'NIRCamTest']
source_list_file_name = os.path.join(
apt_dir, 'seed_im_from_catalog_test_ptsrc_catalog.list')
elif instrument == 'NIRSpec':
apt_file_seeds = ['1164']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'MIRI':
apt_file_seeds = ['1171']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'FGS':
apt_file_seeds = ['MIRAGE-test']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'misc':
apt_xml_files = glob.glob(os.path.join(apt_dir, '*/*.xml'))
apt_file_seeds = [f.split(apt_dir)[1] for f in apt_xml_files]
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
catalogs = {}
for instrument_name in 'fgs nircam niriss miri nirspec'.split():
if instrument_name.lower() == 'nircam':
catalogs[instrument_name.lower()] = {}
catalogs[instrument_name.lower()]['sw'] = source_list_file_name
catalogs[instrument_name.lower()]['lw'] = source_list_file_name
else:
catalogs[instrument_name.lower()] = source_list_file_name
for i, apt_file_seed in enumerate(apt_file_seeds):
print('\n\n' + '=' * 100 + '\n')
# For the moment, skip tests that contain NirissImaging observations
# or that take too long for Travis
skip_for_now = ['DeepField', 'NCam010', '1071']
skip_bool = any([
True if prop in apt_file_seed else False
for prop in skip_for_now
])
if skip_bool:
continue
obs_yaml_files = glob.glob(
os.path.join(temporary_directory, 'jw*.yaml'))
for file in obs_yaml_files:
os.remove(file)
if '.xml' in apt_file_seed:
apt_file_xml = os.path.join(apt_dir, apt_file_seed[1:])
apt_file_pointing = os.path.join(
apt_dir, apt_file_seed[1:].replace('.xml', '.pointing'))
observation_list_file = os.path.join(
temporary_directory, '{}_observation_list.yaml'.format(
apt_file_seed.replace('/', '_').split('.')[0]))
else:
observation_list_file = os.path.join(
temporary_directory,
'{}_observation_list.yaml'.format(apt_file_seed))
apt_file_xml = os.path.join(apt_dir,
'{}.xml'.format(apt_file_seed))
apt_file_pointing = os.path.join(
apt_dir, '{}.pointing'.format(apt_file_seed))
print('Processing program {}'.format(apt_file_xml))
yam = yaml_generator.SimInput(
input_xml=apt_file_xml,
pointing_file=apt_file_pointing,
catalogs=catalogs,
observation_list_file=observation_list_file,
verbose=True,
output_dir=temporary_directory,
simdata_output_dir=temporary_directory,
offline=True)
try:
yam.create_inputs()
except RuntimeError as e:
print('\nERROR detected. Skipping {} because of error\n{}\n\n'.
format(apt_file_xml, e))
continue
yfiles = glob.glob(
os.path.join(yam.output_dir,
'jw*{}*.yaml'.format(yam.info['ProposalID'][0])))
valid_instrument_list = [
s for s in yam.info['Instrument']
if s.lower() in 'fgs nircam niriss'.split()
]
assert len(valid_instrument_list) == len(yfiles)
if os.path.basename(apt_file_xml) in [
'54321_niriss_wfss_prime_nircam_imaging_parallel.xml',
'12345_nircam_imaging_prime_niriss_wfss_parallel.xml'
]:
prog = os.path.basename(apt_file_xml).split('_')[0]
expected_nircam_files = {
'54321': [60, 75],
'12345': [40, 90, 40, 20]
}
expected_niriss_files = {
'54321': [12, 15],
'12345': [8, 18, 8, 0]
}
nircam_files_from_list = np.array([
True if s.lower() == 'nircam' else False
for s in valid_instrument_list
])
num_nircam_files = np.sum(nircam_files_from_list)
num_niriss_files = len(yfiles) - num_nircam_files
assert num_nircam_files == np.sum(
np.array(expected_nircam_files[prog]))
assert num_niriss_files == np.sum(
np.array(expected_niriss_files[prog]))
num_obs = len(expected_nircam_files[prog])
for i in range(1, num_obs + 1):
obs_num = str(i).zfill(3)
obs_yfiles_nircam = glob.glob(
os.path.join(yam.output_dir,
'jw{}{}*nrc*.yaml'.format(prog, obs_num)))
obs_yfiles_niriss = glob.glob(
os.path.join(yam.output_dir,
'jw{}{}*nis*.yaml'.format(prog, obs_num)))
assert len(
obs_yfiles_nircam) == expected_nircam_files[prog][i -
1]
assert len(
obs_yfiles_niriss) == expected_niriss_files[prog][i -
1]
catalogues = {
'niriss':
os.path.join(ami_example_dir, 'stars_field19_20_combined_allfilters.list')
}
parameter_defaults = {'PAV3': 275., 'DATE': '2020-09-20'}
observation_file = os.path.join(output_directory, '793_observation_list.yaml')
generate_observationlist.get_observation_dict(xml_name,
observation_file,
catalogs=catalogues)
yam = yaml_generator.SimInput(input_xml=xml_name,
pointing_file=pointing_name,
catalogs=catalogues,
observation_list_file=observation_file,
verbose=True,
output_dir=output_directory,
simdata_output_dir=simdata_output_directory,
use_JWST_pipeline=True,
offline=False,
parameter_defaults=parameter_defaults)
datatype = 'linear, raw'
yam.datatype = datatype
yam.create_inputs()
yaml_files = glob.glob(os.path.join(output_directory, 'jw*.yaml'))
for file in yaml_files:
# set astrometric reference file to None to use pysiaf
with open(file, 'r') as infile:
catalogs = {
'ROSS-905': {
'point_source': None,
'tso_imaging_catalog': imaging_tso_catalog,
'tso_grism_catalog': grism_tso_catalog,
}
}
background = 'medium'
pav3 = 0.
yam = yaml_generator.SimInput(xml_file,
pointing_file,
catalogs=catalogs,
verbose=True,
output_dir=output_yaml_dir,
simdata_output_dir=output_data_dir,
background=background,
roll_angle=pav3,
dates=None,
datatype='linear, raw',
dateobs_for_background=False,
reffile_defaults='crds')
yam.use_linearized_darks = True
yam.create_inputs()
# ---------- SIMULATE F322W2 GRISMR TSO ----------
gr_tso_yaml_file = os.path.join(output_yaml_dir,
'jw00042001001_01101_00002_nrca5.yaml')
gr_f322w2 = GrismTSO(gr_tso_yaml_file,
SED_file=sed_file,
def test_complete_input_generation():
"""Exercise mirage input generation from APT files (.xml and .pointing)."""
# generate output directory
temporary_directory()
for instrument in ['NIRCam', 'NIRISS', 'NIRSpec', 'MIRI', 'misc', 'FGS']:
# for instrument in ['NIRISS', 'NIRSpec', 'MIRI', 'FGS', 'NIRCam']:
# for instrument in ['NIRISS', 'NIRSpec', 'MIRI', 'FGS']:
# for instrument in ['NIRISS']:
# for instrument in ['misc']:
# for instrument in ['NIRSpec']:
# for instrument in ['MIRI']:
# for instrument in ['FGS']:
# for instrument in ['NIRCam']:
apt_dir = os.path.join(TEST_DATA_DIR, instrument)
if instrument == 'NIRISS':
apt_file_seeds = [
'1087_minimal', '1088', '1087', 'm31_field_test_observation'
]
# apt_file_seeds = ['1087']
# apt_file_seeds = ['1087_minimal']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'NIRCam':
apt_file_seeds = ['1069', '1144-OTE-10', 'NIRCamTest']
# apt_file_seeds = ['NIRCamTest']
# apt_file_seeds = ['1069']
# apt_file_seeds = ['1144-OTE-10']
source_list_file_name = os.path.join(
apt_dir, 'seed_im_from_catalog_test_ptsrc_catalog.list')
elif instrument == 'NIRSpec':
apt_file_seeds = ['1164']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'MIRI':
apt_file_seeds = ['1171']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'FGS':
apt_file_seeds = ['MIRAGE-test']
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
elif instrument == 'misc':
apt_xml_files = glob.glob(os.path.join(apt_dir, '*/*.xml'))
apt_file_seeds = [f.split(apt_dir)[1] for f in apt_xml_files]
source_list_file_name = os.path.join(apt_dir,
'niriss_point_sources.list')
catalogs = {}
for instrument_name in 'fgs nircam niriss miri nirspec'.split():
if instrument_name.lower() == 'nircam':
catalogs[instrument_name.lower()] = {}
catalogs[instrument_name.lower()]['sw'] = source_list_file_name
catalogs[instrument_name.lower()]['lw'] = source_list_file_name
else:
catalogs[instrument_name.lower()] = source_list_file_name
for i, apt_file_seed in enumerate(apt_file_seeds):
print('\n\n' + '=' * 100 + '\n')
if 'DeepField' in apt_file_seed:
continue
obs_yaml_files = glob.glob(os.path.join(TEMPORARY_DIR, 'jw*.yaml'))
for file in obs_yaml_files:
os.remove(file)
if '.xml' in apt_file_seed:
apt_file_xml = os.path.join(apt_dir, apt_file_seed[1:])
apt_file_pointing = os.path.join(
apt_dir, apt_file_seed[1:].replace('.xml', '.pointing'))
observation_list_file = os.path.join(
TEMPORARY_DIR, '{}_observation_list.yaml'.format(
apt_file_seed.replace('/', '_').split('.')[0]))
else:
observation_list_file = os.path.join(
TEMPORARY_DIR,
'{}_observation_list.yaml'.format(apt_file_seed))
apt_file_xml = os.path.join(apt_dir,
'{}.xml'.format(apt_file_seed))
apt_file_pointing = os.path.join(
apt_dir, '{}.pointing'.format(apt_file_seed))
print('Processing program {}'.format(apt_file_xml))
yam = yaml_generator.SimInput(
input_xml=apt_file_xml,
pointing_file=apt_file_pointing,
catalogs=catalogs,
observation_list_file=observation_list_file,
verbose=True,
output_dir=TEMPORARY_DIR,
simdata_output_dir=TEMPORARY_DIR,
offline=True)
try:
yam.create_inputs()
except RuntimeError as e:
print('\nERROR detected. Skipping {} because of error\n{}\n\n'.
format(apt_file_xml, e))
continue
yfiles = glob.glob(
os.path.join(yam.output_dir,
'jw*{}*.yaml'.format(yam.info['ProposalID'][0])))
valid_instrument_list = [
s for s in yam.info['Instrument']
if s.lower() in 'fgs nircam niriss'.split()
]
assert len(valid_instrument_list) == len(yfiles)
def RunAllAPTTemplates(instrument):
'''Parse the given APT files and create a set of .yamls for a given
instrument
'''
# Define .pointing and .xml file locations
pointing_file = os.path.join(TESTS_DIR, 'test_data', instrument,
instrument + 'Test.pointing')
xml_file = os.path.join(TESTS_DIR, 'test_data', instrument,
instrument + 'Test.xml')
# Open XML file, get element tree of the APT proposal to determine how
# many observations there are
with open(xml_file) as f:
tree = etree.parse(f)
observation_data = tree.find(APT_NAMESPACE + 'DataRequests')
obs_results = observation_data.findall('.//' + APT_NAMESPACE +
'Observation')
n_obs = len(obs_results)
# Locate catalogs for target(s)
sw_cats = [
os.path.join(TESTS_DIR, 'test_data',
'2MASS_RA273.09deg_Dec65.60deg.list')
] * n_obs
lw_cats = [
os.path.join(TESTS_DIR, 'test_data',
'WISE_RA273.09deg_Dec65.60deg.list')
] * n_obs
# Point to appropriate output directory
out_dir = os.path.join(TESTS_DIR, 'test_data', instrument,
'APT_{}_out'.format(instrument))
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# Write observationlist.yaml
observationlist_file = os.path.join(out_dir,
instrument + '_observationlist.yaml')
write_observationlist.write_yaml(xml_file,
pointing_file,
observationlist_file,
ps_cat_sw=sw_cats,
ps_cat_lw=lw_cats)
# Create a series of data simulator input yaml files
yam = yaml_generator.SimInput()
yam.input_xml = xml_file
yam.pointing_file = pointing_file
yam.siaf = os.path.expandvars(
'$MIRAGE_DATA/nircam/reference_files/SIAF/NIRCam_SIAF_2018-01-08.csv')
yam.output_dir = out_dir
yam.simdata_output_dir = out_dir
yam.observation_table = observationlist_file
yam.use_JWST_pipeline = False
yam.use_linearized_darks = True
yam.datatype = 'linear'
yam.reffile_setup()
yam.create_inputs()
# Ensure that some of the expected files have been created
assert os.path.exists(os.path.join(out_dir, 'Observation_table_for_' +
instrument +
'Test.xml_with_yaml_parameters.csv')), \
'Observation table not created.'
assert len(glob.glob(os.path.join(out_dir, 'V' + PROPOSAL_ID + '*.yaml'))) \
>= n_obs, 'Fewer yaml files created than observations'
# If a reference observationlist.yaml file exists, ensure that the
# file that was just created matches it
reference_yaml = os.path.join(
out_dir, 'REFERENCE_' + instrument + '_observationlist.yaml')
if os.path.exists(reference_yaml):
assert yaml.load(reference_yaml) == yaml.load(observationlist_file),\
'The created observationlist.yaml file does not match the reference' +\
'observationlist.yaml file. Either the APT parser is malfunctioning,' +\
'or the reference yaml is out of date.'