def test_base_association():
"""Create the simplest of associations"""
items = ['a', 'b', 'c']
asn = asn_from_list(items, rule=Association)
assert asn['asn_rule'] == 'Association'
assert asn['asn_type'] == 'None'
assert asn['members'] == items
def make_level3_association(file_list, asn_filename):
asn = asn_from_list(file_list, product_name='lw_imaging')
outfile = os.path.join(
'/ifs/jwst/wit/nircam/simulationWG/Imaging/CAR-19/Pipeline_Level3/',
asn_filename)
with open(outfile, 'w') as fh:
fh.write(asn.dump()[1])
def run_spec3(jail, rtdata_module):
"""Run the Spec3Pipeline on the single cal result from the Spec2Pipeline run"""
rtdata = rtdata_module
collect_pipeline_cfgs("config")
# Note that we use the truth file from spec2 processing as the input to spec3
rtdata.get_data(TRUTH_PATH + '/' + 'ifushort_ch12_cal.fits')
asn = asn_from_list([rtdata.input], product_name='ifushort_ch12_spec3')
asn.data["program"] = "00024"
asn.data["asn_type"] = "spec3"
asn.sequence = 1
asn_name, serialized = asn.dump(format="json")
with open(asn_name, "w") as f:
f.write(serialized)
rtdata.input = asn_name
args = [
"config/calwebb_spec3.cfg",
rtdata.input,
'--steps.master_background.save_results=true',
'--steps.mrs_imatch.save_results=true',
'--steps.outlier_detection.save_results=true',
'--steps.resample_spec.save_results=true',
'--steps.cube_build.save_results=true',
'--steps.extract_1d.save_results=true',
'--steps.combine_1d.save_results=true',
]
Step.from_cmdline(args)
return rtdata
def run_spec3(jail, run_spec2):
"""Run the Spec3Pipeline on the results from the Spec2Pipeline run"""
rtdata = run_spec2
# Create the level 3 `spec3` association from the output product
# name of `run_spec2`
product = run_spec2.asn['products'][0]['name']
input_name = replace_suffix(product, 'cal') + '.fits'
asn = asn_from_list([input_name], product_name=product)
_, serialized = asn.dump()
asn_name = product + '_spec3_asn.json'
with open(asn_name, 'w') as asn_fh:
asn_fh.write(serialized)
# Set the input in the RegtestData instance to avoid download.
rtdata.input = asn_name
step_params = {
'step': 'calwebb_spec3.cfg',
'args': [
'--steps.master_background.save_results=true',
'--steps.mrs_imatch.save_results=true',
'--steps.outlier_detection.save_results=true',
'--steps.resample_spec.save_results=true',
'--steps.cube_build.save_results=true',
'--steps.extract_1d.save_results=true',
'--steps.combine_1d.save_results=true',
]
}
rtdata = rt.run_step_from_dict(rtdata, **step_params)
return rtdata
def source_folder(tmp_path_factory):
"""Create a set of source associations"""
primary_members = [
('sci_1.txt', 'science'),
('sci_2.txt', 'science'),
('bkg_1.txt', 'background'),
('imprint_1.txt', 'imprint'),
]
source_folder = tmp_path_factory.mktemp('asn_gather_source')
with pushdir(source_folder):
# Create all the files
for expname, exptype in primary_members:
with open(expname, 'w') as fh:
fh.write(expname)
# Create the association
primary = asn_from_list(primary_members,
product_name=PRIMARY_STEM,
with_exptype=True)
_, serialized = primary.dump()
with open(PRIMARY_NAME, 'w') as fh:
fh.write(serialized)
return source_folder
def test_update_path_level2():
members = ['a', 'b', 'c']
new_path = 'new_path'
asn = asn_from_list(members, rule=DMSLevel2bBase)
update_path(asn, new_path)
for product in asn['products']:
for member in product['members']:
assert member['expname'].startswith(new_path)
def test_default_fail():
"""Test default DMS_Level3_Base fail
A product name needs to be included, but is not.
"""
items = ['a']
with pytest.raises((AssociationNotValidError)):
asn = asn_from_list(items)
def run_pipeline(files, flat_field=True):
"""
Runs all stages of the JWST Pipeline on uncalibrated imaging data.
Parameters
----------
files : list
The files to run the pipeline on.
flat_field : bool
Option to run the flat field step in image2.
Returns
-------
outname : str
The name of the final i2d drizzled image.
"""
# Create a name for image3 association and drizzled files
detector = fits.getheader(files[0])['DETECTOR'].lower()
fltr = fits.getheader(files[0])['FILTER'].lower()
pupil = fits.getheader(files[0])['PUPIL'].lower()
target = fits.getheader(files[0])['TARGNAME'].lower()
name = '{}_{}_{}_{}'.format(detector, fltr, pupil, target)
# Run detector1
for f in files:
m = Detector1Pipeline()
#m.jump.override_readnoise = 'jwst_nircam_readnoise_0024_psub.fits'
m.refpix.odd_even_rows = False # only for MIRI
m.ipc.skip = True
m.persistence.skip = True
m.save_results = True
m.output_dir = os.getcwd()
m.run(f)
# Run image2
files = [f.replace('_uncal.fits', '_rate.fits') for f in files]
for f in files:
m = Image2Pipeline()
if not flat_field:
m.flat_field.skip = True
m.save_results = True
m.output_dir = os.getcwd()
m.run(f)
# Run image3
files = [f.replace('_rate.fits', '_cal.fits') for f in files]
asn = asn_from_list(files, rule=Asn_Lv3Image, product_name=name)
asn_file = '{}.json'.format(name)
with open(asn_file, 'w') as f:
f.write(asn.dump()[1])
m = Image3Pipeline()
m.save_results = True
m.output_dir = os.getcwd()
m.run(asn_file)
return '{}_i2d.fits'.format(name)
def test_base_roundtrip():
"""Write/read created base association"""
items = ['a', 'b', 'c']
asn = asn_from_list(items, rule=Association)
name, serialized = asn.dump()
reloaded = load_asn(serialized, registry=None)
assert asn['asn_rule'] == reloaded['asn_rule']
assert asn['asn_type'] == reloaded['asn_type']
assert asn['members'] == reloaded['members']
def make_level2_association(file_list, asn_filename):
idx = file_list[0].find('nrca1')
prod_name = file_list[0][0:idx + 5]
asn = asn_from_list(file_list, rule=DMSLevel2bBase, product_name=prod_name)
outfile = os.path.join(
'/ifs/jwst/wit/nircam/simulationWG/Imaging/CAR-19/Pipeline_Level2/',
asn_filename)
with open(outfile, 'w') as fh:
fh.write(asn.dump()[1])
def wfs_association(tmp_path_factory):
imsize = 10
tmp_path = tmp_path_factory.mktemp("wfs")
im1 = datamodels.ImageModel((imsize, imsize))
im1.meta.wcsinfo = {
'dec_ref': 11.99875540218638,
'ra_ref': 22.02351763251896,
'roll_ref': 0.005076934167039675,
'v2_ref': 86.039011,
'v3_ref': -493.385704,
'v3yangle': -0.07385127,
'vparity': -1,
'wcsaxes': 2
}
im1.meta.instrument = {
'channel': 'SHORT',
'detector': 'NRCA4',
'filter': 'F212N',
'module': 'A',
'name': 'NIRCAM',
'pupil': 'CLEAR'
}
im1.meta.observation = {
'exposure_number': '1',
'date': '2021-10-25',
'time': '16:58:27.258'
}
im1.meta.exposure = {'type': 'NRC_IMAGE'}
im1 = AssignWcsStep.call(im1, sip_approx=False)
im2 = im1.copy()
im2.meta.observation = {
'exposure_number': '2',
'date': '2021-10-25',
'time': '17:58:27.258'
}
path1 = str(tmp_path / "image1_cal.fits")
path2 = str(tmp_path / "image2_cal.fits")
im1.save(path1)
im2.save(path2)
asn = asn_from_list(
[path1, path2],
product_name='jw00024-a3001_t001_nircam_nrca4_{suffix}')
asn.data["program"] = "00024"
asn.data["asn_type"] = "wfs-image2"
asn.sequence = 1
asn_name, serialized = asn.dump(format="json")
path_asn = tmp_path / asn_name
with open(path_asn, "w") as f:
f.write(serialized)
return path_asn, path1, path2
def generate_tso3_asn():
"""Generate an association file that references the output of run_spec2_pipeline."""
asn = asn_from_list(
[f"{DATASET_ID}_calints.fits"],
product_name=PRODUCT_NAME,
)
name, serialized = asn.dump(format="json")
with open(name, "w") as f:
f.write(serialized)
return name, asn.acid.id
def generate_tso3_asn():
"""Generate an association file that references the output of run_spec2_pipeline."""
asn = asn_from_list([f"{DATASET_ID}_calints.fits"], product_name=PRODUCT_NAME)
asn.data["program"] = PROGRAM
asn.data["asn_type"] = "tso3"
asn.sequence = 1
name, serialized = asn.dump(format="json")
with open(name, "w") as f:
f.write(serialized)
return name, asn["asn_id"]
def test_modelcontainer_error_from_asn(tmpdir):
from jwst.associations.asn_from_list import asn_from_list
asn = asn_from_list(["foo.fits"], product_name="foo_out")
name, serialized = asn.dump(format="json")
path = str(tmpdir.join(name))
with open(path, "w") as f:
f.write(serialized)
# The foo.fits file doesn't exist
with pytest.raises(FileNotFoundError):
datamodels.open(path)
def test_modelcontainer_error_from_asn(tmp_path):
asn = asn_from_list(["foo.fits"], product_name="foo_out")
name, serialized = asn.dump(format="json")
# The following Path object needs to be stringified because
# datamodels.open() doesn't deal with pathlib objects when they are
# .json files
path = str(tmp_path / name)
with open(path, "w") as f:
f.write(serialized)
# The foo.fits file doesn't exist
with pytest.raises(FileNotFoundError):
datamodels.open(path)
def test_file_ext():
"""check that the filename extension is correctly appended"""
items = ['a', 'b', 'c']
asn = asn_from_list(items, rule=DMSLevel2bBase)
#check that extension defaults to json
name, serialized = asn.dump()
#check that extension with format = 'json' returns json
assert name.endswith('json')
name, serialized = asn.dump(format='json')
assert name.endswith('json')
#check that extension with format = 'yaml' returns yaml
name, serialized = asn.dump(format='yaml')
assert name.endswith('yaml')
def test_api_list():
"""Test api call with simple list"""
product_name = 'test_product'
inlist = ['a', 'b', 'c']
asn = asn_from_list(inlist, product_name=product_name)
assert len(asn['products']) == 1
assert asn['products'][0]['name'] == product_name
members = asn['products'][0]['members']
expnames = [
member['expname']
for member in members
]
assert inlist == expnames
def test_default_simple():
"""Default Level3 association"""
product_name = 'test_product'
items = ['a', 'b', 'c']
asn = asn_from_list(items, product_name=product_name)
assert asn['asn_rule'] == 'DMS_Level3_Base'
assert asn['asn_type'] == 'None'
assert len(asn['products']) == 1
product = asn['products'][0]
assert product['name'] == product_name
assert len(product['members']) == len(items)
for member in product['members']:
assert member['expname'] in items
assert member['exptype'] == 'science'
def test_default_roundtrip():
"""Create/Write/Read a Level3 association"""
product_name = 'test_product'
items = {
'a': 'science',
'b': 'target_acq',
'c': 'somethingelse'
}
asn = asn_from_list(
[(item, type_) for item, type_ in items.items()],
product_name=product_name,
with_exptype=True
)
name, serialized = asn.dump()
reloaded = load_asn(serialized)
assert asn['asn_rule'] == reloaded['asn_rule']
assert asn['asn_type'] == reloaded['asn_type']
assert len(asn['products']) == len(reloaded['products'])
def test_level2():
"""Create a level 2 association"""
items = ['a', 'b', 'c']
asn = asn_from_list(items, rule=DMSLevel2bBase)
assert asn['asn_rule'] == 'DMSLevel2bBase'
assert asn['asn_type'] == 'None'
products = asn['products']
assert len(products) == len(items)
for product in products:
assert product['name'] in items
members = product['members']
assert len(members) == 1
member = members[0]
assert member['expname'] == product['name']
assert member['exptype'] == 'science'
name, serialized = asn.dump()
assert name.startswith('jwnoprogram-o999_none')
assert isinstance(serialized, str)
def run_image3(file_list, prefix, obs_number, suffix, filter, output_dir,
verbose):
mosaic = prefix + obs_number + '_' + suffix + '_' + filter
print("run_image3 : mosaic ", mosaic)
asn = asn_from_list(file_list, rule=DMS_Level3_Base, product_name=mosaic)
json_file = output_dir + prefix + obs_number + '_' + suffix + '_' + filter + '_level-3_association.json'
if (verbose > 0):
print("def run_image3: json_file is ", json_file)
with open(json_file, 'w') as fh:
fh.write(asn.dump()[1])
asn_file = os.path.join(output_dir, json_file)
with open(asn_file) as f_obj:
asn_data = json.load(f_obj)
if (verbose > 0):
print("def run_image3: asn_data", asn_data)
asn_data
# return
# Create an instance of the pipeline class
image3 = calwebb_image3.Image3Pipeline()
# Set some parameters that pertain to the
# entire pipeline
image3.output_dir = output_dir
image3.save_results = True
# Set some parameters that pertain to some of
# the individual steps
# image3.tweakreg.snr_threshold = 10.0 # 5.0 is the default
# image3.tweakreg.kernel_fwhm = 2.302 # 2.5 is the default
# image3.tweakreg.brightest = 20 # 100 is the default
image3.tweakreg.skip = True
image3.source_catalog.kernel_fwhm = 2.302 # pixels
image3.source_catalog.snr_threshold = 10.
# Call the run() method
image3.run(asn_file)
return
def test_level2_tuple():
"""Test level 2 association when passing in a tuple"""
items = [('file_1.fits', 'science'), ('file_2.fits', 'background'),
('file_3.fits', 'target_acquisition'),('file_4.fits', '')]
asn = asn_from_list(items, rule=DMSLevel2bBase)
assert asn['asn_rule'] == 'DMSLevel2bBase'
assert asn['asn_type'] == 'None'
products = asn['products']
assert len(products) == len(items)
for product in products:
assert product['name'] in ','.join(','.join(map(str, row)) for row in items)
members = product['members']
assert len(members) == 1
member = members[0]
assert os.path.splitext(member['expname'])[0] == product['name']
assert member['exptype'] == product['members'][0]['exptype']
# make sure '' defaults to 'science'
if not items[3][1]:
assert product['members'][0]['exptype'] == 'science'
def test_api_with_type():
"""Test api call with type tuple"""
product_name = 'test_product'
inlist = [
('a', 'science'),
('b', 'psf'),
('c', 'science')
]
asn = asn_from_list(inlist, product_name=product_name, with_exptype=True)
assert len(asn['products']) == 1
assert asn['products'][0]['name'] == product_name
members = asn['products'][0]['members']
members_dict = {
member['expname']: member['exptype']
for member in members
}
for name, type_ in inlist:
assert members_dict[name] == type_
def test_default_with_type():
"""Level3 association with types specified"""
product_name = 'test_product'
items = {
'a': 'science',
'b': 'target_acq',
'c': 'somethingelse'
}
asn = asn_from_list(
[(item, type_) for item, type_ in items.items()],
product_name=product_name,
with_exptype=True
)
assert asn['asn_rule'] == 'DMS_Level3_Base'
assert asn['asn_type'] == 'None'
assert len(asn['products']) == 1
product = asn['products'][0]
assert product['name'] == product_name
assert len(product['members']) == len(items)
for member in product['members']:
assert member['expname'] in items
assert member['exptype'] == items[member['expname']]
steps={"jump": {
"rejection_threshold": 10.
}})
det1s.append(det1)
# now identify the filename of the rateints file, which is the one we want to continue with. load into a model and add to list.
di = det1.meta.filename.split('.')[0] + 'ints.fits'
dimod = datamodels.open(od + di)
dimods.append(dimod)
# run the Image2Pipeline
im2 = Image2Pipeline.call(dimod,
config_file='cfg_files/calwebb_tso-image2.cfg',
save_results=True,
output_dir=od)
im2mods.append(im2[0])
#pdb.set_trace()
for ii, od, sarr in zip(im2mods, odirs, subarr):
# create an association file for the TSO3 Pipeline
asn3_files = [od + ii.meta.filename]
asn_fname = 'tso3_tsoim_{0}_asn.json'.format(sarr)
asn3 = asn_from_list(asn3_files,
rule=DMS_Level3_Base,
product_name=asn_fname.split('.')[0])
with open(asn_fname, 'w') as fp:
fp.write(asn3.dump()[1])
im3 = Tso3Pipeline.call(asn_fname, save_results=True, output_dir=od)
im3mods.append(im3)
def test_asn_version():
"""Test version in association is package version"""
asn = asn_from_list(['a', 'b', 'c'], product_name='aproduct')
assert asn['code_version'] == __version__
output_dir = reduced
print("output_dir ", output_dir)
# exit(0)
for visit in range(1, nvisits + 1):
visit_number = "%03d" % (visit)
visit_prefix = reduced + prefix + visit_number + sca + '*rate.fits'
print("visit_prefix is", visit_prefix)
file_list = sorted(glob.glob(visit_prefix))
if (len(file_list) == 0):
print('no files for ', visit_prefix)
continue
print(file_list)
asn = asn_from_list(file_list,
rule=DMSLevel2bBase,
products='calibrated_image')
json_file = reduced + 'apt_0' + apt_number + visit_number + '_level-2b_association.json'
with open(json_file, 'w') as fh:
fh.write(asn.dump()[1])
asn_file = os.path.join(output_dir, json_file)
with open(asn_file) as f_obj:
asn_data = json.load(f_obj)
print("asn_data")
asn_data
# Create an instance of the pipeline class
image2 = calwebb_image2.Image2Pipeline()
