def test_base_association(): """Create the simplest of associations""" items = ['a', 'b', 'c'] asn = asn_from_list(items, rule=Association) assert asn['asn_rule'] == 'Association' assert asn['asn_type'] == 'None' assert asn['members'] == items
def make_level3_association(file_list, asn_filename): asn = asn_from_list(file_list, product_name='lw_imaging') outfile = os.path.join( '/ifs/jwst/wit/nircam/simulationWG/Imaging/CAR-19/Pipeline_Level3/', asn_filename) with open(outfile, 'w') as fh: fh.write(asn.dump()[1])
def run_spec3(jail, rtdata_module): """Run the Spec3Pipeline on the single cal result from the Spec2Pipeline run""" rtdata = rtdata_module collect_pipeline_cfgs("config") # Note that we use the truth file from spec2 processing as the input to spec3 rtdata.get_data(TRUTH_PATH + '/' + 'ifushort_ch12_cal.fits') asn = asn_from_list([rtdata.input], product_name='ifushort_ch12_spec3') asn.data["program"] = "00024" asn.data["asn_type"] = "spec3" asn.sequence = 1 asn_name, serialized = asn.dump(format="json") with open(asn_name, "w") as f: f.write(serialized) rtdata.input = asn_name args = [ "config/calwebb_spec3.cfg", rtdata.input, '--steps.master_background.save_results=true', '--steps.mrs_imatch.save_results=true', '--steps.outlier_detection.save_results=true', '--steps.resample_spec.save_results=true', '--steps.cube_build.save_results=true', '--steps.extract_1d.save_results=true', '--steps.combine_1d.save_results=true', ] Step.from_cmdline(args) return rtdata
def run_spec3(jail, run_spec2): """Run the Spec3Pipeline on the results from the Spec2Pipeline run""" rtdata = run_spec2 # Create the level 3 `spec3` association from the output product # name of `run_spec2` product = run_spec2.asn['products'][0]['name'] input_name = replace_suffix(product, 'cal') + '.fits' asn = asn_from_list([input_name], product_name=product) _, serialized = asn.dump() asn_name = product + '_spec3_asn.json' with open(asn_name, 'w') as asn_fh: asn_fh.write(serialized) # Set the input in the RegtestData instance to avoid download. rtdata.input = asn_name step_params = { 'step': 'calwebb_spec3.cfg', 'args': [ '--steps.master_background.save_results=true', '--steps.mrs_imatch.save_results=true', '--steps.outlier_detection.save_results=true', '--steps.resample_spec.save_results=true', '--steps.cube_build.save_results=true', '--steps.extract_1d.save_results=true', '--steps.combine_1d.save_results=true', ] } rtdata = rt.run_step_from_dict(rtdata, **step_params) return rtdata
def source_folder(tmp_path_factory): """Create a set of source associations""" primary_members = [ ('sci_1.txt', 'science'), ('sci_2.txt', 'science'), ('bkg_1.txt', 'background'), ('imprint_1.txt', 'imprint'), ] source_folder = tmp_path_factory.mktemp('asn_gather_source') with pushdir(source_folder): # Create all the files for expname, exptype in primary_members: with open(expname, 'w') as fh: fh.write(expname) # Create the association primary = asn_from_list(primary_members, product_name=PRIMARY_STEM, with_exptype=True) _, serialized = primary.dump() with open(PRIMARY_NAME, 'w') as fh: fh.write(serialized) return source_folder
def test_update_path_level2(): members = ['a', 'b', 'c'] new_path = 'new_path' asn = asn_from_list(members, rule=DMSLevel2bBase) update_path(asn, new_path) for product in asn['products']: for member in product['members']: assert member['expname'].startswith(new_path)
def test_default_fail(): """Test default DMS_Level3_Base fail A product name needs to be included, but is not. """ items = ['a'] with pytest.raises((AssociationNotValidError)): asn = asn_from_list(items)
def run_pipeline(files, flat_field=True): """ Runs all stages of the JWST Pipeline on uncalibrated imaging data. Parameters ---------- files : list The files to run the pipeline on. flat_field : bool Option to run the flat field step in image2. Returns ------- outname : str The name of the final i2d drizzled image. """ # Create a name for image3 association and drizzled files detector = fits.getheader(files[0])['DETECTOR'].lower() fltr = fits.getheader(files[0])['FILTER'].lower() pupil = fits.getheader(files[0])['PUPIL'].lower() target = fits.getheader(files[0])['TARGNAME'].lower() name = '{}_{}_{}_{}'.format(detector, fltr, pupil, target) # Run detector1 for f in files: m = Detector1Pipeline() #m.jump.override_readnoise = 'jwst_nircam_readnoise_0024_psub.fits' m.refpix.odd_even_rows = False # only for MIRI m.ipc.skip = True m.persistence.skip = True m.save_results = True m.output_dir = os.getcwd() m.run(f) # Run image2 files = [f.replace('_uncal.fits', '_rate.fits') for f in files] for f in files: m = Image2Pipeline() if not flat_field: m.flat_field.skip = True m.save_results = True m.output_dir = os.getcwd() m.run(f) # Run image3 files = [f.replace('_rate.fits', '_cal.fits') for f in files] asn = asn_from_list(files, rule=Asn_Lv3Image, product_name=name) asn_file = '{}.json'.format(name) with open(asn_file, 'w') as f: f.write(asn.dump()[1]) m = Image3Pipeline() m.save_results = True m.output_dir = os.getcwd() m.run(asn_file) return '{}_i2d.fits'.format(name)
def test_base_roundtrip(): """Write/read created base association""" items = ['a', 'b', 'c'] asn = asn_from_list(items, rule=Association) name, serialized = asn.dump() reloaded = load_asn(serialized, registry=None) assert asn['asn_rule'] == reloaded['asn_rule'] assert asn['asn_type'] == reloaded['asn_type'] assert asn['members'] == reloaded['members']
def make_level2_association(file_list, asn_filename): idx = file_list[0].find('nrca1') prod_name = file_list[0][0:idx + 5] asn = asn_from_list(file_list, rule=DMSLevel2bBase, product_name=prod_name) outfile = os.path.join( '/ifs/jwst/wit/nircam/simulationWG/Imaging/CAR-19/Pipeline_Level2/', asn_filename) with open(outfile, 'w') as fh: fh.write(asn.dump()[1])
def wfs_association(tmp_path_factory): imsize = 10 tmp_path = tmp_path_factory.mktemp("wfs") im1 = datamodels.ImageModel((imsize, imsize)) im1.meta.wcsinfo = { 'dec_ref': 11.99875540218638, 'ra_ref': 22.02351763251896, 'roll_ref': 0.005076934167039675, 'v2_ref': 86.039011, 'v3_ref': -493.385704, 'v3yangle': -0.07385127, 'vparity': -1, 'wcsaxes': 2 } im1.meta.instrument = { 'channel': 'SHORT', 'detector': 'NRCA4', 'filter': 'F212N', 'module': 'A', 'name': 'NIRCAM', 'pupil': 'CLEAR' } im1.meta.observation = { 'exposure_number': '1', 'date': '2021-10-25', 'time': '16:58:27.258' } im1.meta.exposure = {'type': 'NRC_IMAGE'} im1 = AssignWcsStep.call(im1, sip_approx=False) im2 = im1.copy() im2.meta.observation = { 'exposure_number': '2', 'date': '2021-10-25', 'time': '17:58:27.258' } path1 = str(tmp_path / "image1_cal.fits") path2 = str(tmp_path / "image2_cal.fits") im1.save(path1) im2.save(path2) asn = asn_from_list( [path1, path2], product_name='jw00024-a3001_t001_nircam_nrca4_{suffix}') asn.data["program"] = "00024" asn.data["asn_type"] = "wfs-image2" asn.sequence = 1 asn_name, serialized = asn.dump(format="json") path_asn = tmp_path / asn_name with open(path_asn, "w") as f: f.write(serialized) return path_asn, path1, path2
def generate_tso3_asn(): """Generate an association file that references the output of run_spec2_pipeline.""" asn = asn_from_list( [f"{DATASET_ID}_calints.fits"], product_name=PRODUCT_NAME, ) name, serialized = asn.dump(format="json") with open(name, "w") as f: f.write(serialized) return name, asn.acid.id
def generate_tso3_asn(): """Generate an association file that references the output of run_spec2_pipeline.""" asn = asn_from_list([f"{DATASET_ID}_calints.fits"], product_name=PRODUCT_NAME) asn.data["program"] = PROGRAM asn.data["asn_type"] = "tso3" asn.sequence = 1 name, serialized = asn.dump(format="json") with open(name, "w") as f: f.write(serialized) return name, asn["asn_id"]
def test_modelcontainer_error_from_asn(tmpdir): from jwst.associations.asn_from_list import asn_from_list asn = asn_from_list(["foo.fits"], product_name="foo_out") name, serialized = asn.dump(format="json") path = str(tmpdir.join(name)) with open(path, "w") as f: f.write(serialized) # The foo.fits file doesn't exist with pytest.raises(FileNotFoundError): datamodels.open(path)
def test_modelcontainer_error_from_asn(tmp_path): asn = asn_from_list(["foo.fits"], product_name="foo_out") name, serialized = asn.dump(format="json") # The following Path object needs to be stringified because # datamodels.open() doesn't deal with pathlib objects when they are # .json files path = str(tmp_path / name) with open(path, "w") as f: f.write(serialized) # The foo.fits file doesn't exist with pytest.raises(FileNotFoundError): datamodels.open(path)
def test_file_ext(): """check that the filename extension is correctly appended""" items = ['a', 'b', 'c'] asn = asn_from_list(items, rule=DMSLevel2bBase) #check that extension defaults to json name, serialized = asn.dump() #check that extension with format = 'json' returns json assert name.endswith('json') name, serialized = asn.dump(format='json') assert name.endswith('json') #check that extension with format = 'yaml' returns yaml name, serialized = asn.dump(format='yaml') assert name.endswith('yaml')
def test_api_list(): """Test api call with simple list""" product_name = 'test_product' inlist = ['a', 'b', 'c'] asn = asn_from_list(inlist, product_name=product_name) assert len(asn['products']) == 1 assert asn['products'][0]['name'] == product_name members = asn['products'][0]['members'] expnames = [ member['expname'] for member in members ] assert inlist == expnames
def test_default_simple(): """Default Level3 association""" product_name = 'test_product' items = ['a', 'b', 'c'] asn = asn_from_list(items, product_name=product_name) assert asn['asn_rule'] == 'DMS_Level3_Base' assert asn['asn_type'] == 'None' assert len(asn['products']) == 1 product = asn['products'][0] assert product['name'] == product_name assert len(product['members']) == len(items) for member in product['members']: assert member['expname'] in items assert member['exptype'] == 'science'
def test_default_roundtrip(): """Create/Write/Read a Level3 association""" product_name = 'test_product' items = { 'a': 'science', 'b': 'target_acq', 'c': 'somethingelse' } asn = asn_from_list( [(item, type_) for item, type_ in items.items()], product_name=product_name, with_exptype=True ) name, serialized = asn.dump() reloaded = load_asn(serialized) assert asn['asn_rule'] == reloaded['asn_rule'] assert asn['asn_type'] == reloaded['asn_type'] assert len(asn['products']) == len(reloaded['products'])
def test_level2(): """Create a level 2 association""" items = ['a', 'b', 'c'] asn = asn_from_list(items, rule=DMSLevel2bBase) assert asn['asn_rule'] == 'DMSLevel2bBase' assert asn['asn_type'] == 'None' products = asn['products'] assert len(products) == len(items) for product in products: assert product['name'] in items members = product['members'] assert len(members) == 1 member = members[0] assert member['expname'] == product['name'] assert member['exptype'] == 'science' name, serialized = asn.dump() assert name.startswith('jwnoprogram-o999_none') assert isinstance(serialized, str)
def run_image3(file_list, prefix, obs_number, suffix, filter, output_dir, verbose): mosaic = prefix + obs_number + '_' + suffix + '_' + filter print("run_image3 : mosaic ", mosaic) asn = asn_from_list(file_list, rule=DMS_Level3_Base, product_name=mosaic) json_file = output_dir + prefix + obs_number + '_' + suffix + '_' + filter + '_level-3_association.json' if (verbose > 0): print("def run_image3: json_file is ", json_file) with open(json_file, 'w') as fh: fh.write(asn.dump()[1]) asn_file = os.path.join(output_dir, json_file) with open(asn_file) as f_obj: asn_data = json.load(f_obj) if (verbose > 0): print("def run_image3: asn_data", asn_data) asn_data # return # Create an instance of the pipeline class image3 = calwebb_image3.Image3Pipeline() # Set some parameters that pertain to the # entire pipeline image3.output_dir = output_dir image3.save_results = True # Set some parameters that pertain to some of # the individual steps # image3.tweakreg.snr_threshold = 10.0 # 5.0 is the default # image3.tweakreg.kernel_fwhm = 2.302 # 2.5 is the default # image3.tweakreg.brightest = 20 # 100 is the default image3.tweakreg.skip = True image3.source_catalog.kernel_fwhm = 2.302 # pixels image3.source_catalog.snr_threshold = 10. # Call the run() method image3.run(asn_file) return
def test_level2_tuple(): """Test level 2 association when passing in a tuple""" items = [('file_1.fits', 'science'), ('file_2.fits', 'background'), ('file_3.fits', 'target_acquisition'),('file_4.fits', '')] asn = asn_from_list(items, rule=DMSLevel2bBase) assert asn['asn_rule'] == 'DMSLevel2bBase' assert asn['asn_type'] == 'None' products = asn['products'] assert len(products) == len(items) for product in products: assert product['name'] in ','.join(','.join(map(str, row)) for row in items) members = product['members'] assert len(members) == 1 member = members[0] assert os.path.splitext(member['expname'])[0] == product['name'] assert member['exptype'] == product['members'][0]['exptype'] # make sure '' defaults to 'science' if not items[3][1]: assert product['members'][0]['exptype'] == 'science'
def test_api_with_type(): """Test api call with type tuple""" product_name = 'test_product' inlist = [ ('a', 'science'), ('b', 'psf'), ('c', 'science') ] asn = asn_from_list(inlist, product_name=product_name, with_exptype=True) assert len(asn['products']) == 1 assert asn['products'][0]['name'] == product_name members = asn['products'][0]['members'] members_dict = { member['expname']: member['exptype'] for member in members } for name, type_ in inlist: assert members_dict[name] == type_
def test_default_with_type(): """Level3 association with types specified""" product_name = 'test_product' items = { 'a': 'science', 'b': 'target_acq', 'c': 'somethingelse' } asn = asn_from_list( [(item, type_) for item, type_ in items.items()], product_name=product_name, with_exptype=True ) assert asn['asn_rule'] == 'DMS_Level3_Base' assert asn['asn_type'] == 'None' assert len(asn['products']) == 1 product = asn['products'][0] assert product['name'] == product_name assert len(product['members']) == len(items) for member in product['members']: assert member['expname'] in items assert member['exptype'] == items[member['expname']]
steps={"jump": { "rejection_threshold": 10. }}) det1s.append(det1) # now identify the filename of the rateints file, which is the one we want to continue with. load into a model and add to list. di = det1.meta.filename.split('.')[0] + 'ints.fits' dimod = datamodels.open(od + di) dimods.append(dimod) # run the Image2Pipeline im2 = Image2Pipeline.call(dimod, config_file='cfg_files/calwebb_tso-image2.cfg', save_results=True, output_dir=od) im2mods.append(im2[0]) #pdb.set_trace() for ii, od, sarr in zip(im2mods, odirs, subarr): # create an association file for the TSO3 Pipeline asn3_files = [od + ii.meta.filename] asn_fname = 'tso3_tsoim_{0}_asn.json'.format(sarr) asn3 = asn_from_list(asn3_files, rule=DMS_Level3_Base, product_name=asn_fname.split('.')[0]) with open(asn_fname, 'w') as fp: fp.write(asn3.dump()[1]) im3 = Tso3Pipeline.call(asn_fname, save_results=True, output_dir=od) im3mods.append(im3)
def test_asn_version(): """Test version in association is package version""" asn = asn_from_list(['a', 'b', 'c'], product_name='aproduct') assert asn['code_version'] == __version__
output_dir = reduced print("output_dir ", output_dir) # exit(0) for visit in range(1, nvisits + 1): visit_number = "%03d" % (visit) visit_prefix = reduced + prefix + visit_number + sca + '*rate.fits' print("visit_prefix is", visit_prefix) file_list = sorted(glob.glob(visit_prefix)) if (len(file_list) == 0): print('no files for ', visit_prefix) continue print(file_list) asn = asn_from_list(file_list, rule=DMSLevel2bBase, products='calibrated_image') json_file = reduced + 'apt_0' + apt_number + visit_number + '_level-2b_association.json' with open(json_file, 'w') as fh: fh.write(asn.dump()[1]) asn_file = os.path.join(output_dir, json_file) with open(asn_file) as f_obj: asn_data = json.load(f_obj) print("asn_data") asn_data # Create an instance of the pipeline class image2 = calwebb_image2.Image2Pipeline()