def reduce(file_list,
label,
calib_files,
recipe_name=None,
save_to=None,
user_pars=None):
"""
Helper function used to prevent replication of code.
Parameters
----------
file_list : list
List of files that will be reduced.
label : str
Labed used on log files name.
calib_files : list
List of calibration files properly formatted for DRAGONS Reduce().
recipe_name : str, optional
Name of the recipe used to reduce the data.
save_to : str, optional
Stores the calibration files locally in a list.
user_pars : list, optional
List of user parameters
Returns
-------
str : Output reduced file.
list : An updated list of calibration files.
"""
objgraph = pytest.importorskip("objgraph")
logutils.get_logger().info("\n\n\n")
logutils.config(file_name=f"test_image_{label}.log")
r = Reduce()
r.files = file_list
r.ucals = normalize_ucals(r.files, calib_files)
r.uparms = user_pars
if recipe_name:
r.recipename = recipe_name
r.runr()
output_file = r.output_filenames[0]
if save_to:
calib_files.append("{}:{}".format(
save_to,
os.path.join("calibrations", save_to, r.output_filenames[0])))
[os.remove(f) for f in r.output_filenames]
# check that we are not leaking objects
assert len(objgraph.by_type('NDAstroData')) == 0
return output_file, calib_files
def test_reduce_image(path_to_inputs):
calib_files = []
all_files = glob.glob(
os.path.join(path_to_inputs, 'GSAOI/test_reduce/', '*.fits'))
all_files.sort()
assert len(all_files) > 1
list_of_darks = dataselect.select_data(all_files, ['DARK'], [])
list_of_darks.sort()
list_of_kshort_flats = dataselect.select_data(
all_files, ['FLAT'], [],
dataselect.expr_parser('filter_name=="Kshort"'))
list_of_kshort_flats.sort()
list_of_h_flats = dataselect.select_data(
all_files, ['FLAT'], [], dataselect.expr_parser('filter_name=="H"'))
list_of_h_flats.sort()
list_of_science_files = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser(
'observation_class=="science" and exposure_time==60.'))
list_of_science_files.sort()
for darks in [list_of_darks]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
logutils.config(file_name='gsaoi_test_reduce_dark.log', mode='quiet')
reduce_darks.runr()
del reduce_darks
logutils.config(file_name='gsaoi_test_reduce_bpm.log', mode='quiet')
reduce_bpm = Reduce()
reduce_bpm.files.extend(list_of_h_flats)
reduce_bpm.files.extend(list_of_darks)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
del reduce_bpm
logutils.config(file_name='gsaoi_test_reduce_flats.log', mode='quiet')
reduce_flats = Reduce()
reduce_flats.files.extend(list_of_kshort_flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
calib_files.append('processed_flat:{}'.format(
reduce_flats.output_filenames[0]))
del reduce_flats
logutils.config(file_name='gsaoi_test_reduce_science.log', mode='quiet')
reduce_target = Reduce()
reduce_target.files.extend(list_of_science_files)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.ucals = normalize_ucals(reduce_target.files, calib_files)
reduce_target.runr()
del reduce_target
def test_reduce_image(test_path, caldb):
logutils.config(file_name='gsaoi_test_reduce_image.log')
caldb.init(wipe=True)
all_files = glob.glob(
os.path.join(test_path, 'GSAOI/test_reduce/', '*.fits'))
assert len(all_files) > 1
list_of_darks = dataselect.select_data(all_files, ['DARK'], [])
list_of_kshort_flats = dataselect.select_data(
all_files, ['FLAT'], [],
dataselect.expr_parser('filter_name=="Kshort"'))
list_of_h_flats = dataselect.select_data(
all_files, ['FLAT'], [], dataselect.expr_parser('filter_name=="H"'))
list_of_std_LHS_2026 = dataselect.select_data(
all_files, [], [], dataselect.expr_parser('object=="LHS 2026"'))
list_of_std_cskd8 = dataselect.select_data(
all_files, [], [], dataselect.expr_parser('object=="cskd-8"'))
list_of_science_files = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser(
'observation_class=="science" and exposure_time==60.'))
for darks in [list_of_darks]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
reduce_darks.runr()
caldb.add_cal(reduce_darks.output_filenames[0])
reduce_bpm = Reduce()
reduce_bpm.files.extend(list_of_h_flats)
reduce_bpm.files.extend(list_of_darks)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
reduce_flats = Reduce()
reduce_flats.files.extend(list_of_kshort_flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
caldb.add_cal(reduce_flats.output_filenames[0])
reduce_target = Reduce()
reduce_target.files.extend(list_of_science_files)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.runr()
for f in caldb.list_files():
print(f)
def test_reduce_image(change_working_dir):
with change_working_dir():
calib_files = []
all_files = [download_from_archive(f) for f in datasets]
all_files.sort()
assert len(all_files) > 1
darks_3s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==3'))
darks_3s.sort()
darks_20s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==20'))
darks_20s.sort()
darks_120s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==120'))
darks_120s.sort()
flats = dataselect.select_data(
all_files, ['F2', 'FLAT', 'RAW'], [],
dataselect.expr_parser('filter_name=="Y"'))
flats.sort()
science = dataselect.select_data(
all_files, ['F2', 'RAW'], ['CAL'],
dataselect.expr_parser('filter_name=="Y"'))
science.sort()
for darks in [darks_3s, darks_20s, darks_120s]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
logutils.config(file_name='f2_test_reduce_darks.log', mode='quiet')
reduce_darks.runr()
calib_files.append('processed_dark:{}'.format(
reduce_darks.output_filenames[0]))
logutils.config(file_name='f2_test_reduce_bpm.log', mode='quiet')
reduce_bpm = Reduce()
reduce_bpm.files.extend(flats)
assert len(reduce_bpm.files) == len(flats)
reduce_bpm.files.extend(darks_3s)
assert len(reduce_bpm.files) == len(flats) + len(darks_3s)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
logutils.config(file_name='f2_test_reduce_flats.log', mode='quiet')
reduce_flats = Reduce()
reduce_flats.files.extend(flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
calib_files.append('processed_flat:{}'.format(
reduce_flats.output_filenames[0]))
logutils.config(file_name='f2_test_reduce_science.log', mode='quiet')
reduce_target = Reduce()
reduce_target.files.extend(science)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.ucals = normalize_ucals(reduce_target.files, calib_files)
reduce_target.runr()
def test_reduce_image(test_path, caldb):
logutils.config(file_name='gsaoi_test_reduce_image.log')
caldb.init(wipe=True)
all_files = glob.glob(
os.path.join(test_path, 'GSAOI/test_reduce/', '*.fits'))
assert len(all_files) > 1
list_of_darks = dataselect.select_data(
all_files, ['DARK'], [])
list_of_kshort_flats = dataselect.select_data(
all_files, ['FLAT'], [],
dataselect.expr_parser('filter_name=="Kshort"'))
list_of_h_flats = dataselect.select_data(
all_files, ['FLAT'], [],
dataselect.expr_parser('filter_name=="H"'))
list_of_std_LHS_2026 = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser('object=="LHS 2026"'))
list_of_std_cskd8 = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser('object=="cskd-8"'))
list_of_science_files = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser('observation_class=="science" and exposure_time==60.'))
for darks in [list_of_darks]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
reduce_darks.runr()
caldb.add_cal(reduce_darks.output_filenames[0])
reduce_bpm = Reduce()
reduce_bpm.files.extend(list_of_h_flats)
reduce_bpm.files.extend(list_of_darks)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
reduce_flats = Reduce()
reduce_flats.files.extend(list_of_kshort_flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
caldb.add_cal(reduce_flats.output_filenames[0])
reduce_target = Reduce()
reduce_target.files.extend(list_of_science_files)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.runr()
for f in caldb.list_files():
print(f)
def test_reduce_image(test_path, caldb):
logutils.config(file_name='f2_test_reduce_image.log')
caldb.init(wipe=True)
all_files = glob.glob(
os.path.join(test_path, 'F2/test_reduce/', '*.fits'))
assert len(all_files) > 1
darks_3s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==3'))
darks_20s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==20'))
darks_120s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==120'))
flats = dataselect.select_data(
all_files, ['F2', 'FLAT', 'RAW'], [],
dataselect.expr_parser('filter_name=="Y"'))
science = dataselect.select_data(
all_files, ['F2', 'RAW'], ['CAL'],
dataselect.expr_parser('filter_name=="Y"'))
for darks in [darks_3s, darks_20s, darks_120s]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
reduce_darks.runr()
caldb.add_cal(reduce_darks.output_filenames[0])
reduce_bpm = Reduce()
reduce_bpm.files.extend(flats)
reduce_bpm.files.extend(darks_3s)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
reduce_flats = Reduce()
reduce_flats.files.extend(flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
caldb.add_cal(reduce_flats.output_filenames[0])
reduce_target = Reduce()
reduce_target.files.extend(science)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.runr()
for f in caldb.list_files():
print(f)
def test_reduce_image(path_to_inputs):
calib_files = []
all_files = glob.glob(
os.path.join(path_to_inputs, 'F2/test_reduce/', '*.fits'))
all_files.sort()
assert len(all_files) > 1
darks_3s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==3'))
darks_3s.sort()
darks_20s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==20'))
darks_20s.sort()
darks_120s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==120'))
darks_120s.sort()
flats = dataselect.select_data(all_files, ['F2', 'FLAT', 'RAW'], [],
dataselect.expr_parser('filter_name=="Y"'))
flats.sort()
science = dataselect.select_data(
all_files, ['F2', 'RAW'], ['CAL'],
dataselect.expr_parser('filter_name=="Y"'))
science.sort()
for darks in [darks_3s, darks_20s, darks_120s]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
logutils.config(file_name='f2_test_reduce_darks.log', mode='quiet')
reduce_darks.runr()
calib_files.append('processed_dark:{}'.format(
reduce_darks.output_filenames[0]))
logutils.config(file_name='f2_test_reduce_bpm.log', mode='quiet')
reduce_bpm = Reduce()
reduce_bpm.files.extend(flats)
reduce_bpm.files.extend(darks_3s)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
logutils.config(file_name='f2_test_reduce_flats.log', mode='quiet')
reduce_flats = Reduce()
reduce_flats.files.extend(flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
calib_files.append('processed_flat:{}'.format(
reduce_flats.output_filenames[0]))
logutils.config(file_name='f2_test_reduce_science.log', mode='quiet')
reduce_target = Reduce()
reduce_target.files.extend(science)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.ucals = normalize_ucals(reduce_target.files, calib_files)
reduce_target.runr()
def test_reduce_image_GS_HAM_2x2_i_std(path_to_inputs):
logutils.config(file_name='gmos_test_reduce_image_GS_HAM_1x1_i.log')
calib_files = []
raw_subdir = 'GMOS/GS-2017B-Q-6'
all_files = sorted(glob.glob(
os.path.join(path_to_inputs, raw_subdir, '*.fits')))
assert len(all_files) > 1
list_of_sci_bias = dataselect.select_data(
all_files,
['BIAS'],
[],
dataselect.expr_parser('detector_x_bin==2 and detector_y_bin==2')
)
list_of_sci_flats = dataselect.select_data(
all_files,
['TWILIGHT'],
[],
dataselect.expr_parser(
'filter_name=="i" and detector_x_bin==2 and detector_y_bin==2'
)
)
list_of_science_files = dataselect.select_data(
all_files, [],
[],
dataselect.expr_parser(
'observation_class=="partnerCal" and filter_name=="i"'
)
)
reduce_bias = Reduce()
assert len(reduce_bias.files) == 0
reduce_bias.files.extend(list_of_sci_bias)
assert len(reduce_bias.files) == len(list_of_sci_bias)
reduce_bias.runr()
calib_files.append(
'processed_bias:{}'.format(reduce_bias.output_filenames[0])
)
reduce_flats = Reduce()
reduce_flats.files.extend(list_of_sci_flats)
# reduce_flats.uparms = [('addDQ:user_bpm', 'fixed_bpm_2x2_FullFrame.fits')]
reduce_flats.ucals = normalize_ucals(reduce_flats.files, calib_files)
reduce_flats.runr()
calib_files.append(
'processed_flat:{}'.format(reduce_flats.output_filenames[0])
)
reduce_target = Reduce()
reduce_target.files.extend(list_of_science_files)
reduce_target.ucals = normalize_ucals(reduce_target.files, calib_files)
reduce_target.uparms = [
('stackFrames:memory', 1),
# ('addDQ:user_bpm', 'fixed_bpm_2x2_FullFrame.fits'),
('resampleToCommonFrame:interpolator', 'spline3')
]
reduce_target.runr()
def test_reduce_image(test_path, caldb):
logutils.config(file_name='f2_test_reduce_image.log')
caldb.init(wipe=True)
all_files = glob.glob(os.path.join(test_path, 'F2/test_reduce/', '*.fits'))
assert len(all_files) > 1
darks_3s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==3'))
darks_20s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==20'))
darks_120s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==120'))
flats = dataselect.select_data(all_files, ['F2', 'FLAT', 'RAW'], [],
dataselect.expr_parser('filter_name=="Y"'))
science = dataselect.select_data(
all_files, ['F2', 'RAW'], ['CAL'],
dataselect.expr_parser('filter_name=="Y"'))
for darks in [darks_3s, darks_20s, darks_120s]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
reduce_darks.runr()
caldb.add_cal(reduce_darks.output_filenames[0])
reduce_bpm = Reduce()
reduce_bpm.files.extend(flats)
reduce_bpm.files.extend(darks_3s)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
reduce_flats = Reduce()
reduce_flats.files.extend(flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
caldb.add_cal(reduce_flats.output_filenames[0])
reduce_target = Reduce()
reduce_target.files.extend(science)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.runr()
for f in caldb.list_files():
print(f)