def test_exclude_source_objects(monkeypatch):
monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs)
file_list = ['spec1d_file1', 'spec1d_file2']
uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec')
par = pypeitpar.PypeItPar()
par['collate1d']['exclude_serendip'] = True
par['collate1d']['wv_rms_thresh'] = 0.1
filtered_list, excluded_msgs = exclude_source_objects(
uncollated_list, {'3003': 'Test Exclude`'}, par)
assert [so.spec_obj_list[0].NAME for so in filtered_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT5334_SLIT4934_DET02']
assert [so.spec1d_file_list[0]
for so in filtered_list] == ['spec1d_file1', 'spec1d_file1']
par['collate1d']['exclude_serendip'] = False
par['coadd1d']['ex_value'] = 'BOX'
par['collate1d']['wv_rms_thresh'] = None
filtered_list, excluded_msgs = exclude_source_objects(
uncollated_list, dict(), par)
assert [so.spec_obj_list[0].NAME for so in filtered_list] == [
'SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07',
'SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05',
'SPAT6934_SLIT6245_DET05'
]
assert [so.spec1d_file_list[0] for so in filtered_list] == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file2', 'spec1d_file2',
'spec1d_file2'
]
def main(args):
start_time = datetime.now()
(par, spectrograph, spec1d_files) = build_parameters(args)
outdir = par['collate1d']['outdir']
os.makedirs(outdir, exist_ok=True)
# Write the par to disk
if args.par_outfile is None:
args.par_outfile = os.path.join(outdir, 'collate1d.par')
print("Writing the parameters to {}".format(args.par_outfile))
# Gather up config lines for the sections relevant to collate_1d
config_lines = par['collate1d'].to_config(section_name='collate1d',
include_descr=False) + ['']
config_lines += par['coadd1d'].to_config(section_name='coadd1d',
include_descr=False)
if par['collate1d']['flux']:
config_lines += [''] + par['fluxcalib'].to_config(
section_name='fluxcalib', include_descr=False)
with open(args.par_outfile, "w") as f:
for line in config_lines:
print(line, file=f)
# Parse the tolerance based on the match type
if par['collate1d']['match_using'] == 'pixel':
tolerance = float(par['collate1d']['tolerance'])
else:
# For ra/dec matching, the default unit is arcseconds. We check for
# this case by seeing if the passed in tolerance is a floating point number
if is_float(par['collate1d']['tolerance']):
tolerance = float(par['collate1d']['tolerance'])
else:
tolerance = Angle(par['collate1d']['tolerance']).arcsec
# Filter out unwanted source objects based on our parameters.
# First filter them out based on the exclude_slit_trace_bm parameter
if len(par['collate1d']['exclude_slit_trace_bm']) > 0:
spec2d_files = find_spec2d_from_spec1d(spec1d_files)
exclude_map = find_slits_to_exclude(spec2d_files, par)
else:
spec2d_files = []
exclude_map = dict()
# Flux the spec1ds based on a archived sensfunc
failed_fluxing_msgs = []
if par['collate1d']['flux']:
spec1d_files = flux(par, spectrograph, spec1d_files,
failed_fluxing_msgs)
# Build source objects from spec1d file, this list is not collated
source_objects = SourceObject.build_source_objects(
spec1d_files, par['collate1d']['match_using'])
# Filter based on the coadding ex_value, and the exclude_serendip
# boolean
(objects_to_coadd,
excluded_obj_msgs) = exclude_source_objects(source_objects,
exclude_map, par)
# Collate the spectra
source_list = collate_spectra_by_source(objects_to_coadd, tolerance)
# Coadd the spectra
successful_source_list = []
failed_source_msgs = []
for source in source_list:
coaddfile = os.path.join(par['collate1d']['outdir'],
build_coadd_file_name(source))
msgs.info(f'Creating {coaddfile} from the following sources:')
for i in range(len(source.spec_obj_list)):
msgs.info(
f' {source.spec1d_file_list[i]}: {source.spec_obj_list[i].NAME} '
f'({source.spec_obj_list[i].MASKDEF_OBJNAME})')
if not args.dry_run:
try:
coadd(par, coaddfile, source)
successful_source_list.append(source)
except Exception:
formatted_exception = traceback.format_exc()
msgs.warn(formatted_exception)
msgs.warn(f"Failed to coadd {coaddfile}, skipping")
failed_source_msgs.append(f"Failed to coadd {coaddfile}:")
failed_source_msgs.append(formatted_exception)
# Create collate_report.dat
archive = create_report_archive(par)
archive.add(successful_source_list)
archive.save()
total_time = datetime.now() - start_time
write_warnings(par, excluded_obj_msgs, failed_source_msgs,
failed_fluxing_msgs, start_time, total_time)
msgs.info(f'Total duration: {total_time}')
return 0
def test_group_spectra_by_pixel(monkeypatch):
monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs)
file_list = ['spec1d_file1', 'spec1d_file2']
spectrograph = load_spectrograph('keck_deimos')
# Test matching by pixel and that unit argument is ignored
uncollated_list = SourceObject.build_source_objects(file_list, 'pixel')
source_list = collate_spectra_by_source(uncollated_list, 5.0, u.deg)
assert len(source_list) == 7
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[4].spec_obj_list
] == ['SPAT6250_SLIT6235_DET03']
assert source_list[5].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[5].spec_obj_list
] == ['SPAT6256_SLIT6245_DET05']
assert source_list[6].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[6].spec_obj_list
] == ['SPAT6934_SLIT6245_DET05']
source_list = collate_spectra_by_source(uncollated_list, 10.0)
assert len(source_list) == 6
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == ['spec1d_file2', 'spec1d_file2']
assert [x.NAME for x in source_list[4].spec_obj_list
] == ['SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05']
assert source_list[5].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[5].spec_obj_list
] == ['SPAT6934_SLIT6245_DET05']
def test_group_spectra_by_radec(monkeypatch):
monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs)
file_list = ['spec1d_file1', 'spec1d_file2']
uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec')
source_list = collate_spectra_by_source(uncollated_list, 0.0003, u.deg)
assert len(source_list) == 6
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == ['spec1d_file2', 'spec1d_file2']
assert [x.NAME for x in source_list[4].spec_obj_list
] == ['SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05']
assert source_list[5].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[5].spec_obj_list
] == ['SPAT6934_SLIT6245_DET05']
source_list = collate_spectra_by_source(uncollated_list, 1.44)
assert len(source_list) == 5
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == [
'spec1d_file2', 'spec1d_file2', 'spec1d_file2'
]
assert [x.NAME for x in source_list[4].spec_obj_list] == [
'SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05',
'SPAT6934_SLIT6245_DET05'
]