def test_build_coadd_file_name():
mock_sobjs = mock_specobjs('spec1d_file1')
spectrograph = load_spectrograph('keck_deimos')
source = SourceObject(mock_sobjs.specobjs[0], mock_sobjs.header,
'spec1d_file1', spectrograph, 'ra/dec')
assert build_coadd_file_name(
source) == 'J132436.41+271928.56_DEIMOS_20200130_20200130.fits'
source2 = SourceObject(mock_sobjs.specobjs[0], mock_sobjs.header,
'spec1d_file1', spectrograph, 'pixel')
assert build_coadd_file_name(
source2) == 'SPAT1234_DEIMOS_20200130_20200130.fits'
def test_exclude_source_objects(monkeypatch):
monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs)
file_list = ['spec1d_file1', 'spec1d_file2']
uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec')
par = pypeitpar.PypeItPar()
par['collate1d']['exclude_serendip'] = True
par['collate1d']['wv_rms_thresh'] = 0.1
filtered_list, excluded_msgs = exclude_source_objects(
uncollated_list, {'3003': 'Test Exclude`'}, par)
assert [so.spec_obj_list[0].NAME for so in filtered_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT5334_SLIT4934_DET02']
assert [so.spec1d_file_list[0]
for so in filtered_list] == ['spec1d_file1', 'spec1d_file1']
par['collate1d']['exclude_serendip'] = False
par['coadd1d']['ex_value'] = 'BOX'
par['collate1d']['wv_rms_thresh'] = None
filtered_list, excluded_msgs = exclude_source_objects(
uncollated_list, dict(), par)
assert [so.spec_obj_list[0].NAME for so in filtered_list] == [
'SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07',
'SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05',
'SPAT6934_SLIT6245_DET05'
]
assert [so.spec1d_file_list[0] for so in filtered_list] == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file2', 'spec1d_file2',
'spec1d_file2'
]
def main(args):
start_time = datetime.now()
(par, spectrograph, spec1d_files) = build_parameters(args)
outdir = par['collate1d']['outdir']
os.makedirs(outdir, exist_ok=True)
# Write the par to disk
if args.par_outfile is None:
args.par_outfile = os.path.join(outdir, 'collate1d.par')
print("Writing the parameters to {}".format(args.par_outfile))
# Gather up config lines for the sections relevant to collate_1d
config_lines = par['collate1d'].to_config(section_name='collate1d',
include_descr=False) + ['']
config_lines += par['coadd1d'].to_config(section_name='coadd1d',
include_descr=False)
if par['collate1d']['flux']:
config_lines += [''] + par['fluxcalib'].to_config(
section_name='fluxcalib', include_descr=False)
with open(args.par_outfile, "w") as f:
for line in config_lines:
print(line, file=f)
# Parse the tolerance based on the match type
if par['collate1d']['match_using'] == 'pixel':
tolerance = float(par['collate1d']['tolerance'])
else:
# For ra/dec matching, the default unit is arcseconds. We check for
# this case by seeing if the passed in tolerance is a floating point number
if is_float(par['collate1d']['tolerance']):
tolerance = float(par['collate1d']['tolerance'])
else:
tolerance = Angle(par['collate1d']['tolerance']).arcsec
# Filter out unwanted source objects based on our parameters.
# First filter them out based on the exclude_slit_trace_bm parameter
if len(par['collate1d']['exclude_slit_trace_bm']) > 0:
spec2d_files = find_spec2d_from_spec1d(spec1d_files)
exclude_map = find_slits_to_exclude(spec2d_files, par)
else:
spec2d_files = []
exclude_map = dict()
# Flux the spec1ds based on a archived sensfunc
failed_fluxing_msgs = []
if par['collate1d']['flux']:
spec1d_files = flux(par, spectrograph, spec1d_files,
failed_fluxing_msgs)
# Build source objects from spec1d file, this list is not collated
source_objects = SourceObject.build_source_objects(
spec1d_files, par['collate1d']['match_using'])
# Filter based on the coadding ex_value, and the exclude_serendip
# boolean
(objects_to_coadd,
excluded_obj_msgs) = exclude_source_objects(source_objects,
exclude_map, par)
# Collate the spectra
source_list = collate_spectra_by_source(objects_to_coadd, tolerance)
# Coadd the spectra
successful_source_list = []
failed_source_msgs = []
for source in source_list:
coaddfile = os.path.join(par['collate1d']['outdir'],
build_coadd_file_name(source))
msgs.info(f'Creating {coaddfile} from the following sources:')
for i in range(len(source.spec_obj_list)):
msgs.info(
f' {source.spec1d_file_list[i]}: {source.spec_obj_list[i].NAME} '
f'({source.spec_obj_list[i].MASKDEF_OBJNAME})')
if not args.dry_run:
try:
coadd(par, coaddfile, source)
successful_source_list.append(source)
except Exception:
formatted_exception = traceback.format_exc()
msgs.warn(formatted_exception)
msgs.warn(f"Failed to coadd {coaddfile}, skipping")
failed_source_msgs.append(f"Failed to coadd {coaddfile}:")
failed_source_msgs.append(formatted_exception)
# Create collate_report.dat
archive = create_report_archive(par)
archive.add(successful_source_list)
archive.save()
total_time = datetime.now() - start_time
write_warnings(par, excluded_obj_msgs, failed_source_msgs,
failed_fluxing_msgs, start_time, total_time)
msgs.info(f'Total duration: {total_time}')
return 0
def test_coadd(monkeypatch):
class mock_coadd:
def run(self):
return
def save(self, file):
return
def mock_get_instance(*args, **kwargs):
return mock_coadd()
with monkeypatch.context() as m:
monkeypatch.setattr(coadd1d.CoAdd1D, "get_instance", mock_get_instance)
par = pypeitpar.PypeItPar()
par['collate1d'] = pypeitpar.Collate1DPar()
par['coadd1d'] = pypeitpar.Coadd1DPar()
spectrograph = load_spectrograph('keck_deimos')
filenames = ['spec1d_file1', 'spec1d_file2']
specobjs_file1 = mock_specobjs(filenames[0])
specobjs_file2 = mock_specobjs(filenames[1])
header_file1 = mock_header(filenames[0])
header_file2 = mock_header(filenames[1])
# Both source object 1's SpecObj objects will have OPT_FLAM
file1_objects = [
3, # 'SPAT3233_SLIT3235_DET03'
5,
] # 'SPAT3236_SLIT3245_DET05'
# Only one of source object 2's SpecObj objects will have OPT_FLAM
file2_objects = [
0, # 'SPAT3234_SLIT3236_DET03'
4,
] # 'SPAT3237_SLIT3246_DET05'
source_object1 = SourceObject(
specobjs_file1.specobjs[file1_objects[0]], header_file1,
filenames[0], spectrograph, 'ra/dec')
for object in file1_objects[1:]:
source_object1.spec_obj_list.append(
specobjs_file1.specobjs[object])
source_object1.spec1d_file_list.append(filenames[0])
source_object1.spec1d_header_list.append(header_file1)
source_object2 = SourceObject(
specobjs_file2.specobjs[file2_objects[0]], header_file2,
filenames[1], spectrograph, 'ra/dec')
for object in file2_objects:
source_object2.spec_obj_list.append(
specobjs_file2.specobjs[object])
source_object2.spec1d_file_list.append(filenames[1])
source_object2.spec1d_header_list.append(header_file2)
# Test with out using fluxed data
par['collate1d']['ignore_flux'] = True
par['coadd1d']['flux_value'] = True
test_file = "test_coadd_name"
coadd(par, test_file, source_object1)
assert par['coadd1d']['flux_value'] == False
# Test using fluxed data
par['collate1d']['ignore_flux'] = False
par['coadd1d']['flux_value'] = False
coadd(par, test_file, source_object1)
assert par['coadd1d']['flux_value'] == True
# Test not using fluxed data because not all SpecObj objects had flux data
par['collate1d']['ignore_flux'] = False
par['coadd1d']['flux_value'] = True
coadd(par, test_file, source_object2)
assert par['coadd1d']['flux_value'] == False
def test_get_report_metadata(monkeypatch):
spectrograph = load_spectrograph('keck_deimos')
filenames = ['spec1d_file1', 'spec1d_file2']
specobjs_file1 = mock_specobjs(filenames[0])
specobjs_file2 = mock_specobjs(filenames[1])
header_file1 = mock_header(filenames[0])
header_file2 = mock_header(filenames[1])
file1_objects = [
3, # 'SPAT3233_SLIT3235_DET03'
5,
] # 'SPAT3236_SLIT3245_DET05'
file2_objects = [
0, # 'SPAT3234_SLIT3236_DET03'
4,
] # 'SPAT3237_SLIT3246_DET05'
source_object = SourceObject(specobjs_file1.specobjs[file1_objects[0]],
header_file1, filenames[0], spectrograph,
'ra/dec')
for object in file1_objects[1:]:
source_object.spec_obj_list.append(specobjs_file1.specobjs[object])
source_object.spec1d_file_list.append(filenames[0])
source_object.spec1d_header_list.append(header_file1)
for object in file2_objects:
source_object.spec_obj_list.append(specobjs_file2.specobjs[object])
source_object.spec1d_file_list.append(filenames[1])
source_object.spec1d_header_list.append(header_file2)
monkeypatch.setattr(fits, "getheader", mock_header)
(metadata_rows,
files_to_copy) = get_report_metadata(['DISPNAME', 'MJD', 'GUIDFHWM'],
['MASKDEF_OBJNAME', 'NAME'],
source_object)
dest_file = 'J132500.41+271959.88_DEIMOS_20200130_20200131.fits'
assert len(metadata_rows) == 4
assert metadata_rows[0] == [
dest_file, 'object3', 'SPAT3233_SLIT3235_DET03', 'spec1d_file1',
'830G', '58878.0', None
]
assert metadata_rows[1] == [
dest_file, 'object3', 'SPAT3236_SLIT3245_DET05', 'spec1d_file1',
'830G', '58878.0', None
]
assert metadata_rows[2] == [
dest_file, 'object3', 'SPAT3234_SLIT3236_DET03', 'spec1d_file2',
'830G', '58879.0', None
]
assert metadata_rows[3] == [
dest_file, 'object3', 'SPAT3237_SLIT3246_DET05', 'spec1d_file2',
'830G', '58879.0', None
]
assert files_to_copy is None
assert (None, None) == get_report_metadata(['DISPNAME', 'MJD', 'GUIDFHWM'],
['MASKDEF_OBJNAME', 'NAME'],
"afilename")
def test_config_key_match():
file_list = ['spec1d_file1', 'spec1d_file2']
spectrograph = load_spectrograph('keck_deimos')
header1 = {
'dispname': '830G',
'MJD': '58878.0',
'PYP_SPEC': 'keck_deimos',
'decker': 'Z6CL01B',
'binning': '1,1'
}
header2 = {
'dispname': '830G',
'MJD': '58878.0',
'PYP_SPEC': 'keck_deimos',
'decker': 'foo',
'binning': '1,1'
}
header3 = {
'dispname': '830L',
'MJD': '58878.0',
'PYP_SPEC': 'keck_deimos',
'decker': 'Z6CL01B',
'binning': '1,1'
}
header4 = {
'dispname': '830G',
'MJD': '58878.0',
'PYP_SPEC': 'keck_deimos',
'decker': 'Z6CL01B',
'binning': '1,1',
'amp': 'foo'
}
header5 = {
'dispname': '830G',
'MJD': '58878.0',
'decker': 'foo',
'binning': '1,1'
}
header6 = {
'dispname': '830G',
'MJD': '58878.0',
'PYP_SPEC': 'keck_nires',
'decker': 'foo',
'binning': '1,1'
}
sobjs = mock_specobjs('spec1d_file1')
sobj = sobjs.specobjs[0]
so1 = SourceObject(sobj, header1, 'spec1d_file1', spectrograph, 'ra/dec')
so4 = SourceObject(sobj, header4, 'spec1d_file1', spectrograph, 'ra/dec')
# Test that config keys match regardless of decker and even if neither
# header has amp
assert so1._config_key_match(header2) is True
# Test that config keys don't match if dispname doesn't match
assert so1._config_key_match(header3) is False
# Test that config keys don't match if amp doesn't match
# Also tests that the calling method match calls conifg_key_match correclty
assert so1.match(sobj, header4, 3.0) is False
# Test that config_key matching doesn't depend on order
assert so4._config_key_match(header1) is False
# Test that config keys don't match if there's no PYP_SPEC,
# or the wrong PY_SPEC
assert so1._config_key_match(header5) is False
assert so1._config_key_match(header6) is False
def test_group_spectra_by_pixel(monkeypatch):
monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs)
file_list = ['spec1d_file1', 'spec1d_file2']
spectrograph = load_spectrograph('keck_deimos')
# Test matching by pixel and that unit argument is ignored
uncollated_list = SourceObject.build_source_objects(file_list, 'pixel')
source_list = collate_spectra_by_source(uncollated_list, 5.0, u.deg)
assert len(source_list) == 7
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[4].spec_obj_list
] == ['SPAT6250_SLIT6235_DET03']
assert source_list[5].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[5].spec_obj_list
] == ['SPAT6256_SLIT6245_DET05']
assert source_list[6].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[6].spec_obj_list
] == ['SPAT6934_SLIT6245_DET05']
source_list = collate_spectra_by_source(uncollated_list, 10.0)
assert len(source_list) == 6
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == ['spec1d_file2', 'spec1d_file2']
assert [x.NAME for x in source_list[4].spec_obj_list
] == ['SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05']
assert source_list[5].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[5].spec_obj_list
] == ['SPAT6934_SLIT6245_DET05']
def test_group_spectra_by_radec(monkeypatch):
monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs)
file_list = ['spec1d_file1', 'spec1d_file2']
uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec')
source_list = collate_spectra_by_source(uncollated_list, 0.0003, u.deg)
assert len(source_list) == 6
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == ['spec1d_file2', 'spec1d_file2']
assert [x.NAME for x in source_list[4].spec_obj_list
] == ['SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05']
assert source_list[5].spec1d_file_list == ['spec1d_file2']
assert [x.NAME for x in source_list[5].spec_obj_list
] == ['SPAT6934_SLIT6245_DET05']
source_list = collate_spectra_by_source(uncollated_list, 1.44)
assert len(source_list) == 5
assert source_list[0].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[0].spec_obj_list
] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07']
assert source_list[1].spec1d_file_list == ['spec1d_file1', 'spec1d_file1']
assert [x.NAME for x in source_list[1].spec_obj_list
] == ['SPAT1334_SLIT1234_DET01', 'SPAT1336_SLIT1235_DET07']
assert source_list[2].spec1d_file_list == ['spec1d_file1']
assert [x.NAME for x in source_list[2].spec_obj_list
] == ['SPAT5334_SLIT4934_DET02']
assert source_list[3].spec1d_file_list == [
'spec1d_file1', 'spec1d_file1', 'spec1d_file1', 'spec1d_file2',
'spec1d_file2'
]
assert [x.NAME for x in source_list[3].spec_obj_list] == [
'SPAT3233_SLIT3235_DET03', 'SPAT3232_SLIT3235_DET03',
'SPAT3236_SLIT3245_DET05', 'SPAT3234_SLIT3236_DET03',
'SPAT3237_SLIT3246_DET05'
]
assert source_list[4].spec1d_file_list == [
'spec1d_file2', 'spec1d_file2', 'spec1d_file2'
]
assert [x.NAME for x in source_list[4].spec_obj_list] == [
'SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05',
'SPAT6934_SLIT6245_DET05'
]