def create_quartz_inputs(): """ Creates input data for tests using pre-processed twilight flat data and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ associated_calibrations = { "N20200715S0059.fits": { "quartz": ["N20200715S0059.fits"], }, "S20130601S0121.fits": { "quartz": ["S20130601S0121.fits"], }, "S20190204S0081.fits": { "quartz": ["S20190204S0081.fits"], }, } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/longslit/test_slit_illum_correct/inputs" path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) cwd = os.getcwd() os.chdir(path) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Download raw files') quartz_path = [download_from_archive(f) for f in cals['quartz']] quartz_ad = astrodata.open(quartz_path[0]) data_label = quartz_ad.data_label() print('Reducing quartz lamp:') logutils.config(file_name='log_quartz_{}.txt'.format(data_label)) with warnings.catch_warnings(): warnings.simplefilter("ignore") p = GMOSLongslit([astrodata.open(f) for f in quartz_path]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() # p.biasCorrect(bias=bias_master) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.stackFrames() # Write non-mosaicked data p.writeOutputs(suffix="_quartz", strip=True) os.chdir(cwd)
def test_add_illum_mask_position(filename, start_row): file_on_disk = download_from_archive(filename) ad = astrodata.open(file_on_disk) p = GMOSLongslit([ad]) p.prepare() ad = p.addIllumMaskToDQ().pop() # Chop off the bottom 200 rows because of the bias issue # and the bottom of the longslit mask for pre-Hamamatsu data # (larger CCDs) and choose a middle column in case of edge effects actual_start_row = (ad[0].mask[200:, 100] & 64).argmax() + 200 assert abs(start_row - actual_start_row) <= 2
def process_object(filename, arc, suffix="linearized"): """ Helper recipe to reduce the object file. Returns ------- AstroData Processed arc. """ from astrodata.testing import download_from_archive from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit processed_filename, ext = os.path.splitext(filename) processed_filename += "_{:s}{:s}".format(suffix, ext) if os.path.exists(processed_filename): ad = astrodata.open(processed_filename) else: if os.path.exists(filename): ad = astrodata.open(filename) else: ad = astrodata.open( download_from_archive(filename, path='', env_var='DRAGONS_TEST')) p = GMOSLongslit([ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() # p.biasCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) # p.flatCorrect() # p.applyQECorrection() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() p.linearizeSpectra() # TODO: needed? p.calculateSensitivity() ad = p.streams['main'][0] ad.write(overwrite=True) return ad
def reduce(filename): _p = GMOSLongslit([astrodata.open(filename)]) _p.viewer = geminidr.dormantViewer(_p, None) _p.prepare() _p.addDQ(static_bpm=None) _p.addVAR(read_noise=True) _p.overscanCorrect() _p.ADUToElectrons() _p.addVAR(poisson_noise=True) _p.mosaicDetectors() _p.makeIRAFCompatible() _p.determineWavelengthSolution(suffix="_arc") return _p
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from geminidr.gmos.tests.spect import CREATED_INPUTS_PATH_FOR_TESTS module_name, _ = os.path.splitext(os.path.basename(__file__)) path = os.path.join(CREATED_INPUTS_PATH_FOR_TESTS, module_name) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("inputs/", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename in datasets: print('Downloading files...') basename = filename.split("_")[0] + ".fits" sci_path = download_from_archive(basename) sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.makeIRAFCompatible() os.chdir("inputs/") processed_ad = p.writeOutputs().pop() os.chdir("..") print("Wrote pre-processed file to:\n" " {:s}".format(processed_ad.filename))
def process_arc(filename, suffix="distortionDetermined"): """ Helper recipe to reduce the arc file. Returns ------- AstroData Processed arc. """ from astrodata.testing import download_from_archive from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit processed_filename, ext = os.path.splitext(filename) processed_filename += "_{:s}{:s}".format(suffix, ext) if os.path.exists(processed_filename): ad = astrodata.open(processed_filename) else: if os.path.exists(filename): ad = astrodata.open(filename) else: ad = astrodata.open( download_from_archive(filename, path='', env_var='DRAGONS_TEST')) p = GMOSLongslit([ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() # p.biasCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.makeIRAFCompatible() p.determineWavelengthSolution() p.determineDistortion() ad = p.streams['main'][0] ad.write(overwrite=True) return ad
def create_twilight_inputs(): """ Creates input data for tests using pre-processed twilight flat data and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ associated_calibrations = { "S20190204S0006.fits": { "bias": [ "S20190203S0110.fits", "S20190203S0109.fits", "S20190203S0108.fits", "S20190203S0107.fits", "S20190203S0106.fits" ], "twilight": ["S20190204S0006.fits"], }, "N20190103S0462.fits": { "bias": [ "N20190102S0531.fits", "N20190102S0530.fits", "N20190102S0529.fits", "N20190102S0528.fits", "N20190102S0527.fits" ], "twilight": ["N20190103S0462.fits", "N20190103S0463.fits"], }, "N20190327S0056.fits": { "bias": [ "N20190327S0098.fits", "N20190327S0099.fits", "N20190327S0100.fits", "N20190327S0101.fits", "N20190327S0102.fits" ], "twilight": ["N20190327S0056.fits"], }, "S20130602S0005.fits": { "bias": [ "S20130601S0161.fits", "S20130601S0160.fits", "S20130601S0159.fits", "S20130601S0158.fits", "S20130601S0157.fits", ], "twilight": ["S20130602S0005.fits"], }, "N20190602S0306.fits": { "bias": [ "N20190601S0648.fits", "N20190601S0647.fits", "N20190601S0646.fits", "N20190601S0645.fits", "N20190601S0644.fits", ], "twilight": ["N20190602S0306.fits"], } } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/longslit/test_slit_illumination_correct/inputs" path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) cwd = os.getcwd() os.chdir(path) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Download raw files') bias_path = [download_from_archive(f) for f in cals['bias']] twilight_path = [download_from_archive(f) for f in cals['twilight']] twilight_ad = astrodata.open(twilight_path[0]) data_label = twilight_ad.data_label() print('Reducing BIAS for {:s}'.format(data_label)) logutils.config(file_name='log_bias_{}.txt'.format(data_label)) bias_reduce = Reduce() bias_reduce.files.extend(bias_path) bias_reduce.runr() bias_master = bias_reduce.output_filenames.pop() del bias_reduce print('Reducing twilight flat:') logutils.config(file_name='log_twilight_{}.txt'.format(data_label)) with warnings.catch_warnings(): warnings.simplefilter("ignore") p = GMOSLongslit([astrodata.open(f) for f in twilight_path]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.biasCorrect(bias=bias_master) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.stackFrames() # Write non-mosaicked data twilight = p.writeOutputs(suffix="_twilight", strip=True)[0] # Write mosaicked data p = GMOSLongslit([twilight]) p.makeSlitIllum() p.writeOutputs() os.chdir(cwd) return
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from recipe_system.reduction.coreReduce import Reduce from gempy.utils import logutils associated_calibrations = { "S20190808S0048.fits": { "arcs": ["S20190808S0167.fits"] }, "S20190808S0049.fits": { "arcs": ["S20190808S0168.fits"] }, # "S20190808S0052.fits": {"arcs": ["S20190808S0165.fits"]}, # Can't find aperture "S20190808S0053.fits": { "arcs": ["S20190808S0169.fits"] }, } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0]) path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("./inputs", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Downloading files...') sci_path = download_from_archive(filename) arc_path = [download_from_archive(f) for f in cals['arcs']] sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) arc_reduce = Reduce() arc_reduce.files.extend(arc_path) arc_reduce.runr() arc = arc_reduce.output_filenames.pop() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() os.chdir("inputs/") _ = p.writeOutputs().pop() os.chdir("../")
def create_inputs(): """ Create inputs for `test_plot_spectra_for_qa_single_frame`. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import glob import os from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit from gempy.utils import logutils from recipe_system.reduction.coreReduce import Reduce from recipe_system.utils.reduce_utils import normalize_ucals cwd = os.getcwd() path = f"./dragons_test_inputs/geminidr/core/{__file__.split('.')[0]}/" os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("inputs/", exist_ok=True) for raw_list, bias_list, quartz_list, arc_list in single_aperture_data: if all([ os.path.exists(f"inputs/{s.split('.')[0]}_extracted.fits") for s in raw_list ]): print("Skipping already created input.") continue raw_paths = [download_from_archive(f) for f in raw_list] bias_paths = [download_from_archive(f) for f in bias_list] quartz_paths = [download_from_archive(f) for f in quartz_list] arc_paths = [download_from_archive(f) for f in arc_list] cals = [] raw_ads = [astrodata.open(p) for p in raw_paths] data_label = raw_ads[0].data_label() print('Current working directory:\n {:s}'.format(os.getcwd())) if len(bias_paths): logutils.config(file_name='log_bias_{}.txt'.format(data_label)) r = Reduce() r.files.extend(bias_paths) r.runr() master_bias = r.output_filenames.pop() cals.append(f"processed_bias:{master_bias}") del r else: master_bias = None if len(quartz_paths): logutils.config(file_name='log_quartz_{}.txt'.format(data_label)) r = Reduce() r.files.extend(quartz_paths) r.ucals = normalize_ucals(r.files, cals) r.runr() master_quartz = r.output_filenames.pop() cals.append(f"processed_flat:{master_quartz}") del r else: master_quartz = None logutils.config(file_name='log_arc_{}.txt'.format(data_label)) r = Reduce() r.files.extend(arc_paths) r.ucals = normalize_ucals(r.files, cals) r.runr() master_arc = r.output_filenames.pop() logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit(raw_ads) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.biasCorrect(do_bias=master_bias is not None, bias=master_bias) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.flatCorrect(do_flat=master_quartz is not None, flat=master_quartz) p.QECorrect(arc=master_arc) p.distortionCorrect(arc=master_arc) p.findSourceApertures(max_apertures=3) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() p.linearizeSpectra() [os.remove(s) for s in glob.glob("*_arc.fits")] [os.remove(s) for s in glob.glob("*_bias.fits")] [os.remove(s) for s in glob.glob("*_flat.fits")] [os.remove(s) for s in glob.glob("*_mosaic.fits")] os.chdir("inputs/") print("\n\n Writing processed files for tests into:\n" " {:s}\n\n".format(os.getcwd())) _ = p.writeOutputs() os.chdir("../") os.chdir(cwd)
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from recipe_system.reduction.coreReduce import Reduce from gempy.utils import logutils from astrodata.testing import get_associated_calibrations associated_calibrations = { "S20190808S0048.fits": 'S20190808S0167.fits', "S20190808S0049.fits": 'S20190808S0168.fits', "S20190808S0053.fits": 'S20190808S0169.fits', "N20180106S0025.fits": 'N20180115S0264.fits', "N20180106S0026.fits": 'N20180115S0264.fits', "N20180106S0028.fits": 'N20180115S0264.fits', "N20180106S0029.fits": 'N20180115S0264.fits', } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0]) path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("./inputs", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for fname, arc_fname in associated_calibrations.items(): sci_path = download_from_archive(fname) arc_path = download_from_archive(arc_fname) sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) if os.path.exists( arc_fname.replace('.fits', '_distortionDetermined.fits')): arc = astrodata.open( arc_fname.replace('.fits', '_distortionDetermined.fits')) else: p = GMOSLongslit([astrodata.open(arc_path)]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.makeIRAFCompatible() p.determineWavelengthSolution() p.determineDistortion() arc = p.writeOutputs().pop() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() os.chdir("inputs/") _ = p.writeOutputs().pop() os.chdir("../")