def reduce_twighlight(args): D = Distortion(op.join(args.configdir, 'DeformerDefaults', 'mastertrace_twi_027_L.dist')) for spec in args.specid: spec_ind_twi = np.where(args.twi_df['Specid'] == spec)[0] for amp in config.Amps: amp_ind_twi = np.where(args.twi_df['Amp'] == amp)[0] twi_sel = np.intersect1d(spec_ind_twi, amp_ind_twi) for ind in twi_sel: if args.debug: print("Working on Cal for %s, %s" %(spec, amp)) twi1 = Amplifier(args.twi_df['Files'][ind], args.twi_df['Output'][ind], calpath=args.twi_df['Output'][ind], debug=True, dark_mult=0.0, darkpath=args.darkdir, biaspath=args.biasdir, virusconfig=args.configdir, specname=args.specname[amp], use_pixelflat=(args.pixelflats<1), init_lims=args.wvl_dict[amp], check_fibermodel=True, check_wave=True, fsize=args.fsize, fibmodel_nbins=args.fibmodel_bins, sigma=args.fibmodel_sig, power=args.fibmodel_pow, use_trace_ref=args.use_trace_ref, default_fib = args.default_fib, wave_nbins = args.wave_nbins) #twi1.load_fibers() twi1.get_fiber_to_fiber() twi1.sky_subtraction() twi2 = Amplifier(args.twi_df['Files'][ind].replace(amp, config.Amp_dict[amp][0]), args.twi_df['Output'][ind], calpath=args.twi_df['Output'][ind], debug=True, dark_mult=0.0, darkpath=args.darkdir, biaspath=args.biasdir, virusconfig=args.configdir, specname=args.specname[amp], use_pixelflat=(args.pixelflats<1), init_lims=args.wvl_dict[amp], check_fibermodel=True, check_wave=True, fsize=args.fsize, fibmodel_nbins=args.fibmodel_bins, sigma=args.fibmodel_sig, power=args.fibmodel_pow, use_trace_ref=args.use_trace_ref, default_fib = args.default_fib, wave_nbins = args.wave_nbins) #twi2.load_fibers() twi2.get_fiber_to_fiber() twi2.sky_subtraction() image1 = get_model_image(twi1.image, twi1.fibers, 'fiber_to_fiber', debug=twi1.debug) image2 = get_model_image(twi2.image, twi2.fibers, 'fiber_to_fiber', debug=twi2.debug) outname = op.join(args.twi_df['Output'][ind], 'mastertrace_%s_%s.fits' %(args.twi_df['Specid'][ind], config.Amp_dict[amp][1])) make_spectrograph_image(image1, image2, twi1.header, outname) outname = op.join(args.twi_df['Output'][ind], 'mastertwi_%s_%s.fits' %(args.twi_df['Specid'][ind], config.Amp_dict[amp][1])) make_spectrograph_image(twi1.image, twi2.image, twi1.header, outname) outname = op.join(args.twi_df['Output'][ind], 'normtwi_%s_%s.fits' %(args.twi_df['Specid'][ind], amp)) make_amplifier_image(np.where( np.isfinite(twi1.skyframe)*(twi1.skyframe!=0), twi1.image/twi1.skyframe, 0.0), twi1.header, outname) outname = op.join(args.twi_df['Output'][ind], 'normtwi_%s_%s.fits' %(args.twi_df['Specid'][ind], config.Amp_dict[amp][0])) make_amplifier_image(np.where( np.isfinite(twi2.skyframe)*(twi2.skyframe!=0), twi2.image/twi2.skyframe, 0.0), twi2.header, outname) D = recalculate_dist_coeff(D, twi1, twi2) outname2 = op.join(args.twi_df['Output'][ind], 'mastertrace_%s_%s.dist' %(args.twi_df['Specid'][ind], config.Amp_dict[amp][1])) D.writeto(outname2) twi1.save_fibers() twi2.save_fibers() if args.debug: print("Finished working on Cal for %s, %s" %(spec, amp))
def reduce_science(args): for spec in args.specid: spec_ind_sci = np.where(args.sci_df['Specid'] == spec)[0] for amp in config.Amps: amp_ind_sci = np.where(args.sci_df['Amp'] == amp)[0] sci_sel = np.intersect1d(spec_ind_sci, amp_ind_sci) for ind in sci_sel: if args.instr == "virus": if not args.use_trace_ref: ifucen = np.loadtxt(op.join(args.configdir, 'IFUcen_files', args.ifucen_fn[amp][0] + args.sci_df['Ifuid'][ind] + '.txt'), usecols=[0,1,2,4], skiprows=args.ifucen_fn[amp][1]) else: if args.sci_df['Ifuid'][ind] == '004': ifucen = np.loadtxt(op.join(args.configdir, 'IFUcen_files', 'IFUcen_HETDEX_reverse_R.txt'), usecols=[0,1,2,4], skiprows=args.ifucen_fn[amp][1]) ifucen[224:,:] = ifucen[-1:223:-1,:] else: ifucen = np.loadtxt(op.join(args.configdir, 'IFUcen_files', 'IFUcen_HETDEX.txt'), usecols=[0,1,2,4], skiprows=args.ifucen_fn[amp][1]) else: ifucen = np.loadtxt(op.join(args.configdir, 'IFUcen_files', args.ifucen_fn[amp][0]), usecols=[0,1,2], skiprows=args.ifucen_fn[amp][1]) if args.debug: print("Working on Sci for %s, %s" %(spec, amp)) if args.check_if_twi_exists: fn = op.join(args.twi_dir,'fiber_*_%s_%s_%s_%s.pkl' %(spec, args.sci_df['Ifuslot'][ind], args.sci_df['Ifuid'][ind], amp)) calfiles = glob.glob(fn) if not calfiles: print("No cals found for %s,%s: %s" %(spec, amp, args.sci_df['Files'][ind])) print("If you want to produce cals include " "--reduce_twi") sci1 = Amplifier(args.sci_df['Files'][ind], args.sci_df['Output'][ind], calpath=args.twi_dir, skypath=args.sky_dir, debug=False, refit=False, dark_mult=args.dark_mult[amp], darkpath=args.darkdir, biaspath=args.biasdir, virusconfig=args.configdir, specname=args.specname[amp], use_pixelflat=(args.pixelflats<1), use_trace_ref=args.use_trace_ref, calculate_shift=args.adjust_trace, fiber_date=args.fiber_date, cont_smooth=args.cont_smooth) #sci1.load_fibers() #if sci1.fibers and not args.start_from_scratch: # if sci1.fibers[0].spectrum is not None: # sci1.prepare_image() # sci1.sky_subtraction() # sci1.clean_cosmics() #else: sci1.load_all_cal() if args.adjust_trace: sci1.refit=True sci1.get_trace() sci1.refit=False sci1.fiberextract() if args.refit_fiber_to_fiber: sci1.refit=True sci1.get_fiber_to_fiber() sci1.refit=False sci1.sky_subtraction() sci1.clean_cosmics() sci1.fiberextract() sci1.sky_subtraction() sci2 = Amplifier(args.sci_df['Files'][ind].replace(amp, config.Amp_dict[amp][0]), args.sci_df['Output'][ind], calpath=args.twi_dir, skypath=args.sky_dir, debug=False, refit=False, dark_mult=args.dark_mult[config.Amp_dict[amp][0]], darkpath=args.darkdir, biaspath=args.biasdir, virusconfig=args.configdir, specname=args.specname[amp], use_pixelflat=(args.pixelflats<1), use_trace_ref=args.use_trace_ref, calculate_shift=args.adjust_trace, fiber_date=args.fiber_date, cont_smooth=args.cont_smooth) #sci2.load_fibers() #if sci2.fibers and not args.start_from_scratch: # if sci2.fibers[0].spectrum is not None: # sci2.prepare_image() # sci2.sky_subtraction() # sci2.clean_cosmics() #else: sci2.load_all_cal() if args.adjust_trace: sci2.refit=True sci2.get_trace() sci2.refit=False sci2.fiberextract() if args.refit_fiber_to_fiber: sci2.refit=True sci2.get_fiber_to_fiber() sci2.refit=False sci2.sky_subtraction() sci2.clean_cosmics() sci2.fiberextract() sci2.sky_subtraction() outname = op.join(args.sci_df['Output'][ind], 'S%s_%s_sci_%s.fits' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) make_spectrograph_image(sci1.clean_image, sci2.clean_image, sci1.header, outname) make_spectrograph_image(sci1.error, sci2.error, sci1.header, op.join(op.dirname(outname), 'ee.'+op.basename(outname))) make_error_frame(sci1.clean_image, sci2.clean_image, sci1.mask, sci2.mask, sci1.header, outname) outname = op.join(args.sci_df['Output'][ind], 'cS%s_%s_sci_%s.fits' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) make_spectrograph_image(np.where(sci1.mask==0, sci1.clean_image, 0.0), np.where(sci2.mask==0, sci2.clean_image, 0.0), sci1.header, outname) make_error_frame(sci1.clean_image, sci2.clean_image, sci1.mask, sci2.mask, sci1.header, outname) outname = op.join(args.sci_df['Output'][ind], 'CsS%s_%s_sci_%s.fits' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) make_spectrograph_image(sci1.continuum_sub, sci2.continuum_sub, sci1.header, outname) make_error_frame(sci1.continuum_sub, sci2.continuum_sub, sci1.mask, sci2.mask, sci1.header, outname) outname = op.join(args.sci_df['Output'][ind], 'cCsS%s_%s_sci_%s.fits' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) make_spectrograph_image(np.where(sci1.mask==0, sci1.continuum_sub, 0.0), np.where(sci2.mask==0, sci2.continuum_sub, 0.0), sci1.header, outname) make_error_frame(sci1.continuum_sub, sci2.continuum_sub, sci1.mask, sci2.mask, sci1.header, outname) outname = op.join(args.sci_df['Output'][ind], 'cCsS%s_%s_sci_%s_imstat.png' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) imstat(sci1.residual, sci2.residual, sci1.fibers, sci2.fibers, outname) Fe, FeS = recreate_fiberextract(sci1, sci2, wavelim=args.wvl_dict[amp], disp=args.disp[amp]) outname = op.join(args.sci_df['Output'][ind], 'Fe%s_%s_sci_%s.fits' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) make_fiber_image(Fe, sci1.header, outname, args, amp) make_fiber_error(Fe, sci1.header, outname, args, amp) make_cube_file(args, outname, ifucen, args.cube_scale, config.Amp_dict[amp][1]) outname = op.join(args.sci_df['Output'][ind], 'FeS%s_%s_sci_%s.fits' %( op.basename(args.sci_df['Files'][ind]).split('_')[0], args.sci_df['Ifuslot'][ind], config.Amp_dict[amp][1])) make_fiber_image(FeS, sci1.header, outname, args, amp) make_fiber_error(FeS, sci1.header, outname, args, amp) make_cube_file(args, outname, ifucen, args.cube_scale, config.Amp_dict[amp][1]) if args.save_sci_fibers: sci1.save_fibers() sci2.save_fibers() if args.save_sci_amplifier: sci1.save() sci2.save() if args.debug: print("Finished working on Sci for %s, %s" %(spec, amp))