def run(self, rinput):
flow = self.init_filters(rinput, rinput.obresult.configuration)
hdulist = basic_processing_with_combination(rinput, flow, method=combine.median)
hdr = hdulist[0].header
self.set_base_headers(hdr)
result = self.create_result(master_dark=hdulist)
return result
def run(self, rinput):
from scipy.signal import savgol_filter
self.logger.info('starting slit flat reduction')
flow = self.init_filters(rinput, rinput.obresult.configuration)
reduced = basic_processing_with_combination(rinput, flow, method=combine.median)
hdr = reduced[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced, 'reduced_image.fits')
# Using median filtering... In each channel
l0 = reduced[0].data.shape[0]
l1 = l0 // 2
channel1 = (slice(None, l1), slice(None, None, None))
channel2 = (slice(l1, None, None), slice(None, None, None))
m_window1 = (11, 11)
self.logger.debug('median filtering by channel %s', m_window1)
median1 = numpy.zeros_like(reduced[0].data)
median1[channel1] = median_filter(reduced[0].data[channel1], m_window1)
median1[channel2] = median_filter(reduced[0].data[channel2], m_window1)
self.save_intermediate_array(median1, 'median_image1.fits')
qe1 = reduced[0].data / median1
self.save_intermediate_array(qe1, 'qe_filter1.fits')
m_window2 = (3, 21)
self.logger.debug('median filtering %s', m_window2)
median2 = median_filter(qe1, m_window2)
self.save_intermediate_array(median2 , 'median_image2.fits')
qe2 = qe1 / median2
self.save_intermediate_array(qe2, 'qe_filter2.fits')
self.logger.debug('filtering Inf/NaN in result')
qe2[numpy.isinf(qe2)] = 1.0
qe2[numpy.isnan(qe2)] = 1.0
hdu = fits.PrimaryHDU(qe2.astype('float32'), header=reduced[0].header)
hdu.header['UUID'] = str(uuid.uuid1())
master_slitflat = fits.HDUList([hdu])
self.set_base_headers(master_slitflat[0].header)
self.logger.info('end slit flat recipe')
return self.create_result(master_slitflat=master_slitflat,
reduced_image=reduced)
def run(self, rinput):
flow1 = self.init_filters(rinput, rinput.obresult.configuration)
img = basic_processing_with_combination(rinput, flow1, method=combine.median)
hdr = img[0].header
self.set_base_headers(hdr)
splitter1 = Splitter()
calibrator_aper = ApertureExtractor(rinput.tracemap, self.datamodel)
flipcor = FlipLR()
flow2 = SerialFlow([splitter1, calibrator_aper, flipcor])
reduced_rss = flow2(img)
reduced_rss.writeto('arc_rss.fits', clobber=True)
reduced2d = splitter1.out
self.logger.info('extract fibers, %i', len(rinput.tracemap.contents))
current_vph = rinput.obresult.tags['vph']
current_insmode = rinput.obresult.tags['insmode']
if current_insmode in vph_thr_arc and current_vph in vph_thr_arc[current_insmode]:
threshold = vph_thr_arc[current_insmode][current_vph]['threshold']
min_distance = vph_thr_arc[current_insmode][current_vph]['min_distance']
self.logger.info('rel threshold for %s is %4.2f', current_vph, threshold)
else:
threshold = 0.02
min_distance = 10.0
self.logger.info('rel threshold not defined for %s, using %4.2f', current_vph, threshold)
self.logger.info('min dist not defined for %s, using %4.2f', current_vph, min_distance)
if isinstance(rinput.nlines, collections.Iterable):
nlines = rinput.nlines
else:
nlines = [rinput.nlines]
data_wlcalib, fwhm_image = self.calibrate_wl(reduced_rss[0].data,
rinput.lines_catalog,
rinput.polynomial_degree,
rinput.tracemap,
nlines,
threshold=threshold,
min_distance=min_distance)
data_wlcalib.tags = rinput.obresult.tags
# WL calibration goes here
return self.create_result(arc_image=reduced2d, arc_rss=reduced_rss,
master_wlcalib=data_wlcalib,
fwhm_image=fwhm_image)
def run(self, rinput):
self.logger.info('start trace spectra recipe')
obresult = rinput.obresult
current_vph = obresult.tags['vph']
current_insmode = obresult.tags['insmode']
self.logger.info('start basic reduction')
flow = self.init_filters(rinput, obresult.configuration)
reduced = basic_processing_with_combination(rinput, flow, method=combine.median)
self.logger.info('end basic reduction')
self.save_intermediate_img(reduced, 'reduced.fits')
insconf = obresult.configuration
values = insconf.get('pseudoslit.boxes_positions', **obresult.tags)
cstart = values['ref_column']
box_borders = values['positions']
boxes = insconf.get('pseudoslit.boxes', **obresult.tags)
if current_insmode in vph_thr and current_vph in vph_thr[current_insmode]:
threshold = vph_thr[current_insmode][current_vph]
self.logger.info('rel threshold for %s is %4.2f', current_vph, threshold)
else:
threshold = rinput.relative_threshold
self.logger.info('rel threshold not defined for %s, using %4.2f', current_vph, threshold)
final = megaradrp.products.TraceMap(instrument=obresult.instrument)
fiberconf = self.datamodel.get_fiberconf(reduced)
final.total_fibers = fiberconf.nfibers
final.tags = obresult.tags
contents, error_fitting = self.search_traces(
reduced,
boxes,
box_borders,
cstart=cstart,
threshold=threshold,
poldeg=rinput.polynomial_degree
)
final.contents = contents
final.error_fitting = error_fitting
self.logger.info('end trace spectra recipe')
return self.create_result(fiberflat_frame=reduced,
master_traces=final)
def base_run(self, rinput):
# 2D reduction
flow1 = self.init_filters(rinput, rinput.obresult.configuration)
img = basic_processing_with_combination(rinput, flow1, method=combine.median)
hdr = img[0].header
self.set_base_headers(hdr)
# 1D, extraction, Wl calibration, Flat fielding
reduced2d, reduced_rss = self.run_reduction_1d(
img,
rinput.tracemap,
rinput.wlcalib,
rinput.master_fiberflat
)
return reduced2d, reduced_rss
def run(self, rinput):
import copy
N = len(rinput.obresult.frames)
obresult1 = copy.copy(rinput.obresult)
obresult1.frames = rinput.obresult.frames[:N]
flow1 = self.init_filters(rinput, rinput.obresult.configuration)
img = basic_processing_with_combination(rinput, flow1, method=combine.median)
hdr = img[0].header
self.set_base_headers(hdr)
reduced1 = img
fits.writeto(self.directorio + '/reduced_flat.fits', reduced1[0].data,
overwrite=True)
fits.writeto(self.directorio + '/reduced_flat_bpm.fits',
reduced1[0].data, overwrite=True)
return self.create_result()
def base_run(self, rinput):
# 2D reduction
flow1 = self.init_filters(rinput, rinput.obresult.configuration)
img = basic_processing_with_combination(rinput, flow1, method=combine.median)
hdr = img[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(img, 'reduced_image.fits')
reduced2d = copy_img(img)
# 1D, extraction, Wl calibration, Flat fielding
reduced_rss = self.run_reduction_1d(img,
rinput.master_traces, rinput.master_wlcalib,
rinput.master_fiberflat, rinput.master_twilight,
offset=rinput.extraction_offset
)
self.save_intermediate_img(reduced_rss, 'reduced_rss.fits')
return reduced2d, reduced_rss
def run(self, rinput):
self.logger.info("starting slit flat reduction")
flow = self.init_filters(rinput, rinput.obresult.configuration)
reduced = basic_processing_with_combination(rinput, flow, method=combine.median)
hdr = reduced[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced, "reduced.fits")
self.logger.debug("Compute median")
if rinput.median_window_length:
archivo_mediana = median_filter(reduced[0].data, (1, rinput.median_window_length))
else:
archivo_mediana = reduced[0].data
self.logger.debug("Compute Savitzky-Golay X filter (%d, %d)", rinput.window_length_x, rinput.polyorder)
result = savgol_filter(archivo_mediana, rinput.window_length_x, rinput.polyorder, axis=1)
if rinput.window_length_y:
self.logger.debug("Compute Savitzky-Golay Y filter (%d, %d)", rinput.window_length_y, rinput.polyorder)
result = savgol_filter(result, rinput.window_length_y, rinput.polyorder, axis=0)
qe = reduced[0].data / result
self.logger.debug("Filtering INF/NAN in result")
qe[numpy.isinf(qe)] = 1.0
qe[numpy.isnan(qe)] = 1.0
hdu = fits.PrimaryHDU(qe, header=reduced[0].header)
hdu.header["UUID"] = uuid.uuid1().hex
hdu.header["OBJECT"] = "MASTER SLITFLAT"
# hdu.header['IMAGETYP'] = 'SLIT_FLAT'
master_slitflat = fits.HDUList([hdu])
self.logger.info("end slit flat recipe")
return self.create_result(master_slitflat=master_slitflat)
def run(self, rinput):
"""Execute the recipe.
Parameters
----------
rinput : BiasRecipe.RecipeInput
Returns
-------
BiasRecipe.RecipeResult
"""
self.logger.info('start bias recipe')
flow = self.init_filters(rinput, rinput.obresult.configuration)
errors = False
if not errors:
self.logger.info('not computing errors')
hdulist = basic_processing_with_combination(rinput, flow, method=combine.median, errors=errors)
hdr = hdulist[0].header
self.set_base_headers(hdr)
result = self.create_result(master_bias=hdulist)
self.logger.info('end bias recipe')
return result
def run(self, rinput):
"""Execute the recipe.
Parameters
----------
rinput : TraceMapRecipe.RecipeInput
Returns
-------
TraceMapRecipe.RecipeResult
Raises
------
ValidationError
if the recipe input is invalid
"""
self.logger.info('start trace spectra recipe')
obresult = rinput.obresult
current_vph = obresult.tags['vph']
current_insmode = obresult.tags['insmode']
obresult_meta = obresult.metadata_with(self.datamodel)
debug_plot = rinput.debug_plot if self.intermediate_results else 0
self.logger.info('start basic reduction')
flow = self.init_filters(rinput, obresult.configuration)
reduced = basic_processing_with_combination(rinput, flow, method=combine.median)
self.logger.info('end basic reduction')
self.save_intermediate_img(reduced, 'reduced_image.fits')
#insconf = obresult.configuration
insconf = obresult.profile
boxes = insconf.get_property('pseudoslit.boxes')
values = insconf.get_property('pseudoslit.boxes_positions')
cstart0 = values['ref_column']
box_borders0 = values['positions']
box_borders, cstart = self.refine_boxes_from_image(reduced, box_borders0, cstart0)
self.logger.debug("original boxes: %s", box_borders0)
self.logger.debug("refined boxes: %s", box_borders)
if current_insmode in vph_thr and current_vph in vph_thr[current_insmode]:
threshold = vph_thr[current_insmode][current_vph]
self.logger.info('rel threshold for %s is %4.2f', current_vph, threshold)
else:
threshold = rinput.relative_threshold
self.logger.info('rel threshold not defined for %s, using %4.2f', current_vph, threshold)
final = megaradrp.products.TraceMap(instrument=obresult.instrument)
fiberconf = self.datamodel.get_fiberconf(reduced)
final.total_fibers = fiberconf.nfibers
final.tags = obresult.tags
final.boxes_positions = box_borders
final.ref_column = cstart
final.update_metadata(self)
final.update_metadata_origin(obresult_meta)
# Temperature in Celsius with 2 decimals
final.tags['temp'] = round(obresult_meta['info'][0]['temp'] - 273.15, 2)
contents, error_fitting = self.search_traces(
reduced,
boxes,
box_borders,
cstart=cstart,
threshold=threshold,
poldeg=rinput.polynomial_degree,
debug_plot=debug_plot
)
final.contents = contents
final.error_fitting = error_fitting
# Perform extraction with own traces
calibrator_aper = ApertureExtractor(final, self.datamodel)
reduced_copy = copy_img(reduced)
reduced_rss = calibrator_aper(reduced_copy)
if self.intermediate_results:
with open('ds9.reg', 'w') as ds9reg:
final.to_ds9_reg(ds9reg, rawimage=False,
numpix=100, fibid_at=2048)
with open('ds9_raw.reg', 'w') as ds9reg:
final.to_ds9_reg(ds9reg, rawimage=True,
numpix=100, fibid_at=2048)
self.logger.info('end trace spectra recipe')
return self.create_result(reduced_image=reduced,
reduced_rss = reduced_rss,
master_traces=final)
def run(self, rinput):
"""Execute the recipe.
Parameters
----------
rinput : ArcCalibrationRecipe.RecipeInput
Returns
-------
ArcCalibrationRecipe.RecipeResult
"""
self.logger.info('starting arc calibration recipe')
debugplot = rinput.debug_plot if self.intermediate_results else 0
obresult = rinput.obresult
obresult_meta = obresult.metadata_with(self.datamodel)
flow1 = self.init_filters(rinput, rinput.obresult.configuration)
img = basic_processing_with_combination(rinput, flow1, method=combine.median)
hdr = img[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(img, 'reduced_image.fits')
splitter1 = Splitter()
calibrator_aper = ApertureExtractor(
rinput.master_apertures,
self.datamodel,
offset=rinput.extraction_offset
)
flipcor = FlipLR()
flow2 = SerialFlow([splitter1, calibrator_aper, flipcor])
reduced_rss = flow2(img)
self.save_intermediate_img(reduced_rss, 'reduced_rss.fits')
reduced2d = splitter1.out
self.logger.info('extract fibers, %i', len(rinput.master_apertures.contents))
current_vph = rinput.obresult.tags['vph']
current_insmode = rinput.obresult.tags['insmode']
if current_insmode in vph_thr_arc and current_vph in vph_thr_arc[current_insmode]:
threshold = vph_thr_arc[current_insmode][current_vph]['threshold']
min_distance = vph_thr_arc[current_insmode][current_vph]['min_distance']
self.logger.info('rel threshold for %s is %4.2f', current_vph, threshold)
else:
threshold = 0.02
min_distance = 10.0
self.logger.info('rel threshold not defined for %s, using %4.2f', current_vph, threshold)
self.logger.info('min dist not defined for %s, using %4.2f', current_vph, min_distance)
# WL calibration goes here
initial_data_wlcalib, data_wlcalib, fwhm_image = self.calibrate_wl(
reduced_rss[0].data,
rinput.lines_catalog,
rinput.polynomial_degree,
rinput.master_apertures, rinput.nlines,
threshold=threshold,
min_distance=min_distance,
debugplot=debugplot,
store_pdf_with_refined_fits=rinput.store_pdf_with_refined_fits
)
initial_data_wlcalib.tags = rinput.obresult.tags
final = initial_data_wlcalib
final.update_metadata(self)
final.update_metadata_origin(obresult_meta)
self.save_structured_as_json(
initial_data_wlcalib,
'initial_master_wlcalib.json'
)
self.logger.info('end arc calibration recipe')
if data_wlcalib is None:
return self.create_result(
reduced_image=reduced2d,
reduced_rss=reduced_rss,
master_wlcalib=initial_data_wlcalib,
fwhm_image=fwhm_image
)
else:
# copy metadata from initial_master_wlcalib to master_wlcalib
data_wlcalib.tags = deepcopy(initial_data_wlcalib.tags)
data_wlcalib.meta_info = deepcopy(initial_data_wlcalib.meta_info)
return self.create_result(
reduced_image=reduced2d,
reduced_rss=reduced_rss,
master_wlcalib=data_wlcalib,
fwhm_image=fwhm_image
)
def run(self, rinput):
self.logger.info('starting model map recipe')
obresult = rinput.obresult
obresult_meta = obresult.metadata_with(self.datamodel)
if rinput.processes == 0:
have = mp.cpu_count()
if have >= 4:
processes = mp.cpu_count() - 2
else:
processes = 1
else:
processes = rinput.processes
self.logger.debug('using %d processes', processes)
self.logger.info('start basic reduction')
flow1 = self.init_filters(rinput, rinput.obresult.configuration)
reduced = basic_processing_with_combination(rinput, flow1, method=combine.median)
self.set_base_headers(reduced[0].header)
self.logger.info('end basic reduction')
self.save_intermediate_img(reduced, 'reduced_image.fits')
self.logger.debug('create result model')
# insconf = obresult.configuration
# boxes = insconf.get_property('pseudoslit.boxes')
# values = insconf.get_property('pseudoslit.boxes_positions')
# cstart0 = values['ref_column']
# box_borders0 = values['positions']
# box_borders, cstart = self.refine_boxes_from_image(reduced, box_borders0, cstart0)
# self.logger.debug("original boxes: %s", box_borders0)
# self.logger.debug("refined boxes: %s", box_borders)
model_map = ModelMap(instrument=obresult.instrument)
self.logger.debug('update metadata in model')
model_map.update_metadata(self)
fiberconf = self.datamodel.get_fiberconf(reduced)
model_map.total_fibers = fiberconf.nfibers
model_map.missing_fibers = rinput.master_traces.missing_fibers
model_map.tags = obresult.tags
# model_map.boxes_positions = box_borders
# model_map.ref_column = cstart
model_map.update_metadata(self)
model_map.update_metadata_origin(obresult_meta)
# Temperature in Celsius with 2 decimals
model_map.tags['temp'] = round(obresult_meta['info'][0]['temp'] - 273.15, 2)
self.logger.info('perform model fitting')
tracemap = rinput.master_traces
data = reduced[0].data
sigma = 1.53
cols = range(100, 4100, 100)
# cols = range(100, 200, 100)
valid = [f.fibid for f in tracemap.contents if f.valid]
ncol = tracemap.total_fibers
nrow = data.shape[0]
nfit = data.shape[1]
results_get = fit_model(data, tracemap, valid, nrow, ncol, sigma, cols, processes)
self.logger.info('perform model fitting end')
self.logger.info('interpolate parameters')
array_mean = np.zeros((ncol, nfit))
array_std = np.zeros((ncol, nfit))
xcol = np.arange(nfit)
mean_splines = {}
std_splines = {}
for fibid in valid:
g_amp = []
g_std = []
g_mean = []
g_col = []
dolog = (fibid % 100 == 0)
if dolog:
self.logger.debug('compute fibid %d', fibid)
for calc_col, vals in results_get:
param = vals[fibid]
g_col.append(calc_col)
g_std.append(param['stddev'])
g_mean.append(param['mean'])
g_amp.append(param['amplitude'])
interpol_std = UnivariateSpline(g_col, g_std, k=5)
interpol_mean = UnivariateSpline(g_col, g_mean, k=3)
if self.intermediate_results:
if dolog:
self.logger.debug('saving plots')
plt.title('std fib{:03d}'.format(fibid))
plt.plot(g_col, g_std, 'b*')
plt.plot(g_col, interpol_std(g_col), 'r')
plt.savefig('fib_{:03d}_std.png'.format(fibid))
plt.close()
plt.title('mean fin{:03d}'.format(fibid))
plt.plot(g_col, g_mean, 'b*')
plt.plot(g_col, interpol_mean(g_col), 'r')
plt.savefig('fib_{:03d}_mean.png'.format(fibid))
plt.close()
if dolog:
self.logger.debug('saving plots end')
mean_splines[fibid] = interpol_mean
std_splines[fibid] = interpol_std
row = fibid - 1
array_mean[row] = interpol_mean(xcol)
array_std[row] = interpol_std(xcol)
params = {'stddev': interpol_std, 'mean': interpol_mean}
model = {'model_name': 'gaussbox', 'params': params}
# if invalid. missing, model = {}
m = GeometricModel(
fibid,
boxid=1, # FIXME: not counting this
start=1,
stop=nfit,
model=model
)
model_map.contents.append(m)
self.logger.info('interpolate parameters end')
# perform extraction with our own calibration
self.logger.info('perform extraction with computed calibration')
calibrator_aper = ApertureExtractor(model_map, self.datamodel)
reduced_copy = copy_img(reduced)
reduced_rss = calibrator_aper(reduced_copy)
if self.intermediate_results:
with open('ds9.reg', 'w') as ds9reg:
model_map.to_ds9_reg(ds9reg, rawimage=False,
numpix=100, fibid_at=2048)
with open('ds9_raw.reg', 'w') as ds9reg:
model_map.to_ds9_reg(ds9reg, rawimage=True,
numpix=100, fibid_at=2048)
self.logger.info('ending model map recipe')
result = self.create_result(reduced_image=reduced,
reduced_rss=reduced_rss,
master_model=model_map)
return result
def process_flat2d(self, rinput):
flow = self.init_filters(rinput, rinput.obresult.configuration)
final_image = basic_processing_with_combination(rinput, flow, method=combine.median)
hdr = final_image[0].header
self.set_base_headers(hdr)
return final_image
