def test_bpm_corrector():
import shutil
from tempfile import mkdtemp
config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c'
date_obs = '2017-11-09T11:00:00.0'
directorio = mkdtemp()
names = crear_archivos(directorio, number=4)
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'MegaraBiasImage'
pkg_paths = ['megaradrp.instrument.configs']
store = asb.load_paths_store(pkg_paths)
insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid')
header = fits.Header()
header['DATE-OBS'] = date_obs
header['INSCONF'] = config_uuid
header['INSTRUME'] = 'MEGARA'
header['VPH'] = 'LR-U'
header['INSMODE'] = 'MOS'
insmodel.configure_with_header(header)
ob.configuration = insmodel
ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names]
recipe = DerivedRecipe(directorio)
ri = recipe.create_input(obresult=ob, master_bias=DataFrame(
filename=open(directorio + '/master_bias_data0.fits').name),
master_bpm=DataFrame(filename=open(
directorio + '/master_bpm.fits').name))
recipe.run(ri)
shutil.rmtree(directorio)
def generate_bias(detector, number, temporary_path):
from megaradrp.simulation.actions import simulate_bias
from megaradrp.recipes.calibration.bias import BiasRecipe
config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c'
date_obs = '2017-11-09T11:00:00.0'
fs = [simulate_bias(detector) for i in range(number)]
header = fits.Header()
header['DATE-OBS'] = date_obs
header['INSCONF'] = config_uuid
header['INSTRUME'] = 'MEGARA'
header['VPH'] = 'LR-U'
header['INSMODE'] = 'MOS'
for aux in range(len(fs)):
fits.writeto('%s/bias_%s.fits' % (temporary_path, aux), fs[aux],
header=header,
overwrite=True)
fs = ["%s/bias_%s.fits" % (temporary_path, i) for i in range(number)]
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'bias_image'
pkg_paths = ['megaradrp.instrument.configs']
store = asb.load_paths_store(pkg_paths)
insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid')
insmodel.configure_with_header(header)
ob.configuration = insmodel
ob.frames = [DataFrame(filename=f) for f in fs]
recipe = BiasRecipe()
ri = recipe.create_input(obresult=ob)
return recipe.run(ri)
def crear_archivos(temporary_path, number=5):
from megaradrp.simulation.actions import simulate_flat
from megaradrp.simulation.detector import ReadParams, MegaraDetectorSat
from megaradrp.recipes.calibration.bpm import BadPixelsMaskRecipe
PSCAN = 50
DSHAPE = (2056 * 2, 2048 * 2)
OSCAN = 50
ron = 2.0
gain = 1.0
bias = 1000.0
qe = 0.8 * np.ones(DSHAPE)
qe[0:15, 0:170] = 0.0
readpars1 = ReadParams(gain=gain, ron=ron, bias=bias)
readpars2 = ReadParams(gain=gain, ron=ron, bias=bias)
detector = MegaraDetectorSat('megara_test_detector', DSHAPE, OSCAN, PSCAN,
qe=qe,
dark=(3.0 / 3600.0),
readpars1=readpars1, readpars2=readpars2,
bins='11')
source2 = 1.0
fs = [simulate_flat(detector, exposure=1.0, source=5000 * source2) for i in
range(number)]
for aux in range(len(fs)):
fits.writeto('%s/flat_%s.fits' % (temporary_path, aux), fs[aux],
clobber=True)
master_bias = generate_bias(detector, number, temporary_path)
master_bias_data = master_bias.master_bias.frame[0].data
fits.writeto('%s/master_bias_data0.fits' % temporary_path,
master_bias_data, clobber=True) # Master Bias
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'bias_image'
ob.configuration = build_instrument_config('4fd05b24-2ed9-457b-b563-a3c618bb1d4c', loader=Loader())
names = []
for aux in range(number):
names.append('%s/flat_%s.fits' % (temporary_path, aux))
ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names]
recipe = BadPixelsMaskRecipe()
ri = recipe.create_input(obresult=ob, master_bias=DataFrame(
filename=open(temporary_path + '/master_bias_data0.fits').name))
aux = recipe.run(ri)
fits.writeto('%s/master_bpm.fits' % temporary_path,
aux.master_bpm.frame[0].data, clobber=True)
return names
def test_bias(drpmocker):
BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/BIAS/%s'
images = ['e99d2937d2c29a27c0ba4eebfcf7918e',
'e99d2937d2c29a27c0ba4eebfcf7918e',
'e99d2937d2c29a27c0ba4eebfcf7918e']
fs = [download_cache(BASE_URL % i) for i in images]
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'bias_image'
ob.frames = [DataFrame(filename=f.name) for f in fs]
drpmocker.add_drp('MEGARA', load_drp)
# Here we could directly import the required pipeline,
# but the idea is to test all the process
insdrp = numina.drps.get_system_drps().query_by_name(ob.instrument)
pipeline = insdrp.pipelines.get('default')
recipe_fqn = pipeline.recipes.get(ob.mode)
RecipeClass = import_object(recipe_fqn)
assert RecipeClass is BiasRecipe
# TODO: these should be created by a build_recipe_input method
recipe = BiasRecipe()
ri = recipe.create_input(obresult=ob)
result = recipe.run(ri)
# assert result.qc >= QC.UNKNOWN
# Checks on the image
hdulist = result.biasframe.open()
assert len(hdulist) == 1
hdu = hdulist[0]
assert hdu.shape == (4112, 4096)
data = hdu.data
mlevel = 0.0
block = data[1980:2020, 1980:2020]
mblock = block.mean()
sblock = block.std()
assert abs(mblock - mlevel) < 5 * sblock
# In the end, remove the files
for f in fs:
os.remove(f.name)
def test_recipe2():
drps = init_drp_system()
BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/BIAS/%s'
images = ['e99d2937d2c29a27c0ba4eebfcf7918e',
'e99d2937d2c29a27c0ba4eebfcf7918e',
'e99d2937d2c29a27c0ba4eebfcf7918e']
fs = [download_cache(BASE_URL % i) for i in images]
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'bias_image'
ob.frames = [DataFrame(filename=f.name) for f in fs]
instrument = drps.get(ob.instrument)
pipeline = instrument.pipelines.get('default')
recipe_fqn = pipeline.recipes.get(ob.mode)
RecipeClass = import_object(recipe_fqn)
assert RecipeClass is BiasRecipe
# TODO: these should be created by a RecipeInputBuilder
recipe = BiasRecipe()
RR = BiasRecipe.RecipeRequirements # @UndefinedVariable
ri = RR(obresult=ob)
result = recipe.run(ri)
# assert result.qc >= QC.UNKNOWN
# Checks on the image
hdulist = result.biasframe.open()
assert len(hdulist) == 1
hdu = hdulist[0]
assert hdu.shape == (4112, 4096)
data = hdu.data
mlevel = 0.0
block = data[1980:2020, 1980:2020]
mblock = block.mean()
sblock = block.std()
assert abs(mblock - mlevel) < 5 * sblock
# In the end, remove the files
for f in fs:
os.remove(f.name)
def build_recipe_input_gtc(cls, obsres, dal, pipeline='default'):
cls.logger.debug('start recipe input builder')
stareImagesIds = obsres.stareImagesIds
cls.logger.debug('LCB image IDS %s: ', stareImagesIds)
stareImages = []
for subresId in stareImagesIds:
subres = dal.getRecipeResult(subresId)
stareImages.append(subres['elements']['final_rss'])
newOR = ObservationResult()
newOR.frames = stareImages
newRI = cls.create_input(obresult=newOR)
cls.logger.debug('end recipe input builder')
return newRI
def buildRecipeInput(self, obsres):
if self.sky_image is None:
print('obtaining SKY image')
sky_cal_result = self.dal.getLastRecipeResult("EMIR", "EMIR", "IMAGE_SKY")
self.sky_image = sky_cal_result['elements']['skyframe']
obsres['master_sky'] = self.sky_image
newOR = ObservationResult()
newOR.frames = obsres['frames']
obsres['obresult'] = newOR
newRI = StareImageRecipeInput(**obsres)
return newRI
def buildRecipeInput(self, obsres):
if self.sky_image is None:
print('obtaining SKY image')
sky_cal_result = self.dal.getLastRecipeResult(
"EMIR", "EMIR", "IMAGE_SKY")
self.sky_image = sky_cal_result['elements']['skyframe']
obsres['master_sky'] = self.sky_image
newOR = ObservationResult()
newOR.frames = obsres['frames']
obsres['obresult'] = newOR
newRI = StareImageRecipeInput(**obsres)
return newRI
def test_bias():
BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/BIAS/%s'
images = ['e99d2937d2c29a27c0ba4eebfcf7918e',
'e99d2937d2c29a27c0ba4eebfcf7918e',
'e99d2937d2c29a27c0ba4eebfcf7918e']
fs = [download_cache(BASE_URL % i) for i in images]
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'MegaraBiasImage'
ob.frames = [DataFrame(filename=f.name) for f in fs]
insdrp = load_drp()
pipeline = insdrp.pipelines.get('default')
recipe = pipeline.get_recipe_object(ob.mode)
assert isinstance(recipe, BiasRecipe)
# TODO: these should be created by a build_recipe_input method
ob.configuration = insdrp.configuration_selector(ob)
ri = recipe.create_input(obresult=ob)
result = recipe.run(ri)
# assert result.qc >= QC.UNKNOWN
# Checks on the image
hdulist = result.master_bias.open()
assert len(hdulist) == 1
hdu = hdulist[0]
assert hdu.shape == (4112, 4096)
data = hdu.data
mlevel = 0.0
block = data[1980:2020, 1980:2020]
mblock = block.mean()
sblock = block.std()
assert abs(mblock - mlevel) < 5 * sblock
# In the end, remove the files
for f in fs:
os.remove(f.name)
def run_single(self, rinput):
self.logger.info('starting spectroscopy ABBA reduction')
flow = self.init_filters(rinput)
nimages = len(rinput.obresult.frames)
self.logger.info('we receive %d images', nimages)
if nimages != 4:
msg = 'Recipe expects 4 images, received %d' % nimages
raise numina.exceptions.RecipeError(msg)
procesed_hdulists = basic_processing(rinput, flow)
# INPUTS are ABBA, so
#
hdulist = self.process_abba(procesed_hdulists)
grism = hdulist[0].header.get('GRISM', 'unknown')
if grism.lower() == 'open':
# perform TEST10 in addition
import emirdrp.recipes.acquisition.maskcheck as mk
from numina.core import ObservationResult, DataFrame
import numpy
try:
import StringIO as S
except ImportError:
import io as S
self.logger.info('GRISM is OPEN, doing a RECIPE10')
sub = mk.MaskCheckRecipe()
sub.configure(instrument='EMIR', mode='TEST10')
o = ObservationResult()
o.__dict__ = rinput.obresult.__dict__
o.frames = [DataFrame(frame=hdulist)]
subd = {}
subd['obresult'] = o
ss = S.StringIO(_NOMINALPOS)
subd['bars_nominal_positions'] = numpy.loadtxt(ss)
subinput = mk.MaskCheckRecipe.RecipeInput(**subd)
sub.run(subinput)
result = self.create_result(reduced_mos_abba=hdulist)
self.logger.info('end spectroscopy ABBA reduction')
return result
def generate_bias(detector, number, temporary_path):
from megaradrp.simulation.actions import simulate_bias
from megaradrp.recipes.calibration.bias import BiasRecipe
fs = [simulate_bias(detector) for i in range(number)]
for aux in range(len(fs)):
fits.writeto('%s/bias_%s.fits' % (temporary_path, aux), fs[aux],
clobber=True)
fs = ["%s/bias_%s.fits" % (temporary_path, i) for i in range(number)]
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'bias_image'
ob.configuration = build_instrument_config('4fd05b24-2ed9-457b-b563-a3c618bb1d4c', loader=Loader())
ob.frames = [DataFrame(filename=f) for f in fs]
recipe = BiasRecipe()
ri = recipe.create_input(obresult=ob)
return recipe.run(ri)
def test_bpm_corrector():
import shutil
from tempfile import mkdtemp
directorio = mkdtemp()
names = crear_archivos(directorio, number=4)
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'bias_image'
ob.configuration = build_instrument_config('4fd05b24-2ed9-457b-b563-a3c618bb1d4c', loader=Loader())
ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names]
recipe = TestRecipe(directorio)
ri = recipe.create_input(obresult=ob, master_bias=DataFrame(
filename=open(directorio + '/master_bias_data0.fits').name),
master_bpm=DataFrame(filename=open(
directorio + '/master_bpm.fits').name))
recipe.run(ri)
shutil.rmtree(directorio)
def build_recipe_input_gtc(self, obsres, dal, pipeline='default'):
newOR = ObservationResult()
# FIXME: this method will work only in GTC
# stareImagesIds = obsres['stareImagesIds']._v
stareImagesIds = obsres.stareImagesIds
obsres.children = stareImagesIds
self.logger.info('Submode result IDs: %s', obsres.children)
stareImages = []
# Field to query the results
key_field = 'frame'
for subresId in obsres.children:
subres = dal.getRecipeResult(subresId)
# This 'frame' is the name of the product in RecipeResult
# there is also a 'sky' field
elements = subres['elements']
stareImages.append(elements[key_field])
newOR.frames = stareImages
naccum = obsres.naccum
self.logger.info('naccum: %d', naccum)
mode_field = "DITHERED_IMAGE"
key_field = 'accum'
if naccum != 1: # if it is not the first dithering loop
self.logger.info("SEARCHING LATEST RESULT of %s", mode_field)
latest_result = dal.getLastRecipeResult("EMIR", "EMIR", mode_field)
elements = latest_result['elements']
accum_dither = elements[key_field]
self.logger.info("FOUND")
else:
self.logger.info("NO ACCUMULATION")
accum_dither = stareImages[0]
newOR.naccum = naccum
newOR.accum = accum_dither
# obsres['obresult'] = newOR
# print('Adding RI parameters ', obsres)
# newRI = DitheredImageARecipeInput(**obsres)
newRI = self.create_input(obresult=newOR)
return newRI
def test_bpm():
number = 5
PSCAN = 50
DSHAPE = (2056 * 2, 2048 * 2)
OSCAN = 50
ron = 2.0
gain = 1.0
bias = 1000.0
qe = 0.8 * np.ones(DSHAPE)
qe[5:6, 0:170] = 0.0
config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c'
date_obs = '2017-11-09T11:00:00.0'
temporary_path = mkdtemp()
fits.writeto('{}/eq.fits'.format(temporary_path), qe, overwrite=True)
readpars1 = ReadParams(gain=gain, ron=ron, bias=bias)
readpars2 = ReadParams(gain=gain, ron=ron, bias=bias)
detector = MegaraDetectorSat('megara_test_detector', DSHAPE, OSCAN, PSCAN,
qe=qe,
dark=(3.0 / 3600.0),
readpars1=readpars1, readpars2=readpars2,
bins='11')
source2 = 1.0
fs = [simulate_flat(detector, exposure=1.0, source=5000 * source2) for i in
range(number)]
fs2 = [simulate_flat(detector, exposure=1.0, source=40000 * source2) for i
in range(number)]
header = fits.Header()
header['DATE-OBS'] = date_obs
header['INSCONF'] = config_uuid
header['INSTRUME'] = 'MEGARA'
header['VPH'] = 'LR-U'
header['INSMODE'] = 'MOS'
for aux in range(len(fs)):
fits.writeto('{}/flat_{}.fits'.format(temporary_path, aux), fs[aux],
header=header,
overwrite=True)
fits.writeto('{}/flat_{}.fits'.format(temporary_path, aux + number), fs2[aux],
header=header,
overwrite=True)
result = generate_bias(detector, number, temporary_path)
result.master_bias.frame.writeto(
'{}/master_bias_data0.fits'.format(temporary_path),
overwrite=True
)
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'MegaraBiasImage'
pkg_paths = ['megaradrp.instrument.configs']
store = asb.load_paths_store(pkg_paths)
insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid')
insmodel.configure_with_header(header)
ob.configuration = insmodel
names = []
for aux in range(number * 2):
names.append('{}/flat_{}.fits'.format(temporary_path, aux))
ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names]
recipe = BadPixelsMaskRecipe()
ri = recipe.create_input(obresult=ob, master_bias=DataFrame(
filename=open(temporary_path + '/master_bias_data0.fits').name))
aux = recipe.run(ri)
aux.master_bpm.frame.writeto('{}/master_bpm.fits'.format(temporary_path), overwrite=True)
shutil.rmtree(temporary_path)
def test_dark():
numpy.random.seed(422992983)
PSCAN = 50
DSHAPE = (2056 * 2, 2048 * 2)
OSCAN = 50
# ron = 2.0
ron = 0.001
gain = 1.0
bias = 1000.0
dark = 3.0 # In 1 hour
exptime = 3600.0
dark_s = dark / exptime
qe = 0.8 * numpy.ones(DSHAPE)
config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c'
date_obs = '2017-11-09T11:00:00.0'
temporary_path = mkdtemp()
fits.writeto('{}/eq.fits'.format(temporary_path), qe, overwrite=True)
readpars1 = ReadParams(gain=gain, ron=ron, bias=bias)
readpars2 = ReadParams(gain=gain, ron=ron, bias=bias)
detector = MegaraDetectorSat('megara_test_detector', DSHAPE, OSCAN, PSCAN, qe=qe,
dark=dark_s,
readpars1=readpars1, readpars2=readpars2,
bins='11')
number = 3
factory = MegaraImageFactory()
fs = simulate_dark_fits(factory, detector, exposure=3600, repeat=number)
for idx, aux in enumerate(fs):
aux.writeto('{}/dark_{}.fits'.format(temporary_path, idx), overwrite=True)
header = fits.Header()
header['DATE-OBS'] = date_obs
header['INSCONF'] = config_uuid
header['INSTRUME'] = 'MEGARA'
header['VPH'] = 'LR-U'
header['INSMODE'] = 'MOS'
master_bias_data = numpy.zeros(DSHAPE)
master_bias_hdul = fits.HDUList(fits.PrimaryHDU(
master_bias_data, header=header)
)
#master_bias_data = master_bias.master_bias.frame[0].data
ob = ObservationResult()
ob.instrument = 'MEGARA'
ob.mode = 'MegaraDarkImage'
pkg_paths = ['megaradrp.instrument.configs']
store = asb.load_paths_store(pkg_paths)
insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid')
insmodel.configure_with_header(header)
ob.configuration = insmodel
names = []
for aux in range(number):
names.append('{}/dark_{}.fits'.format(temporary_path, aux))
ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names]
ob.configuration = insmodel
recipe = DarkRecipe()
ri = recipe.create_input(
obresult=ob,
master_bias=DataFrame(frame=master_bias_hdul),
)
aux = recipe.run(ri)
mean_dark_value = aux.master_dark.frame[0].data.mean()
shutil.rmtree(temporary_path)
assert numpy.allclose(mean_dark_value, dark, atol=0, rtol=1e-1)
