def test_bpm_corrector(): import shutil from tempfile import mkdtemp config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c' date_obs = '2017-11-09T11:00:00.0' directorio = mkdtemp() names = crear_archivos(directorio, number=4) ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'MegaraBiasImage' pkg_paths = ['megaradrp.instrument.configs'] store = asb.load_paths_store(pkg_paths) insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid') header = fits.Header() header['DATE-OBS'] = date_obs header['INSCONF'] = config_uuid header['INSTRUME'] = 'MEGARA' header['VPH'] = 'LR-U' header['INSMODE'] = 'MOS' insmodel.configure_with_header(header) ob.configuration = insmodel ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names] recipe = DerivedRecipe(directorio) ri = recipe.create_input(obresult=ob, master_bias=DataFrame( filename=open(directorio + '/master_bias_data0.fits').name), master_bpm=DataFrame(filename=open( directorio + '/master_bpm.fits').name)) recipe.run(ri) shutil.rmtree(directorio)
def generate_bias(detector, number, temporary_path): from megaradrp.simulation.actions import simulate_bias from megaradrp.recipes.calibration.bias import BiasRecipe config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c' date_obs = '2017-11-09T11:00:00.0' fs = [simulate_bias(detector) for i in range(number)] header = fits.Header() header['DATE-OBS'] = date_obs header['INSCONF'] = config_uuid header['INSTRUME'] = 'MEGARA' header['VPH'] = 'LR-U' header['INSMODE'] = 'MOS' for aux in range(len(fs)): fits.writeto('%s/bias_%s.fits' % (temporary_path, aux), fs[aux], header=header, overwrite=True) fs = ["%s/bias_%s.fits" % (temporary_path, i) for i in range(number)] ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'bias_image' pkg_paths = ['megaradrp.instrument.configs'] store = asb.load_paths_store(pkg_paths) insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid') insmodel.configure_with_header(header) ob.configuration = insmodel ob.frames = [DataFrame(filename=f) for f in fs] recipe = BiasRecipe() ri = recipe.create_input(obresult=ob) return recipe.run(ri)
def crear_archivos(temporary_path, number=5): from megaradrp.simulation.actions import simulate_flat from megaradrp.simulation.detector import ReadParams, MegaraDetectorSat from megaradrp.recipes.calibration.bpm import BadPixelsMaskRecipe PSCAN = 50 DSHAPE = (2056 * 2, 2048 * 2) OSCAN = 50 ron = 2.0 gain = 1.0 bias = 1000.0 qe = 0.8 * np.ones(DSHAPE) qe[0:15, 0:170] = 0.0 readpars1 = ReadParams(gain=gain, ron=ron, bias=bias) readpars2 = ReadParams(gain=gain, ron=ron, bias=bias) detector = MegaraDetectorSat('megara_test_detector', DSHAPE, OSCAN, PSCAN, qe=qe, dark=(3.0 / 3600.0), readpars1=readpars1, readpars2=readpars2, bins='11') source2 = 1.0 fs = [simulate_flat(detector, exposure=1.0, source=5000 * source2) for i in range(number)] for aux in range(len(fs)): fits.writeto('%s/flat_%s.fits' % (temporary_path, aux), fs[aux], clobber=True) master_bias = generate_bias(detector, number, temporary_path) master_bias_data = master_bias.master_bias.frame[0].data fits.writeto('%s/master_bias_data0.fits' % temporary_path, master_bias_data, clobber=True) # Master Bias ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'bias_image' ob.configuration = build_instrument_config('4fd05b24-2ed9-457b-b563-a3c618bb1d4c', loader=Loader()) names = [] for aux in range(number): names.append('%s/flat_%s.fits' % (temporary_path, aux)) ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names] recipe = BadPixelsMaskRecipe() ri = recipe.create_input(obresult=ob, master_bias=DataFrame( filename=open(temporary_path + '/master_bias_data0.fits').name)) aux = recipe.run(ri) fits.writeto('%s/master_bpm.fits' % temporary_path, aux.master_bpm.frame[0].data, clobber=True) return names
def test_bias(drpmocker): BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/BIAS/%s' images = ['e99d2937d2c29a27c0ba4eebfcf7918e', 'e99d2937d2c29a27c0ba4eebfcf7918e', 'e99d2937d2c29a27c0ba4eebfcf7918e'] fs = [download_cache(BASE_URL % i) for i in images] ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'bias_image' ob.frames = [DataFrame(filename=f.name) for f in fs] drpmocker.add_drp('MEGARA', load_drp) # Here we could directly import the required pipeline, # but the idea is to test all the process insdrp = numina.drps.get_system_drps().query_by_name(ob.instrument) pipeline = insdrp.pipelines.get('default') recipe_fqn = pipeline.recipes.get(ob.mode) RecipeClass = import_object(recipe_fqn) assert RecipeClass is BiasRecipe # TODO: these should be created by a build_recipe_input method recipe = BiasRecipe() ri = recipe.create_input(obresult=ob) result = recipe.run(ri) # assert result.qc >= QC.UNKNOWN # Checks on the image hdulist = result.biasframe.open() assert len(hdulist) == 1 hdu = hdulist[0] assert hdu.shape == (4112, 4096) data = hdu.data mlevel = 0.0 block = data[1980:2020, 1980:2020] mblock = block.mean() sblock = block.std() assert abs(mblock - mlevel) < 5 * sblock # In the end, remove the files for f in fs: os.remove(f.name)
def test_recipe2(): drps = init_drp_system() BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/BIAS/%s' images = ['e99d2937d2c29a27c0ba4eebfcf7918e', 'e99d2937d2c29a27c0ba4eebfcf7918e', 'e99d2937d2c29a27c0ba4eebfcf7918e'] fs = [download_cache(BASE_URL % i) for i in images] ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'bias_image' ob.frames = [DataFrame(filename=f.name) for f in fs] instrument = drps.get(ob.instrument) pipeline = instrument.pipelines.get('default') recipe_fqn = pipeline.recipes.get(ob.mode) RecipeClass = import_object(recipe_fqn) assert RecipeClass is BiasRecipe # TODO: these should be created by a RecipeInputBuilder recipe = BiasRecipe() RR = BiasRecipe.RecipeRequirements # @UndefinedVariable ri = RR(obresult=ob) result = recipe.run(ri) # assert result.qc >= QC.UNKNOWN # Checks on the image hdulist = result.biasframe.open() assert len(hdulist) == 1 hdu = hdulist[0] assert hdu.shape == (4112, 4096) data = hdu.data mlevel = 0.0 block = data[1980:2020, 1980:2020] mblock = block.mean() sblock = block.std() assert abs(mblock - mlevel) < 5 * sblock # In the end, remove the files for f in fs: os.remove(f.name)
def build_recipe_input_gtc(cls, obsres, dal, pipeline='default'): cls.logger.debug('start recipe input builder') stareImagesIds = obsres.stareImagesIds cls.logger.debug('LCB image IDS %s: ', stareImagesIds) stareImages = [] for subresId in stareImagesIds: subres = dal.getRecipeResult(subresId) stareImages.append(subres['elements']['final_rss']) newOR = ObservationResult() newOR.frames = stareImages newRI = cls.create_input(obresult=newOR) cls.logger.debug('end recipe input builder') return newRI
def buildRecipeInput(self, obsres): if self.sky_image is None: print('obtaining SKY image') sky_cal_result = self.dal.getLastRecipeResult("EMIR", "EMIR", "IMAGE_SKY") self.sky_image = sky_cal_result['elements']['skyframe'] obsres['master_sky'] = self.sky_image newOR = ObservationResult() newOR.frames = obsres['frames'] obsres['obresult'] = newOR newRI = StareImageRecipeInput(**obsres) return newRI
def buildRecipeInput(self, obsres): if self.sky_image is None: print('obtaining SKY image') sky_cal_result = self.dal.getLastRecipeResult( "EMIR", "EMIR", "IMAGE_SKY") self.sky_image = sky_cal_result['elements']['skyframe'] obsres['master_sky'] = self.sky_image newOR = ObservationResult() newOR.frames = obsres['frames'] obsres['obresult'] = newOR newRI = StareImageRecipeInput(**obsres) return newRI
def test_bias(): BASE_URL = 'http://guaix.fis.ucm.es/~spr/megara_test/BIAS/%s' images = ['e99d2937d2c29a27c0ba4eebfcf7918e', 'e99d2937d2c29a27c0ba4eebfcf7918e', 'e99d2937d2c29a27c0ba4eebfcf7918e'] fs = [download_cache(BASE_URL % i) for i in images] ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'MegaraBiasImage' ob.frames = [DataFrame(filename=f.name) for f in fs] insdrp = load_drp() pipeline = insdrp.pipelines.get('default') recipe = pipeline.get_recipe_object(ob.mode) assert isinstance(recipe, BiasRecipe) # TODO: these should be created by a build_recipe_input method ob.configuration = insdrp.configuration_selector(ob) ri = recipe.create_input(obresult=ob) result = recipe.run(ri) # assert result.qc >= QC.UNKNOWN # Checks on the image hdulist = result.master_bias.open() assert len(hdulist) == 1 hdu = hdulist[0] assert hdu.shape == (4112, 4096) data = hdu.data mlevel = 0.0 block = data[1980:2020, 1980:2020] mblock = block.mean() sblock = block.std() assert abs(mblock - mlevel) < 5 * sblock # In the end, remove the files for f in fs: os.remove(f.name)
def run_single(self, rinput): self.logger.info('starting spectroscopy ABBA reduction') flow = self.init_filters(rinput) nimages = len(rinput.obresult.frames) self.logger.info('we receive %d images', nimages) if nimages != 4: msg = 'Recipe expects 4 images, received %d' % nimages raise numina.exceptions.RecipeError(msg) procesed_hdulists = basic_processing(rinput, flow) # INPUTS are ABBA, so # hdulist = self.process_abba(procesed_hdulists) grism = hdulist[0].header.get('GRISM', 'unknown') if grism.lower() == 'open': # perform TEST10 in addition import emirdrp.recipes.acquisition.maskcheck as mk from numina.core import ObservationResult, DataFrame import numpy try: import StringIO as S except ImportError: import io as S self.logger.info('GRISM is OPEN, doing a RECIPE10') sub = mk.MaskCheckRecipe() sub.configure(instrument='EMIR', mode='TEST10') o = ObservationResult() o.__dict__ = rinput.obresult.__dict__ o.frames = [DataFrame(frame=hdulist)] subd = {} subd['obresult'] = o ss = S.StringIO(_NOMINALPOS) subd['bars_nominal_positions'] = numpy.loadtxt(ss) subinput = mk.MaskCheckRecipe.RecipeInput(**subd) sub.run(subinput) result = self.create_result(reduced_mos_abba=hdulist) self.logger.info('end spectroscopy ABBA reduction') return result
def generate_bias(detector, number, temporary_path): from megaradrp.simulation.actions import simulate_bias from megaradrp.recipes.calibration.bias import BiasRecipe fs = [simulate_bias(detector) for i in range(number)] for aux in range(len(fs)): fits.writeto('%s/bias_%s.fits' % (temporary_path, aux), fs[aux], clobber=True) fs = ["%s/bias_%s.fits" % (temporary_path, i) for i in range(number)] ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'bias_image' ob.configuration = build_instrument_config('4fd05b24-2ed9-457b-b563-a3c618bb1d4c', loader=Loader()) ob.frames = [DataFrame(filename=f) for f in fs] recipe = BiasRecipe() ri = recipe.create_input(obresult=ob) return recipe.run(ri)
def test_bpm_corrector(): import shutil from tempfile import mkdtemp directorio = mkdtemp() names = crear_archivos(directorio, number=4) ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'bias_image' ob.configuration = build_instrument_config('4fd05b24-2ed9-457b-b563-a3c618bb1d4c', loader=Loader()) ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names] recipe = TestRecipe(directorio) ri = recipe.create_input(obresult=ob, master_bias=DataFrame( filename=open(directorio + '/master_bias_data0.fits').name), master_bpm=DataFrame(filename=open( directorio + '/master_bpm.fits').name)) recipe.run(ri) shutil.rmtree(directorio)
def build_recipe_input_gtc(self, obsres, dal, pipeline='default'): newOR = ObservationResult() # FIXME: this method will work only in GTC # stareImagesIds = obsres['stareImagesIds']._v stareImagesIds = obsres.stareImagesIds obsres.children = stareImagesIds self.logger.info('Submode result IDs: %s', obsres.children) stareImages = [] # Field to query the results key_field = 'frame' for subresId in obsres.children: subres = dal.getRecipeResult(subresId) # This 'frame' is the name of the product in RecipeResult # there is also a 'sky' field elements = subres['elements'] stareImages.append(elements[key_field]) newOR.frames = stareImages naccum = obsres.naccum self.logger.info('naccum: %d', naccum) mode_field = "DITHERED_IMAGE" key_field = 'accum' if naccum != 1: # if it is not the first dithering loop self.logger.info("SEARCHING LATEST RESULT of %s", mode_field) latest_result = dal.getLastRecipeResult("EMIR", "EMIR", mode_field) elements = latest_result['elements'] accum_dither = elements[key_field] self.logger.info("FOUND") else: self.logger.info("NO ACCUMULATION") accum_dither = stareImages[0] newOR.naccum = naccum newOR.accum = accum_dither # obsres['obresult'] = newOR # print('Adding RI parameters ', obsres) # newRI = DitheredImageARecipeInput(**obsres) newRI = self.create_input(obresult=newOR) return newRI
def test_bpm(): number = 5 PSCAN = 50 DSHAPE = (2056 * 2, 2048 * 2) OSCAN = 50 ron = 2.0 gain = 1.0 bias = 1000.0 qe = 0.8 * np.ones(DSHAPE) qe[5:6, 0:170] = 0.0 config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c' date_obs = '2017-11-09T11:00:00.0' temporary_path = mkdtemp() fits.writeto('{}/eq.fits'.format(temporary_path), qe, overwrite=True) readpars1 = ReadParams(gain=gain, ron=ron, bias=bias) readpars2 = ReadParams(gain=gain, ron=ron, bias=bias) detector = MegaraDetectorSat('megara_test_detector', DSHAPE, OSCAN, PSCAN, qe=qe, dark=(3.0 / 3600.0), readpars1=readpars1, readpars2=readpars2, bins='11') source2 = 1.0 fs = [simulate_flat(detector, exposure=1.0, source=5000 * source2) for i in range(number)] fs2 = [simulate_flat(detector, exposure=1.0, source=40000 * source2) for i in range(number)] header = fits.Header() header['DATE-OBS'] = date_obs header['INSCONF'] = config_uuid header['INSTRUME'] = 'MEGARA' header['VPH'] = 'LR-U' header['INSMODE'] = 'MOS' for aux in range(len(fs)): fits.writeto('{}/flat_{}.fits'.format(temporary_path, aux), fs[aux], header=header, overwrite=True) fits.writeto('{}/flat_{}.fits'.format(temporary_path, aux + number), fs2[aux], header=header, overwrite=True) result = generate_bias(detector, number, temporary_path) result.master_bias.frame.writeto( '{}/master_bias_data0.fits'.format(temporary_path), overwrite=True ) ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'MegaraBiasImage' pkg_paths = ['megaradrp.instrument.configs'] store = asb.load_paths_store(pkg_paths) insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid') insmodel.configure_with_header(header) ob.configuration = insmodel names = [] for aux in range(number * 2): names.append('{}/flat_{}.fits'.format(temporary_path, aux)) ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names] recipe = BadPixelsMaskRecipe() ri = recipe.create_input(obresult=ob, master_bias=DataFrame( filename=open(temporary_path + '/master_bias_data0.fits').name)) aux = recipe.run(ri) aux.master_bpm.frame.writeto('{}/master_bpm.fits'.format(temporary_path), overwrite=True) shutil.rmtree(temporary_path)
def test_dark(): numpy.random.seed(422992983) PSCAN = 50 DSHAPE = (2056 * 2, 2048 * 2) OSCAN = 50 # ron = 2.0 ron = 0.001 gain = 1.0 bias = 1000.0 dark = 3.0 # In 1 hour exptime = 3600.0 dark_s = dark / exptime qe = 0.8 * numpy.ones(DSHAPE) config_uuid = '4fd05b24-2ed9-457b-b563-a3c618bb1d4c' date_obs = '2017-11-09T11:00:00.0' temporary_path = mkdtemp() fits.writeto('{}/eq.fits'.format(temporary_path), qe, overwrite=True) readpars1 = ReadParams(gain=gain, ron=ron, bias=bias) readpars2 = ReadParams(gain=gain, ron=ron, bias=bias) detector = MegaraDetectorSat('megara_test_detector', DSHAPE, OSCAN, PSCAN, qe=qe, dark=dark_s, readpars1=readpars1, readpars2=readpars2, bins='11') number = 3 factory = MegaraImageFactory() fs = simulate_dark_fits(factory, detector, exposure=3600, repeat=number) for idx, aux in enumerate(fs): aux.writeto('{}/dark_{}.fits'.format(temporary_path, idx), overwrite=True) header = fits.Header() header['DATE-OBS'] = date_obs header['INSCONF'] = config_uuid header['INSTRUME'] = 'MEGARA' header['VPH'] = 'LR-U' header['INSMODE'] = 'MOS' master_bias_data = numpy.zeros(DSHAPE) master_bias_hdul = fits.HDUList(fits.PrimaryHDU( master_bias_data, header=header) ) #master_bias_data = master_bias.master_bias.frame[0].data ob = ObservationResult() ob.instrument = 'MEGARA' ob.mode = 'MegaraDarkImage' pkg_paths = ['megaradrp.instrument.configs'] store = asb.load_paths_store(pkg_paths) insmodel = asb.assembly_instrument(store, config_uuid, date_obs, by_key='uuid') insmodel.configure_with_header(header) ob.configuration = insmodel names = [] for aux in range(number): names.append('{}/dark_{}.fits'.format(temporary_path, aux)) ob.frames = [DataFrame(filename=open(nombre).name) for nombre in names] ob.configuration = insmodel recipe = DarkRecipe() ri = recipe.create_input( obresult=ob, master_bias=DataFrame(frame=master_bias_hdul), ) aux = recipe.run(ri) mean_dark_value = aux.master_dark.frame[0].data.mean() shutil.rmtree(temporary_path) assert numpy.allclose(mean_dark_value, dark, atol=0, rtol=1e-1)