def concatImages(self, sweepType, removeBkg=True): """ This won't work well if the background level or QE of the pixel changes between sweeps... Should remove this first """ sweepType = sweepType.lower() assert sweepType in ('x', 'y') imageList = None for s in self.config.beammap.sweep.sweeps: if s.sweeptype in sweepType: getLogger('beammap').info('loading: ' + str(s)) imList = self.loadSweepImgs(s).astype(np.float) if removeBkg: bkgndList = np.median(imList, axis=0) imList -= bkgndList direction = -1 if s.sweepdirection == '-' else 1 if imageList is None: imageList = imList[::direction, :, :] else: imageList = np.concatenate( (imageList, imList[::direction, :, :]), axis=0) if sweepType == 'x': self.x_images = imageList else: self.y_images = imageList return imageList
def __init__(self, config=None): self.config_file = DEFAULT_CONFIG_FILE if config is None else config self.cfg = mkidpipeline.config.load_task_config(config) self.use_wavecal = self.cfg.flatcal.use_wavecal self.dataDir = self.cfg.paths.data self.out_directory = self.cfg.paths.out self.intTime = self.cfg.flatcal.chunk_time self.dark_start = [] self.dark_int = [] self.xpix = self.cfg.beammap.ncols self.ypix = self.cfg.beammap.nrows self.deadtime = self.cfg.instrument.deadtime self.wvlStart = self.cfg.instrument.wvl_start self.wvlStop = self.cfg.instrument.wvl_stop self.countRateCutoff = self.cfg.flatcal.rate_cutoff self.fractionOfChunksToTrim = self.cfg.flatcal.trim_fraction self.timeSpacingCut = None self.obs = None self.beamImage = None self.wvlFlags = None self.wvlBinEdges = None self.wavelengths = None if self.use_wavecal: sol_file = self.cfg.wavcal #TODO make sure this works then utilize this in speccal as well sol = mkidpipeline.calibration.wavecal.Solution(sol_file[0]) r, resid = sol.find_resolving_powers(cache=True) self.r_list = np.nanmedian(r, axis=0) else: self.r_list = None self.save_plots = self.cfg.flatcal.plots.lower() == 'all' self.summary_plot = self.cfg.flatcal.plots.lower() in ('all', 'summary') if self.save_plots: getLogger(__name__).warning( "Comanded to save debug plots, this will add ~30 min to runtime." ) self.spectralCubes = None self.cubeEffIntTimes = None self.countCubes = None self.cubeWeights = None self.weightErr = None self.totalCube = None self.totalFrame = None self.flatWeights = None self.countCubesToSave = None self.flatWeightErr = None self.flatFlags = None self.flatWeightsforplot = None self.plotName = None self.fig = None
def getAbsOffset(self, shiftedTimes, auto=True, locLimit=None): """ The autocorrelation function can only calculate relative time differences between pixels. This function defines the absolute time reference (ie. the location of the peak) INPUTS: shiftedTimes: a list of pixel time streams shifted to match up auto: if False then ask user to click on a plot """ if not np.isfinite( locLimit) or locLimit < 0 or locLimit >= len(shiftedTimes): locLimit = -1 offset = np.argmax(np.sum(shiftedTimes[:locLimit], axis=0)) if auto: return offset getLogger('beammap').info("Please click the correct peak") fig, ax = plt.subplots() for p_i in range(len(shiftedTimes)): ax.plot(shiftedTimes[p_i]) ax.plot(np.sum(shiftedTimes, axis=0), 'k-') ln = ax.axvline(offset, c='b') def onclick(event): if fig.canvas.manager.toolbar._active is None: offset = event.xdata getLogger('beammap').info(offset) ln.set_xdata(offset) plt.draw() cid = fig.canvas.mpl_connect('button_press_event', onclick) plt.show() return offset
def from_yaml(cls, loader, node): d = dict(loader.construct_pairs(node)) if 'approximate_time' in d: d.pop('file', None) return cls(d.pop('name'), d.pop('wavecal', None), d.pop('flatcal', None), d.pop('wcscal'), d.pop('speccal', None), byTimestamp=d.pop('approximate_time'), use=d.pop('use', None), _common=d) if not os.path.isfile(d['file']): getLogger(__name__).info( 'Treating {} as relative dither path.'.format(d['file'])) d['file'] = os.path.join(config.paths.dithers, d['file']) return cls(d.pop('name'), d.pop('wavecal', None), d.pop('flatcal', None), d.pop('wcscal'), d.pop('speccal', None), byLegacyFile=d.pop('file'), use=d.pop('use', None), _common=d)
def buildtables(timeranges, config=None, ncpu=1, asynchronous=False, remake=False, **kwargs): timeranges = list(set(timeranges)) b2h_configs = gen_configs(timeranges, config) builders = [HDFBuilder(c, force=remake, **kwargs) for c in b2h_configs] if ncpu == 1: for b in builders: try: b.run(**kwargs) except MemoryError: getLogger(__name__).error( 'Insufficient memory to process {}'.format(b.h5file)) return timeranges pool = mp.Pool(min(ncpu, mp.cpu_count())) if asynchronous: getLogger(__name__).debug('Running async on {} builders'.format( len(builders))) async_res = pool.map_async(runbuilder, builders) pool.close() return timeranges, async_res else: pool.map(runbuilder, builders) pool.close() pool.join()
def fetch_spectra(self): ''' called from get(), searches either a URL, ESO catalog or uses astroquery.SDSS to search the SDSS catalog. Puts the retrieved spectrum in a '/spectrum/' folder in self.save_dir :return: ''' if self.std_path is not None: if self.std_path.endswith('.txt'): data = np.loadtxt(self.std_path) else: self.spectrum_file = fetch_spectra_URL(object_name=self.object_name, url_path=self.std_path, save_dir=self.save_dir) data = np.loadtxt(self.spectrum_file) return data else: self.spectrum_file = fetch_spectra_ESO(object_name=self.object_name, save_dir=self.save_dir) if not self.spectrum_file: self.spectrum_file = fetch_spectra_SDSS(object_name=self.object_name, save_dir=self.save_dir, coords=self.coords) try: data = np.loadtxt(self.spectrum_file) return data except ValueError: getLogger(__name__).error( 'Could not find standard spectrum for this object, please find a spectrum and point to it in ' 'the standard_path in your pipe.yml') sys.exit() data = np.loadtxt(self.spectrum_file) # to convert to the appropriate units if ESO spectra data[:, 1] = data[:, 1] * 10**(-16) return data
def handle_existing(self): """ Handles existing h5 files, deleting them if appropriate""" if os.path.exists(self.cfg.h5file): if self.force: getLogger(__name__).info('Remaking {} forced'.format( self.cfg.h5file)) done = False else: try: done = Photontable( self.cfg.h5file).duration >= self.cfg.inttime if not done: getLogger(__name__).info( ('{} does not contain full duration, ' 'will remove and rebuild').format( self.cfg.h5file)) except: done = False getLogger(__name__).info( ('{} presumed corrupt,' ' will remove and rebuild').format(self.cfg.h5file), exc_info=True) if not done: try: os.remove(self.cfg.h5file) getLogger(__name__).info('Deleted {}'.format( self.cfg.h5file)) except FileNotFoundError: pass else: getLogger(__name__).info( 'H5 {} already built. Remake not requested. Done.'.format( self.cfg.h5file)) self.done = True
def extend_spectrum(self, x, y): """ BB Fit to extend standard spectrum to 1500 nm and to convolve it with a gaussian kernel corresponding to the energy resolution of the detector """ r = np.median(np.nanmedian(self.cfg.r_list, axis=0)) if np.round(x[-1]) < self.cfg.wvlStop: fraction = 1.0 / 3.0 nirX = np.arange(int(x[int((1.0 - fraction) * len(x))]), self.cfg.wvlStop) T, nirY = fitBlackbody(x, y, fraction=fraction, newWvls=nirX) if np.any(x >= self.cfg.wvlStop): self.bb_wvls = x self.bb_flux = y else: self.bb_wvls = np.concatenate((x, nirX[nirX > max(x)])) self.bb_flux = np.concatenate((y, nirY[nirX > max(x)])) # Gaussian convolution to smooth std spectrum to MKIDs median resolution newX, newY = gaussianConvolution(self.bb_wvls, self.bb_flux, xEnMin=self.cfg.energyStop, xEnMax=self.cfg.energyStart, fluxUnits="lambda", r=r, plots=False) else: getLogger(__name__).info('Standard Spectrum spans whole energy range - no need to perform blackbody fit') # Gaussian convolution to smooth std spectrum to MKIDs median resolution std_stop = (c.h * c.c) / (self.std_wvls[0] * 10**(-10) * c.e) std_start = (c.h * c.c) / (self.std_wvls[-1] * 10 ** (-10) * c.e) newX, newY = gaussianConvolution(x, y, xEnMin=std_start, xEnMax=std_stop, fluxUnits="lambda", r=r, plots=False) return newX, newY
def write(self, file): dir = self.datadir if not glob(os.path.join(dir, '*.bin')): dir = os.path.join( self.datadir, datetime.utcfromtimestamp(self.starttime).strftime('%Y%m%d')) else: getLogger(__name__).debug( 'bin files found in data directory. Will not append YYYMMDD') try: file.write( self._template.format(datadir=dir, starttime=self.starttime, inttime=self.inttime, beamfile=self.beamfile, outdir=self.outdir, x=self.x, y=self.y)) except AttributeError: with open(file, 'w') as wavefile: wavefile.write( self._template.format(datadir=dir, starttime=self.starttime, inttime=self.inttime, beamfile=self.beamfile, outdir=self.outdir, x=self.x, y=self.y))
def get_dark_frame(self): ''' takes however many dark files that are specified in the pipe.yml and computes the counts/pixel/sec for the sum of all the dark obs. This creates a stitched together long dark obs from all of the smaller obs given. This is useful for legacy data where there may not be a specified dark observation but parts of observations where the filter wheel was closed. If self.use_wavecal is True then a dark is not subtracted off since this just takes into account total counts and not energy information :return: expected dark counts for each pixel over a flat observation ''' if not self.dark_h5_file_names: dark_frame = np.zeros_like(self.spectralCubes[0][:, :, 0]) else: self.dark_start = [self.cfg.flatcal.dark_data['start_times']] self.dark_int = [self.cfg.flatcal.dark_data['int_times']] self.dark_h5_file_names = [ os.path.join(self.h5_directory, str(t) + '.h5') for t in self.dark_start ] frames = np.zeros((140, 146, len(self.dark_start))) getLogger(__name__).info('Loading dark frames for Laser flat') for i, file in enumerate(self.dark_h5_file_names): obs = Photontable(file) frame = obs.getPixelCountImage( integrationTime=self.dark_int[i])['image'] frames[:, :, i] = frame total_counts = np.sum(frames, axis=2) total_int_time = float(np.sum(self.dark_int)) counts_per_sec = total_counts / total_int_time dark_frame = counts_per_sec return dark_frame
def allowed(self, key, value): if key + '._a' in self: getLogger(__name__).warning( str(key) + ' has restrictions no allowed values but checking has not yet been ' 'implemented') return True
def saveInferenceFile(self): metadatafile = self.inferenceFile.rpartition('.')[0] + '_metadata.txt' try: flNum = int( re.search('fl\d', self.inferenceFile, re.IGNORECASE).group()[-1]) except AttributeError: getLogger(__name__).warning( 'Could not guess feedline from filename.') flNum = 0 ws_good_inds = self.goodPeakIndices freqs = np.append(self.inferenceData.freqs[ws_good_inds], self.inferenceData.freqs[ws_bad_inds]) sort_inds = np.argsort(freqs) resIds = np.arange(freqs.size) + flNum * 10000 flag = np.fill(freqs.size, sweepdata.ISBAD) flag[self.goodPeakIndices] = sweepdata.ISGOOD smd = sweepdata.SweepMetadata(resid=resIds, flag=flag[sort_inds], wsfreq=freqs[sort_inds], file=metadatafile) smd.save()
def read_from_file(self, filename): with open(filename, 'r') as f: old = '' for line in f: line = line.strip() if len(line) == 0 or line[0] == '#': continue s = line.split('#') line = s[0] s = line.split('\\') if len(s) > 1: old = string.join([old, s[0]]) continue else: line = string.join([old, s[0]]) old = '' for i in xrange(len(line)): if line[i] != ' ': line = line[i:] break exec(line) s = line.split('=') if len(s) != 2: getLogger(__name__).warning( "Error parsing line:\n\t'{}'".format(line)) continue key = s[0].strip() val = eval(s[1].strip()) # XXX:make safer self[key] = val
def __init__(self, yml): self.yml = yml self.meta = mkidcore.config.load(yml) names = [d.name for d in self.meta] if len(names) != len(set(names)): msg = 'Duplicate names not allowed in {}.'.format(yml) getLogger(__name__).critical(msg) raise ValueError(msg) wcdict = {w.name: w for w in self.wavecals} fcdict = {f.name: f for f in self.flatcals} wcsdict = {w.name: w for w in self.wcscals} scdict = {s.name: s for s in self.spectralcals} for o in self.all_observations: o.wavecal = wcdict.get(o.wavecal, o.wavecal) o.speccal = scdict.get(o.speccal, o.speccal) o.flatcal = fcdict.get(o.flatcal, o.flatcal) o.wcscal = wcsdict.get(o.wcscal, o.wcscal) for o in self.science_observations: o.flatcal = fcdict.get(o.flatcal, o.flatcal) o.wcscal = wcsdict.get(o.wcscal, o.wcscal) o.speccal = scdict.get(o.speccal, o.speccal) for fc in self.flatcals: try: fc.wavecal = wcdict.get(fc.wavecal, fc.wavecal) except AttributeError: pass for sc in self.spectralcals: for d in sc.data: try: d.wavecal = wcdict.get(d.wavecal, d.wavecal) except AttributeError: pass try: d.flatcal = fcdict.get(d.flatcal, d.flatcal) except AttributeError: pass try: d.wcscal = wcsdict.get(d.wcscal, d.wcscal) except AttributeError: pass for d in self.dithers: try: d.wavecal = wcdict.get(d.wavecal, d.wavecal) except AttributeError: pass try: d.flatcal = fcdict.get(d.flatcal, d.flatcal) except AttributeError: pass try: d.wcscal = wcsdict.get(d.wcscal, d.wcscal) except AttributeError: pass
def _load(self, filename): """ Loads beammap data from filename """ self.file = filename getLogger(__name__).debug('Reading {}'.format(self.file)) self.resIDs, self.flags, self.xCoords, self.yCoords = np.loadtxt( filename, unpack=True)
def runbuilder(b): getLogger(__name__).debug('Calling run on {}'.format(b.cfg.h5file)) try: b.run() except Exception as e: getLogger(__name__).critical( 'Caught exception during run of {}'.format(b.cfg.h5file), exc_info=True)
def loadData(self): ''' placeholder function to maintain universality of the makeCalibration function ''' getLogger(__name__).info( 'No need to load wavecal solution data for a Laser Flat, passing through' ) pass
def saveRoughBeammap(self): getLogger('beammap').info('Saving: '.format(self.outputBeammapFn)) allResIDs = self.resIDsMap.flatten() flags = self.flagMap.flatten() x = self.x_loc.flatten() y = self.y_loc.flatten() args = np.argsort(allResIDs) data = np.asarray([allResIDs[args], flags[args], x[args], y[args]]).T np.savetxt(self.outputBeammapFn, data, fmt='%7d %3d %7f %7f')
def satellite_spot_contrast(lam): """ :param lam: wavelength in angstroms :return: contrast """ getLogger(__name__).info('Using satellite spot contrast for a 25 nm astrogrid') ref = 1.55*10**4 contrast = 2.72e-3*(ref / lam)**2 # 2.72e-3 number from Currie et. al. 2018b return contrast
def create_spectra_directory(save_dir): """ creates a spectrum directory in the save directory to put the spectra. If not called then the spectrum will just be saved in save_path """ if os.path.exists(save_dir + '/spectra/'): getLogger(__name__).info('Spectrum directory already exists in {}, not going to make a new one'.format(save_dir)) else: os.mkdir(save_dir + '/spectra/') return save_dir + '/spectra/'
def _makeimage(xs, ys, img_shape, verbose=False): """See documentation in ParseBin.image() for best info""" tic = time.time() ret = np.zeros(tuple(img_shape)) np.add.at(ret, (ys, xs), 1) toc = time.time() if verbose: getLogger('binparse').debug( "Time to make image is {:4.2f} seconds".format(toc - tic)) return ret
def __getitem__(self, key): if key not in self: if self.ask: getLogger(__name__).info("Parameter '%s' not found" % key) val = ask_for(key) getLogger(__name__).info("Setting '%s' = %s" % (key, repr(val))) dict.__setitem__(self, key, val) else: return None return dict.__getitem__(self, key)
def validate_metadata(md, warn=True, error=False): fail = False for k in REQUIRED_KEYS: if k not in md: if error: raise KeyError(msg) fail = True msg = '{} missing from {}'.format(k, md) if warn: getLogger(__name__).warning(msg) return fail
def save(self): """Save the time-stream data to the object's file path.""" try: np.savez(self.file_path, self.phase) except IOError: path = self.file_path.rsplit('/', 1) if len(path) <= 1: raise getLogger(__name__).info('Making directory: ' + path[0]) os.mkdir(path[0]) np.savez(self.file_path, self.phase)
def load_observing_metadata(files=tuple(), include_database=True): """Return a list of mkidcore.config.ConfigThings with the contents of the metadata from observing""" global config files = list(files) if config is not None and include_database: files += glob(os.path.join(config.paths.database, 'obslog*.json')) elif include_database: getLogger(__name__).warning('No pipleline database configured.') metadata = [] for f in files: metadata += parse_obslog(f) return metadata
def save(self, save_name=None): """Save the solution to a file. The directory is given by the configuration.""" if save_name is None: save_path = os.path.join(self.cfg.save_directory, self.name) else: save_path = os.path.join(self.cfg.save_directory, save_name) if not save_path.endswith('.npz'): save_path += '.npz' getLogger(__name__).info("Saving spectrophotometric response curve to {}".format(save_path)) np.savez(save_path, curve=self.curve, wvl_bin_widths=self.wvl_bin_widths, wvl_bin_centers=self.wvl_bin_centers, cube=self.cube, errors=self.errors, configuration=self.cfg) self._file_path = save_path # new file_path for the solution
def __init__(self, file): self.count_images = {} with open(file, 'rb') as f: self._r = pickle.load(f) try: with open(file + '.count_images', 'rb') as f: self.count_images = pickle.load(f) except IOError: getLogger(__name__).info( 'No count image file found. Data will need to be regenerated.') pass settings = [ dict(index=ndx, chunkshape=cshape, timesort=tsort, shuffle=shuffle, bitshuffle=bshuffle, ndx_bitshuffle=ndx_bshuffle, ndx_shuffle=ndx_shuffle) for ndx in [(k, l) for k in ('full', 'medium', 'ultralight') for l in (3, 6, 9)] for tsort in (False, True) for cshape in (20, 100, 180, 260, 340, 500, 1000, 5000, 10000, None) for ndx_shuffle in (True, False) for ndx_bshuffle in (True, False) for shuffle in (True, False) for bshuffle in (True, False) ] for k, v in self._r.items(): for s in settings: if setting_id(s) in k: v['kwargs'] = repr(s) break for k, v in self._r.items(): if 'kwargs' not in v: print(v['table_nfo'], k) raise ValueError for k in self._r: matches = re.search(r"chunkshape := \((\d+),\)", self._r[k]['table_nfo']) self._r[k]['chunkshape'] = int(matches.group(1)) if not all([d in self.count_images for d in self.datasets]): print('Will determine count images, may take some time.') self.determine_mean_photperRID() with open(file + '.count_images', 'wb') as f: pickle.dump(self.count_images, f)
def run(self, save=True, plot=None): """ Compute the spectrophotometric calibration for the data specified in the configuration object. This method runs load_relative_spectrum() and load_sky_spectrum() or load_absolute_sepctrum(), load_standard_spectrum(), and calculate_specweights() sequentially. Args: save: a boolean specifying if the result will be saved. plot: a boolean specifying if a summary plot for the computation will be saved. """ try: getLogger(__name__).info("Loading Spectrum from MEC") self.load_absolute_spectrum() getLogger(__name__).info("Loading Standard Spectrum") self.load_standard_spectrum() getLogger(__name__).info("Calculating Spectrophotometric Response Curve") self.calculate_response_curve() self.solution = ResponseCurve(configuration=self.cfg, curve=self.curve, wvl_bin_widths=self.wvl_bin_widths, wvl_bin_centers=self.wvl_bin_centers, cube=self.cube, solution_name=self.solution_name, errors=self.errors) if save: self.solution.save(save_name=self.solution_name if isinstance(self.solution_name, str) else None) if plot or (plot is None and self.cfg.summary_plot): save_name = self.solution_name.rpartition(".")[0] + ".pdf" self.plot_summary(save_name=save_name) except KeyboardInterrupt: getLogger(__name__).info("Keyboard shutdown requested ... exiting")
def guessFeedline(filename): # TODO generaize and find a home for this function try: flNum = int(re.search('fl\d', filename, re.IGNORECASE).group()[-1]) except AttributeError: try: ip = int(os.path.splitext(filename)[0][-3:]) flNum = int(MEC_NUM_FL_MAP[ip][:-1]) except (KeyError, ValueError, IndexError): getLogger(__name__).warning('Could not guess feedline from filename {}.') raise ValueError('Unable to guess feedline') getLogger(__name__).debug('Guessing FL{} for filename {}'.format(flNum, os.path.basename(filename))) return flNum
def fetch_spectra_SDSS(object_name, save_dir, coords): """ saves a textfile in self.save_dir where the first column is the wavelength in angstroms and the second column is flux in erg cm-2 s-1 AA-1 :return: the path to the saved spectrum file """ if os.path.exists(save_dir + object_name + 'spectrum.dat'): getLogger(__name__).info('Spectrum already loaded, will not be reloaded') spectrum_file = save_dir + object_name + 'spectrum.dat' return spectrum_file getLogger(__name__).info('Looking for {} spectrum in SDSS catalog'.format(object_name)) result = SDSS.query_region(coords, spectro=True) if not result: getLogger(__name__).warning('Could not find spectrum for {} at {},{} in SDSS catalog'.format(object_name, coords.ra, coords.dec)) spectrum_file = None return spectrum_file spec = SDSS.get_spectra(matches=result) data = spec[0][1].data lamb = 10**data['loglam'] * u.AA flux = data['flux'] * 10 ** -17 * u.Unit('erg cm-2 s-1 AA-1') spectrum = Spectrum1D(spectral_axis=lamb, flux=flux) res = np.array([spectrum.spectral_axis, spectrum.flux]) res = res.T spectrum_file = save_dir + object_name + 'spectrum.dat' np.savetxt(spectrum_file, res, fmt='%1.4e') getLogger(__name__).info('Spectrum loaded for {} from SDSS catalog'.format(object_name)) return spectrum_file