def _runfliptest(self, a, flip, aflip): is_a = imageseries.open(None, 'array', data=a) ops = [('flip', flip)] is_p = process.ProcessedImageSeries(is_a, ops) is_aflip = imageseries.open(None, 'array', data=aflip) diff = compare(is_aflip, is_p) msg = "flipped [%s] image series failed" % flip self.assertAlmostEqual(diff, 0., msg=msg)
def test_process_framelist(self): a = make_array() is_a = imageseries.open(None, 'array', data=a) ops = [] frames = [0, 2] is_p = process.ProcessedImageSeries(is_a, ops, frame_list=frames) is_a2 = imageseries.open(None, 'array', data=a[tuple(frames), ...]) diff = compare(is_a2, is_p) self.assertAlmostEqual(diff, 0., msg="frame list failed")
def test_process_dark(self): """Processed image series: dark image""" a = make_array() dark = np.ones_like(a[0]) is_a = imageseries.open(None, 'array', data=a) apos = np.where(a >= 1, a-1, 0) is_a1 = imageseries.open(None, 'array', data=apos) ops = [('dark', dark)] is_p = process.ProcessedImageSeries(is_a, ops) diff = compare(is_a1, is_p) self.assertAlmostEqual(diff, 0., msg="dark image failed")
def test_fmtfc(self): """save/load frame-cache format""" imageseries.write(self.is_a, self.fcfile, self.fmt, threshold=self.thresh, cache_file=self.cache_file) is_fc = imageseries.open(self.fcfile, self.fmt, style='yml') diff = compare(self.is_a, is_fc) self.assertAlmostEqual(diff, 0., "frame-cache reconstruction failed") self.assertTrue(compare_meta(self.is_a, is_fc))
def imageseries(self): """return the imageseries without checking for omega metadata""" if self._imser is None: self._imser = imageseries.open(self.filename, self.format, **self.args) plist = self.process.process_list if plist: self._imser = imageseries.process.ProcessedImageSeries(self._imser, plist) return self._imser
def test_fmth5(self): """save/load HDF5 format""" imageseries.write(self.is_a, self.h5file, self.fmt, path=self.h5path) is_h = imageseries.open(self.h5file, self.fmt, path=self.h5path) diff = compare(self.is_a, is_h) self.assertAlmostEqual(diff, 0., "h5 reconstruction failed") self.assertTrue(compare_meta(self.is_a, is_h))
def test_stats_max(self): """Processed imageseries: median""" a = make_array() is_a = imageseries.open(None, 'array', data=a) ismax = stats.max(is_a) amax = np.max(a, axis=0) err = np.linalg.norm(amax - ismax) self.assertAlmostEqual(err, 0., msg="max image failed")
def test_fmth5_nocompress(self): """HDF5 options: no compression""" imageseries.write(self.is_a, self.h5file, self.fmt, path=self.h5path, gzip=0) is_h = imageseries.open(self.h5file, self.fmt, path=self.h5path) diff = compare(self.is_a, is_h) self.assertAlmostEqual(diff, 0., "h5 reconstruction failed") self.assertTrue(compare_meta(self.is_a, is_h))
def test_process_shape(self): a = make_array() is_a = imageseries.open(None, 'array', data=a) ops = [] is_p = process.ProcessedImageSeries(is_a, ops) pshape = is_p.shape fshape = is_p[0].shape for i in range(2): self.assertEqual(fshape[i], pshape[i])
def test_fmth5_chunk(self): """HDF5 options: chunk size""" imageseries.write(self.is_a, self.h5file, self.fmt, path=self.h5path, chunk_rows=0) is_h = imageseries.open(self.h5file, self.fmt, path=self.h5path) diff = compare(self.is_a, is_h) self.assertAlmostEqual(diff, 0., "h5 reconstruction failed") self.assertTrue(compare_meta(self.is_a, is_h))
def test_fmtfc(self): """save/load frame-cache format""" imageseries.write(self.is_a, self.fcfile, self.fmt, threshold=self.thresh, cache_file=self.cache_file) is_fc = imageseries.open(self.fcfile, self.fmt) diff = compare(self.is_a, is_fc) self.assertAlmostEqual(diff, 0., "frame-cache reconstruction failed") self.assertTrue(compare_meta(self.is_a, is_fc))
def test_fmth5_nparray(self): """HDF5 format with numpy array metadata""" key = 'np-array' npa = np.array([0,2.0,1.3]) self.is_a.metadata[key] = npa imageseries.write(self.is_a, self.h5file, self.fmt, path=self.h5path) is_h = imageseries.open(self.h5file, self.fmt, path=self.h5path) meta = is_h.metadata diff = np.linalg.norm(meta[key] - npa) self.assertAlmostEqual(diff, 0., "h5 numpy array metadata failed")
def load_dummy_images(self, initial=False): HexrdConfig().clear_images(initial) detectors = HexrdConfig().detector_names iconfig = HexrdConfig().instrument_config for det in detectors: cols = iconfig['detectors'][det]['pixels']['columns'] rows = iconfig['detectors'][det]['pixels']['rows'] shape = (rows, cols) data = np.ones(shape, dtype=np.uint8) ims = imageseries.open(None, 'array', data=data) HexrdConfig().imageseries_dict[det] = ims
def test_fmth5_nparray(self): """HDF5 format with numpy array metadata""" key = 'np-array' npa = np.array([0, 2.0, 1.3]) self.is_a.metadata[key] = npa imageseries.write(self.is_a, self.h5file, self.fmt, path=self.h5path) is_h = imageseries.open(self.h5file, self.fmt, path=self.h5path) meta = is_h.metadata diff = np.linalg.norm(meta[key] - npa) self.assertAlmostEqual(diff, 0., "h5 numpy array metadata failed")
def test_fmtfc_nparray(self): """frame-cache format with numpy array metadata""" key = 'np-array' npa = np.array([0,2.0,1.3]) self.is_a.metadata[key] = npa imageseries.write(self.is_a, self.fcfile, self.fmt, threshold=self.thresh, cache_file=self.cache_file) is_fc = imageseries.open(self.fcfile, self.fmt) meta = is_fc.metadata diff = np.linalg.norm(meta[key] - npa) self.assertAlmostEqual(diff, 0., "frame-cache numpy array metadata failed")
def __init__(self, fname, omw, flips, frame_start=0): """Constructor for PP_Dexela""" # self.fname = fname self.omwedges = omw self.flips = flips self.frame_start = frame_start self.use_frame_list = (self.frame_start > 0) self.raw = imageseries.open(self.fname, self.RAWFMT, path=self.RAWPATH) self._dark = None print 'On Init: ', self.nframes, self.fname, self.omwedges.nframes,\ len(self.raw) return
def imageseries(self): """return the imageseries dictionary""" if self._image_dict is None: self._image_dict = dict() fmt = self.format for ispec in self.data: fname = ispec['file'] args = ispec['args'] ims = imageseries.open(fname, fmt, **args) oms = imageseries.omega.OmegaImageSeries(ims) panel = ims.metadata['panel'] self._image_dict[panel] = ims return self._image_dict
def imageseries(self): """return the imageseries dictionary""" if self._image_dict is None: self._image_dict = dict() fmt = self.format for ispec in self.data: fname = ispec['file'] args = ispec['args'] ims = imageseries.open(fname, fmt, **args) oms = imageseries.omega.OmegaImageSeries(ims) panel = oms.metadata['panel'] self._image_dict[panel] = oms return self._image_dict
def __init__(self, fname, omw, flips, panel_id, frame_start=0, raw_format='hdf5'): """Constructor for PP_Dexela""" self._panel_id = panel_id self.fname = fname self.omwedges = omw self.flips = flips self.frame_start = frame_start self.use_frame_list = (self.frame_start > 0) if raw_format.lower() == 'hdf5': self.raw = imageseries.open(self.fname, self.RAWFMT, path=self.RAWPATH) else: self.raw = imageseries.open(self.fname, raw_format.lower()) self._dark = None print('On Init:\n\t%s, %d frames, %d omw, %d total' % (self.fname, self.nframes, self.omwedges.nframes, len(self.raw)))
def test_fmtfc_nparray(self): """frame-cache format with numpy array metadata""" key = 'np-array' npa = np.array([0, 2.0, 1.3]) self.is_a.metadata[key] = npa imageseries.write(self.is_a, self.fcfile, self.fmt, threshold=self.thresh, cache_file=self.cache_file) is_fc = imageseries.open(self.fcfile, self.fmt) meta = is_fc.metadata diff = np.linalg.norm(meta[key] - npa) self.assertAlmostEqual(diff, 0., "frame-cache numpy array metadata failed")
def open_file(self, f): # f could be either a file or numpy array ext = os.path.splitext(f)[1] if isinstance(f, str) else None if ext is None: ims = imageseries.open(None, 'array', data=f) elif ext in self.HDF4_FILE_EXTS: from pyhdf.SD import SD, SDC hdf = SD(f, SDC.READ) dset = hdf.select(self.path[1]) ims = imageseries.open(None, 'array', data=dset) elif ext in self.HDF5_FILE_EXTS: data = h5py.File(f, 'r') dset = data['/'.join(self.path)][()] if dset.ndim < 3: # Handle raw two dimesional data ims = imageseries.open(None, 'array', data=dset) else: data.close() ims = imageseries.open(f, 'hdf5', path=self.path[0], dataname=self.path[1]) elif ext == '.npz': ims = imageseries.open(f, 'frame-cache') elif ext == '.yml': data = yaml.load(open(f)) form = next(iter(data)) ims = imageseries.open(f, form) else: # elif ext in self.IMAGE_FILE_EXTS: input_dict = {'image-files': {}} input_dict['image-files']['directory'] = os.path.dirname(f) input_dict['image-files']['files'] = os.path.basename(f) input_dict['options'] = {} input_dict['meta'] = {} temp = tempfile.NamedTemporaryFile(delete=False) try: data = yaml.dump(input_dict).encode('utf-8') temp.write(data) temp.close() ims = imageseries.open(temp.name, 'image-files') finally: # Ensure the file gets removed from the filesystem os.remove(temp.name) # else: # ims = imageseries.open(f, 'array') return ims
def image_series(self): """Return the imageseries dictionary.""" if not hasattr(self, '_image_dict'): self._image_dict = dict() fmt = self.get('image_series:format') imsdata = self.get('image_series:data') for ispec in imsdata: fname = self.check_filename(ispec['file'], self.working_dir) args = ispec['args'] ims = imageseries.open(fname, fmt, **args) oms = imageseries.omega.OmegaImageSeries(ims) try: panel = ispec['panel'] except (KeyError): panel = oms.metadata['panel'] self._image_dict[panel] = oms return self._image_dict
def image_series(self): """return the imageseries dictionary""" if not hasattr(self, '_image_dict'): self._image_dict = dict() fmt = self.get('image_series:format') imsdata = self.get('image_series:data') for ispec in imsdata: fname = ispec['file'] args = ispec['args'] ims = imageseries.open(fname, fmt, **args) oms = imageseries.omega.OmegaImageSeries(ims) try: panel=ispec['panel'] except KeyError: panel = ims.metadata['panel'] self._image_dict[panel] = ims return self._image_dict
def __init__(self, ims, fmt='hdf5', **kwargs): """Initialize frame readerOmegaFrameReader *ims* is either an imageseries instance or a filename *fmt* is the format to be passed to imageseries.open() *kwargs* is the option list to be passed to imageseries.open() NOTES: * The shape returned from imageseries is cast to int from numpy.uint64 to allow for addition of indices with regular ints """ if isinstance(ims, imageseries.imageseriesabc.ImageSeriesABC): self._imseries = ims else: self._imseries = imageseries.open(ims, fmt, **kwargs) self._nframes = len(self._imseries) self._shape = self._imseries.shape self._meta = self._imseries.metadata if self.OMEGA_TAG not in self._meta: raise RuntimeError('No omega data found in data file')
def __init__(self, ims, fmt='hdf5', **kwargs): """Initialize frame readerOmegaFrameReader *ims* is either an imageseries instance or a filename *fmt* is the format to be passed to imageseries.open() *kwargs* is the option list to be passed to imageseries.open() NOTES: * The shape returned from imageseries is cast to int from numpy.uint64 to allow for addition of indices with regular ints """ if isinstance(ims, imageseries.imageseriesabc.ImageSeriesABC): self._imseries = ims else: self._imseries = imageseries.open(ims, fmt, **kwargs) self._nframes = len(self._imseries) self._shape = self._imseries.shape self._meta = self._imseries.metadata if self.OMEGA_TAG not in self._meta: #raise ImageIOError('No omega data found in data file') pass
def test_process_dtype(self): a = make_array() is_a = imageseries.open(None, 'array', data=a) ops = [] is_p = process.ProcessedImageSeries(is_a, ops) self.assertEqual(is_p.dtype, is_p[0].dtype)
def make_array_ims(): is_a = imageseries.open(None, 'array', data=make_array(), meta=make_meta()) return is_a
#============================================================================== #%% LOOP WRITES OVER DETECTOR PANELS #============================================================================== for det_key in det_keys: image_list_str = '' for i in fn_dict[det_key]: image_list_str += image_stem %i + '\n' output_str = yml_tmplate % ( image_dir, image_list_str, det_key) rawfname = "raw_images_%s.yml" with open(rawfname %det_key, 'w') as f: print >> f, output_str # load basic imageseries: no flip, no omegas ims = imageseries.open(rawfname %det_key, 'image-files') if len(ims) != nf: import pbd; pdb.set_trace() # generate omegas w = imageseries.omega.OmegaWedges(nf) w.addwedge(*wedge_args) meta = ims.metadata meta['omega'] = w.omegas w.save_omegas('omegas_NF.npy') print ims.metadata # make dark print "making dark image" dark = imageseries.stats.median(ims, nframes=120) np.save('median_dark_%s.npy' %det_key, dark)
def load_images(yml): return imageseries.open(yml, "frame-cache")
tif_file_template = samp_name + '_%06d-%s.tif' raw_data_dir_template = os.path.join(image_dir, 'raw_images_%s_%06d-%s.yml') yml_string = """ image-files: directory: %s files: "%s" options: empty-frames: 0 max-frames: 0 meta: panel: %s """ ims = imageseries.open(os.path.join(image_dir, 'ceria_cal.h5'), 'hdf5', path='/imageseries') metadata = ims.metadata det_keys = np.array(metadata['panels'], dtype=str) for i, det_key in enumerate(det_keys): yml_file = open(raw_data_dir_template % (samp_name, scan_number, det_key), 'w') tiff_fname = tif_file_template % (scan_number, det_key) print(yml_string % (image_dir, tiff_fname, det_key), file=yml_file) io.imsave(os.path.join(image_dir, tiff_fname), ims[i]) pass
if grain_ids_arg.lower() in ['none', '']: grain_ids = None else: grain_ids = np.array(grain_ids_arg.split(','), dtype=int) if make_images: if ',' in imageseries_file: fnames = imageseries_file.split(',') else: fnames = glob.glob(imageseries_file) print "found the folowing files" ims_in = {} for fname in fnames: print "\t%s" % fname ims = imageseries.open(fname, 'frame-cache') m = ims.metadata ims_in[str(m['panel'])] = ims else: print "loading single multipanel imageseries %s" % imageseries_file make_images = False ims_in = load_images(imageseries_file) gplist = np.atleast_2d(gtable[:, 3:15]) iv = IView(instr, ims_in, pdata, gplist, grain_ids=grain_ids, make_images=make_images, ome_tol=ome_tol,
if make_max_frames: max_frames_output_name = os.path.join( data_dir, "%s_%d-maxframes.hdf5" % (samp_name, scan_number) ) if os.path.exists(max_frames_output_name): os.remove(max_frames_output_name) max_frames = dict.fromkeys(det_keys) for det_key in det_keys: max_frames[det_key] = imageseries.stats.max(imsd[det_key]) ims_out = imageseries.open( None, 'array', data=np.array([max_frames[i] for i in max_frames]), meta={'panels': max_frames.keys()} ) imageseries.write( ims_out, max_frames_output_name, 'hdf5', path='/imageseries' ) #%% class GenerateEtaOmeMaps(object): """ eta-ome map class derived from new image_series and YAML config ...for now... must provide:
PIS = imageseries.process.ProcessedImageSeries # load instrument instr = load_instrument(instrument_filename) det_keys = instr.detectors.keys() # grab imageseries filenames file_names = glob.glob(raw_data_dir_template % (expt_name, samp_name, scan_number)) check_files_exist = [os.path.exists(file_name) for file_name in file_names] if not np.all(check_files_exist): raise RuntimeError("files don't exist!") img_dict = dict.fromkeys(det_keys) for file_name in file_names: ims = imageseries.open(file_name, format='hdf5', path='/imageseries') this_key = file_name.split('/')[-1].split('_')[0].upper() img_dict[this_key] = PIS(ims, panel_opts[this_key])[0] # %% # ============================================================================= # INSTRUMENT # ============================================================================= plane_data = load_pdata(mat_filename, mat_key) instr = load_instrument(instrument_filename) # get powder line profiles # # output is as follows:
image_dir = os.path.join(os.getcwd(), 'imageseries') samp_name = "Ruby1_hydra" scan_number = 0 print("Making requested max frame...") max_frames_output_name = os.path.join( data_dir, "%s_%d-maxframes.hdf5" % (samp_name, scan_number) ) if os.path.exists(max_frames_output_name): os.remove(max_frames_output_name) max_frames = dict.fromkeys(cfg.instrument.hedm.detectors) for det_key in max_frames.iterkeys(): fc_file = os.path.join( image_dir, "%s_%06d-fc_%%s.npz" % (samp_name, scan_number)) ims = imageseries.open(fc_file % det_key, 'frame-cache') max_frames[det_key] = imageseries.stats.max(ims) ims_out = imageseries.open( None, 'array', data=np.array([max_frames[i] for i in max_frames]), meta={'panels': max_frames.keys()} ) imageseries.write( ims_out, max_frames_output_name, 'hdf5', path='/imageseries' )
def make_ims(nf, meta): a = np.zeros((nf, 2, 2)) ims = imageseries.open(None, 'array', data=a, meta=meta) return ims
#%% #instr_cfg_file = open('./dexela2_new.yml', 'r') #instr_cfg = yaml.load(instr_cfg_file) #instr = instrument.HEDMInstrument(instr_cfg) # #data_path = './' #img_series_root = 'ff2_00047' instr_cfg_file = open('./ge_detector_new.yml', 'r') instr_cfg = yaml.load(instr_cfg_file) instr = instrument.HEDMInstrument(instr_cfg) data_path = './' img_series_root = 'ge_scan_114' img_series = imageseries.open( os.path.join( img_series_root + '-fcache-dir', img_series_root + '-fcache.yml' ) , 'frame-cache') nframes = 240 average_frame = imageseries.stats.average(img_series) #%% wlen = constants.keVToAngstrom(instr_cfg['beam']['energy']) matl = make_matl('LSHR', 225, [3.5905,]) pd = matl.planeData pd.wavelength = instr_cfg['beam']['energy'] # takes keV pd.exclusions = np.zeros_like(pd.exclusions, dtype=bool)
def load_images(filename): return imageseries.open(filename, "hdf5", path='/imageseries')
def setUp(self): self._nfxy = (3, 6, 6) a = np.zeros(self._nfxy) ind = np.array([0, 1, 2]) a[ind, 1, 2] = 1 + ind self.is_a = imageseries.open(None, 'array', data=a)
# LOOP WRITES OVER DETECTOR PANELS # ============================================================================= for det_id in instr.detectors: suffix = det_id.lower() fill_tmpl = [raw_data_dir] \ + [suffix for i in range(n_wedges)] \ + [empty_frames, max_frames, det_id] # make yml string output_str = raw_img_tmplate % tuple(fill_tmpl) rawfname = "raw_images_%s-%s.yml" with open(rawfname % (output_stem, det_id), 'w') as f: print(output_str, file=f) # load basic imageseries: no flip, no omegas ims = imageseries.open(rawfname % (output_stem, det_id), 'image-files') # generate omegas nf = len(ims) w = imageseries.omega.OmegaWedges(nf) w.addwedge(start_ome, stop_ome, nf) meta = ims.metadata meta['omega'] = w.omegas w.save_omegas('omegas_FF.npy') print(ims.metadata) # handle dark if dark_file is None: print("making dark image") dark = imageseries.stats.median(ims, nframes=120) np.save('background_%s-%s.npy' % (output_stem, det_id), dark)
@author: bernier2 """ import os import numpy as np from hexrd import imageseries data_dir = os.getcwd() image_dir = os.path.join(os.getcwd(), 'imageseries') samp_name = "Ruby1_hydra" scan_number = 0 print("Making requested max frame...") max_frames_output_name = os.path.join( data_dir, "%s_%d-maxframes.hdf5" % (samp_name, scan_number)) if os.path.exists(max_frames_output_name): os.remove(max_frames_output_name) max_frames = dict.fromkeys(cfg.instrument.hedm.detectors) for det_key in max_frames.iterkeys(): fc_file = os.path.join(image_dir, "%s_%06d-fc_%%s.npz" % (samp_name, scan_number)) ims = imageseries.open(fc_file % det_key, 'frame-cache') max_frames[det_key] = imageseries.stats.max(ims) ims_out = imageseries.open(None, 'array', data=np.array([max_frames[i] for i in max_frames]), meta={'panels': max_frames.keys()}) imageseries.write(ims_out, max_frames_output_name, 'hdf5', path='/imageseries')
def load_images(yml): return imageseries.open(yml, "image-files")
def _load_images(self, ims): # load images from imageseries # ... add processing here print "loading images" if self._make_images: assert isinstance(ims, dict), \ "To make images, ims input must be a dictionary" print "making max frames to spec..." max_frames = [] for panel_id in self.panel_ids: panel = self.instr.detectors[panel_id] oims = Oimgs(ims[panel_id]) # now have OmegaImageSeries del_ome = oims.omega[0, 1] - oims.omega[0, 0] # degrees simd = panel.simulate_rotation_series( self.planeData, self.gpl, ome_ranges=self.ome_ranges, chi=self.chi, tVec_s=self.tvec) pred_omes = np.degrees(np.vstack(simd[2])[:, 2]) # in DEGREES ndiv, tol_grid = make_tolerance_grid(del_ome, self._ome_tol, 1, adjust_window=True) frame_indices = [] for ome in pred_omes: expanded_omes = ome + tol_grid fidxs = oims.omegarange_to_frames(expanded_omes[0], expanded_omes[-1]) if len(fidxs) > 0: frame_indices += fidxs if len(frame_indices) == 0: raise RuntimeError( "no omegas in speficied imageseries range(s)") max_frames.append( np.max(np.array([oims[k] for k in frame_indices]), axis=0)) pass # closes loop on panels # max array-based ims of max frames # NOTE: this assumes that the frames are all the same, which # is ok for NF detector at different L distances... ims = imageseries.open(None, 'array', data=np.array(max_frames), meta=dict(panels=self.panel_ids)) gid_str = '' for s in ['%s-' % i for i in self._grain_ids]: gid_str += s if save_max_frames: imageseries.write(ims, 'imageseries-max_grains_%s.h5' % gid_str[:-1], 'hdf5', path='data') pass # closes conditional on make_images m = ims.metadata panel_ids = m['panels'] d = dict(zip(panel_ids, range(len(panel_ids)))) if 'process' in m: pspec = m['process'] ops = [] for p in pspec: k = p.keys()[0] ops.append((k, p[k])) pims = Pimgs(ims, ops) else: pims = ims self.images = [] for panel_id in self.panel_ids: self.images.append(pims[d[panel_id]])
def load_images(yml): return imageseries.open(yml, format="frame-cache", style="npz")
#%% #instr_cfg_file = open('./dexela2_new.yml', 'r') #instr_cfg = yaml.load(instr_cfg_file) #instr = instrument.HEDMInstrument(instr_cfg) # #data_path = './' #img_series_root = 'ff2_00047' instr_cfg_file = open('./ge_detector_new.yml', 'r') instr_cfg = yaml.load(instr_cfg_file) instr = instrument.HEDMInstrument(instr_cfg) data_path = './' img_series_root = 'ge_scan_114' img_series = imageseries.open( os.path.join(img_series_root + '-fcache-dir', img_series_root + '-fcache.yml'), 'frame-cache') nframes = 240 average_frame = imageseries.stats.average(img_series) #%% wlen = constants.keVToAngstrom(instr_cfg['beam']['energy']) matl = make_matl('LSHR', 225, [ 3.5905, ]) pd = matl.planeData pd.wavelength = instr_cfg['beam']['energy'] # takes keV