def doFolder_dataRed(azavStorage, funcForAveraging=np.nanmean, outStorageFile='auto', reference='min', chi2_0_max='auto', saveTxt=True, first=None, last=None, idx=None, split_angle=False): """ azavStorage if a DataStorage instance or the filename to read """ if isinstance(azavStorage, DataStorage): azav = azavStorage folder = azavStorage.folder elif os.path.isfile(azavStorage): folder = os.path.dirname(azavStorage) azav = DataStorage(azavStorage) else: # assume is just a folder name folder = azavStorage azavStorage = folder + "/pyfai_1d" + default_extension azav = DataStorage(azavStorage) if split_angle: angles = np.unique(azav.log.angle) diffs = [] for angle in angles: idx = azav.log.angle == angle diffs.append( doFolder_dataRed(azav, funcForAveraging=funcForAveraging, outStorageFile=None, reference=reference, chi2_0_max=chi2_0_max, saveTxt=False, idx=idx, split_angle=False)) ret = DataStorage(angles=angles, diffs=diffs) if outStorageFile == 'auto': if not os.path.isdir(folder): folder = "./" outStorageFile = folder + "/diffs" + default_extension if outStorageFile is not None: ret.save(outStorageFile) return ret azav = copy.deepcopy(azav) if last is not None or first is not None and idx is None: idx = slice(first, last) if idx is not None: azav.log.delay = azav.log.delay[idx] azav.data_norm = azav.data_norm[idx] azav.err_norm = azav.err_norm[idx] # laser off is saved as -10s, if using the automatic "min" # preventing from using the off images # use reference=-10 if this is what you want if reference == "min": reference = azav.log.delay[azav.log.delay != -10].min() # calculate differences tr = dataReduction.calcTimeResolvedSignal( azav.log.delay, azav.data_norm, err=azav.err_norm, q=azav.q, reference=reference, funcForAveraging=funcForAveraging, chi2_0_max=chi2_0_max) tr.folder = folder tr.twotheta_rad = azav.twotheta_rad tr.twotheta_deg = azav.twotheta_deg tr.info = azav.pyfai_info if outStorageFile == 'auto': if not os.path.isdir(folder): folder = "./" outStorageFile = folder + "/diffs" + default_extension tr.filename = outStorageFile # save txt and npz file if saveTxt: dataReduction.saveTxt(folder, tr, info=azav.pyfai_info) if outStorageFile is not None: tr.save(outStorageFile) return tr
def doFolder(folder="./", files='*.edf*', nQ=1500, force=False, mask=None, dark=10, qlims=None, monitor='auto', save_pyfai=False, saveChi=True, poni='pyfai.poni', storageFile='auto', save=True, logDict=None, dezinger=None, skip_first=0, last=None, azimuth_range=None): """ calculate 1D curves from files in folder Parameters ---------- folder : str folder to work on files : str regular expression to look for ccd images (use edf* for including gzipped giles) nQ : int number of Q-points (equispaced) monitor : array or (qmin,qmax) or None normalization array (or list for q range normalization) force : True|False if True, redo from beginning even if previous data are found if False, do only new files mask : can be a list of [filenames|array of booleans|mask string] pixels that are True are dis-regarded saveChi : True|False if False, chi files (text based for each image) are not saved dezinger : None or 0<float<100 use pyfai function 'separate' to remove zingers. The value is the percentile used to find the liquicd baseline, 50 (i.e. median value) if a good approximation. Dezinger takes ~200ms per 4M pixel image. Needs good center and mask logDict : None or dictionary(-like) each key is a field. if given it has to have 'file' key poni : informationation necessary to build an AzimuthalIntegrator: → an AzimuthalIntegrator instance → a filename that will be look for in 1 'folder' first 2 in ../folder 3 in ../../folder .... n-1 in pwd n in homefolder → a dictionary (use to bootstrap an AzimuthalIntegrator using AzimuthalIntegrator(**poni) save_pyfai : True|False if True, it stores all pyfai's internal arrays (~110 MB) skip_first : int skip the first images (the first one is sometime not ideal) last : int skip evey image after 'last' """ func = inspect.currentframe() args = inspect.getargvalues(func) files_reg = files # store argument for saving .. args = dict([(arg, args.locals[arg]) for arg in args.args]) folder = folder.replace("//", "/").rstrip("/") # can't store aritrary objects if isinstance(args['poni'], pyFAI.azimuthalIntegrator.AzimuthalIntegrator): args['poni'] = ai_as_dict(args['poni']) if storageFile == 'auto': fname = "pyfai_1d" + g_default_extension if not os.path.isdir(folder): # do not overide folder, it might be useful storageFile = os.path.join(".", fname) else: storageFile = os.path.join(folder, fname) if os.path.isfile(storageFile) and not force: saved = DataStorage(storageFile) log.info("Found %d images in storage file" % saved.data.shape[0]) ai = getAI(poni, folder) # consistency check (saved images done with same parameters ?) if ai is not None: # pyfai cannot be compared (except for its string representation) # because before first image some fields are None keys_to_compare = "nQ mask dark dezinger skip_first last" keys_to_compare = keys_to_compare.split() # recursively transform in plain dict and limit comparison to given keys saved_args = DataStorage(saved.args).toDict() now_args = DataStorage(args).toDict() saved_args = dict([(k, saved_args[k]) for k in keys_to_compare]) now_args = dict([(k, now_args[k]) for k in keys_to_compare]) if (not compare_pyfai(saved.pyfai,ai)) or \ np.any( saved.mask != interpretMasks(mask,saved.mask.shape)) or \ not utils.is_same(saved_args,now_args) : log.warn( "Found inconsistency between curves already saved and new ones" ) log.warn("Redoing saved ones with new parameters") if (saved.pyfai_info != ai_as_str(ai)): log.warn("pyfai parameters changed from:\n%s" % saved.pyfai_info + "\nto:\n%s" % ai_as_str(ai)) if np.any( saved.mask != interpretMasks(mask, saved.mask.shape)): log.warn("Mask changed from:\n%s" % saved.mask + "\nto:\n%s" % interpretMasks(mask, saved.mask.shape)) if not utils.is_same(saved_args, now_args): for k in set(now_args.keys()) - set(['mask']): if not utils.is_same(saved_args[k], now_args[k]): if isinstance(saved_args[k], dict): for kk in saved_args[k].keys(): if not utils.is_same( saved_args[k][kk], now_args[k][kk]): log.warn( "Parameter %s.%s" % (k, kk) + "IS DIFFERENT", saved_args[k][kk], now_args[k][kk]) else: log_str = " %s to %s" % (saved_args[k], now_args[k]) if len(log_str) > 20: log_str = ":\n%s\nto:\n%s" % ( saved_args[k], now_args[k]) log.warn("Parameter '%s' changed from" % k + log_str) args['force'] = True saved = doFolder(**args) else: saved = None files = utils.getFiles(folder, files) if logDict is not None: files = [f for f in files if utils.getBasename(f) in logDict['file']] # sometime one deletes images but not corresponding lines in logfiles... if len(files) < len(logDict['file']): basenames = np.asarray([utils.getBasename(file) for file in files]) idx_to_keep = np.asarray([f in basenames for f in logDict['file']]) for key in logDict.keys(): logDict[key] = logDict[key][idx_to_keep] log.warn( "More files in log than actual images, truncating loginfo") files = files[skip_first:last] if saved is not None: files = [ f for f in files if utils.getBasename(f) not in saved["files"] ] log.info("Will do azimuthal integration for %d files" % (len(files))) files = np.asarray(files) basenames = np.asarray([utils.getBasename(file) for file in files]) if len(files) > 0: # which poni file to use: ai = getAI(poni, folder) _msg = "could not interpret poni info or find poni file" if ai is None: raise ValueError(_msg) shape = read(files[0]).shape mask = interpretMasks(mask, shape) data = np.empty((len(files), nQ)) err = np.empty((len(files), nQ)) pbar = utils.progressBar(len(files)) for ifname, fname in enumerate(files): img = read(fname) q, i, e = do1d(ai, img, mask=mask, npt_radial=nQ, dark=dark, dezinger=dezinger, azimuth_range=azimuth_range) data[ifname] = i err[ifname] = e if saveChi: chi_fname = utils.removeExt(fname) + ".chi" utils.saveTxt(chi_fname, q, np.vstack((i, e)), info=ai_as_str(ai), overwrite=True) pbar.update(ifname + 1) pbar.finish() if saved is not None: files = np.concatenate((saved.orig.files, basenames)) data = np.concatenate((saved.orig.data, data)) err = np.concatenate((saved.orig.err, err)) else: files = basenames twotheta_rad = utils.qToTwoTheta(q, wavelength=ai.wavelength * 1e10) twotheta_deg = utils.qToTwoTheta(q, wavelength=ai.wavelength * 1e10, asDeg=True) orig = dict(data=data.copy(), err=err.copy(), q=q.copy(), twotheta_deg=twotheta_deg, twotheta_rad=twotheta_rad, files=files) ret = dict(folder=folder, files=files, orig=orig, pyfai=ai_as_dict(ai), pyfai_info=ai_as_str(ai), mask=mask, args=args) if not save_pyfai: ret['pyfai']['chia'] = None ret['pyfai']['dssa'] = None ret['pyfai']['qa'] = None ret['pyfai']['ttha'] = None ret = DataStorage(ret) else: ret = saved if ret is None: return None if qlims is not None: idx = (ret.orig.q >= qlims[0]) & (ret.orig.q <= qlims[1]) else: idx = np.ones_like(ret.orig.q, dtype=bool) ret.orig.twotheta_deg = utils.qToTwoTheta(ret.orig.q, wavelength=ai.wavelength * 1e10, asDeg=True) ret.orig.twotheta_rad = utils.qToTwoTheta(ret.orig.q, wavelength=ai.wavelength * 1e10) ret.data = ret.orig.data[:, idx] ret.err = ret.orig.err[:, idx] ret.q = ret.orig.q[idx] ret.twotheta_rad = ret.orig.twotheta_rad[idx] ret.twotheta_deg = ret.orig.twotheta_deg[idx] if isinstance(monitor, str): if monitor == 'auto': monitor = ret.data.mean(1) else: raise ValueError( "'monitor' must be ndarray, 2-D tuple/list, 'auto' or None.") elif isinstance(monitor, (tuple, list)): if len(monitor) == 2: idx_norm = (ret.q >= monitor[0]) & (ret.q <= monitor[1]) monitor = ret.data[:, idx_norm].mean(1) else: raise ValueError( "'monitor' must be ndarray, 2-D tuple/list, 'auto' or None.") elif not isinstance(monitor, np.ndarray) and monitor is not None: raise ValueError( "'monitor' must be ndarray, 2-D tuple/list, 'auto' or None.") if monitor is not None: ret["data_norm"] = ret.data / monitor[:, np.newaxis] ret["err_norm"] = ret.err / monitor[:, np.newaxis] ret["monitor"] = monitor[:, np.newaxis] else: ret["data_norm"] = None ret["err_norm"] = None ret["monitor"] = None # add info from logDict if provided if logDict is not None: ret['log'] = logDict # sometime saving is not necessary (if one has to do it after subtracting background if storageFile is not None and save: ret.save(storageFile) return ret