Exemplo n.º 1
0
def doFolder_dataRed(azavStorage,
                     funcForAveraging=np.nanmean,
                     outStorageFile='auto',
                     reference='min',
                     chi2_0_max='auto',
                     saveTxt=True,
                     first=None,
                     last=None,
                     idx=None,
                     split_angle=False):
    """ azavStorage if a DataStorage instance or the filename to read 
  """

    if isinstance(azavStorage, DataStorage):
        azav = azavStorage
        folder = azavStorage.folder
    elif os.path.isfile(azavStorage):
        folder = os.path.dirname(azavStorage)
        azav = DataStorage(azavStorage)
    else:
        # assume is just a folder name
        folder = azavStorage
        azavStorage = folder + "/pyfai_1d" + default_extension
        azav = DataStorage(azavStorage)

    if split_angle:
        angles = np.unique(azav.log.angle)
        diffs = []
        for angle in angles:
            idx = azav.log.angle == angle
            diffs.append(
                doFolder_dataRed(azav,
                                 funcForAveraging=funcForAveraging,
                                 outStorageFile=None,
                                 reference=reference,
                                 chi2_0_max=chi2_0_max,
                                 saveTxt=False,
                                 idx=idx,
                                 split_angle=False))
        ret = DataStorage(angles=angles, diffs=diffs)
        if outStorageFile == 'auto':
            if not os.path.isdir(folder): folder = "./"
            outStorageFile = folder + "/diffs" + default_extension
        if outStorageFile is not None:
            ret.save(outStorageFile)
        return ret

    azav = copy.deepcopy(azav)

    if last is not None or first is not None and idx is None:
        idx = slice(first, last)

    if idx is not None:
        azav.log.delay = azav.log.delay[idx]
        azav.data_norm = azav.data_norm[idx]
        azav.err_norm = azav.err_norm[idx]

    # laser off is saved as -10s, if using the automatic "min"
    # preventing from using the off images
    # use reference=-10 if this is what you want
    if reference == "min":
        reference = azav.log.delay[azav.log.delay != -10].min()

    # calculate differences
    tr = dataReduction.calcTimeResolvedSignal(
        azav.log.delay,
        azav.data_norm,
        err=azav.err_norm,
        q=azav.q,
        reference=reference,
        funcForAveraging=funcForAveraging,
        chi2_0_max=chi2_0_max)

    tr.folder = folder
    tr.twotheta_rad = azav.twotheta_rad
    tr.twotheta_deg = azav.twotheta_deg
    tr.info = azav.pyfai_info

    if outStorageFile == 'auto':
        if not os.path.isdir(folder): folder = "./"
        outStorageFile = folder + "/diffs" + default_extension
    tr.filename = outStorageFile

    # save txt and npz file
    if saveTxt: dataReduction.saveTxt(folder, tr, info=azav.pyfai_info)

    if outStorageFile is not None:
        tr.save(outStorageFile)

    return tr
Exemplo n.º 2
0
def doFolder(folder="./",
             files='*.edf*',
             nQ=1500,
             force=False,
             mask=None,
             dark=10,
             qlims=None,
             monitor='auto',
             save_pyfai=False,
             saveChi=True,
             poni='pyfai.poni',
             storageFile='auto',
             save=True,
             logDict=None,
             dezinger=None,
             skip_first=0,
             last=None,
             azimuth_range=None):
    """ calculate 1D curves from files in folder

      Parameters
      ----------
      folder : str
          folder to work on
      files : str
          regular expression to look for ccd images (use edf* for including
          gzipped giles)
      nQ : int
          number of Q-points (equispaced)
      monitor : array or (qmin,qmax) or None
          normalization array (or list for q range normalization)
      force : True|False
          if True, redo from beginning even if previous data are found
          if False, do only new files
      mask : can be a list of [filenames|array of booleans|mask string]
          pixels that are True are dis-regarded
      saveChi : True|False
          if False, chi files (text based for each image) are not saved
      dezinger : None or 0<float<100
          use pyfai function 'separate' to remove zingers. The value is the 
          percentile used to find the liquicd baseline, 50 (i.e. median value)
          if a good approximation. Dezinger takes ~200ms per 4M pixel image.
          Needs good center and mask
      logDict : None or dictionary(-like)
          each key is a field. if given it has to have 'file' key
      poni : informationation necessary to build an AzimuthalIntegrator:
          → an AzimuthalIntegrator instance
          → a filename that will be look for in
               1 'folder' first
               2 in ../folder
               3 in ../../folder
               ....
               n-1 in pwd
               n   in homefolder
          → a dictionary (use to bootstrap an AzimuthalIntegrator using 
              AzimuthalIntegrator(**poni)
      save_pyfai : True|False
          if True, it stores all pyfai's internal arrays (~110 MB)
      skip_first : int
          skip the first images (the first one is sometime not ideal)
      last : int
          skip evey image after 'last'
 """

    func = inspect.currentframe()
    args = inspect.getargvalues(func)
    files_reg = files
    # store argument for saving ..
    args = dict([(arg, args.locals[arg]) for arg in args.args])

    folder = folder.replace("//", "/").rstrip("/")

    # can't store aritrary objects
    if isinstance(args['poni'], pyFAI.azimuthalIntegrator.AzimuthalIntegrator):
        args['poni'] = ai_as_dict(args['poni'])

    if storageFile == 'auto':
        fname = "pyfai_1d" + g_default_extension
        if not os.path.isdir(folder):
            # do not overide folder, it might be useful
            storageFile = os.path.join(".", fname)
        else:
            storageFile = os.path.join(folder, fname)

    if os.path.isfile(storageFile) and not force:
        saved = DataStorage(storageFile)
        log.info("Found %d images in storage file" % saved.data.shape[0])
        ai = getAI(poni, folder)
        # consistency check (saved images done with same parameters ?)
        if ai is not None:
            # pyfai cannot be compared (except for its string representation)
            # because before first image some fields are None
            keys_to_compare = "nQ mask dark dezinger skip_first last"
            keys_to_compare = keys_to_compare.split()
            # recursively transform in plain dict and limit comparison to given keys
            saved_args = DataStorage(saved.args).toDict()
            now_args = DataStorage(args).toDict()
            saved_args = dict([(k, saved_args[k]) for k in keys_to_compare])
            now_args = dict([(k, now_args[k]) for k in keys_to_compare])

            if (not compare_pyfai(saved.pyfai,ai)) or  \
                np.any( saved.mask != interpretMasks(mask,saved.mask.shape))  or \
                not utils.is_same(saved_args,now_args) :
                log.warn(
                    "Found inconsistency between curves already saved and new ones"
                )
                log.warn("Redoing saved ones with new parameters")
                if (saved.pyfai_info != ai_as_str(ai)):
                    log.warn("pyfai parameters changed from:\n%s" %
                             saved.pyfai_info + "\nto:\n%s" % ai_as_str(ai))
                if np.any(
                        saved.mask != interpretMasks(mask, saved.mask.shape)):
                    log.warn("Mask changed from:\n%s" % saved.mask +
                             "\nto:\n%s" %
                             interpretMasks(mask, saved.mask.shape))
                if not utils.is_same(saved_args, now_args):
                    for k in set(now_args.keys()) - set(['mask']):
                        if not utils.is_same(saved_args[k], now_args[k]):
                            if isinstance(saved_args[k], dict):
                                for kk in saved_args[k].keys():
                                    if not utils.is_same(
                                            saved_args[k][kk],
                                            now_args[k][kk]):
                                        log.warn(
                                            "Parameter %s.%s" % (k, kk) +
                                            "IS DIFFERENT", saved_args[k][kk],
                                            now_args[k][kk])
                            else:
                                log_str = " %s to %s" % (saved_args[k],
                                                         now_args[k])
                                if len(log_str) > 20:
                                    log_str = ":\n%s\nto:\n%s" % (
                                        saved_args[k], now_args[k])
                                log.warn("Parameter '%s' changed from" % k +
                                         log_str)
                args['force'] = True
                saved = doFolder(**args)
    else:
        saved = None

    files = utils.getFiles(folder, files)
    if logDict is not None:
        files = [f for f in files if utils.getBasename(f) in logDict['file']]

        # sometime one deletes images but not corresponding lines in logfiles...
        if len(files) < len(logDict['file']):
            basenames = np.asarray([utils.getBasename(file) for file in files])
            idx_to_keep = np.asarray([f in basenames for f in logDict['file']])
            for key in logDict.keys():
                logDict[key] = logDict[key][idx_to_keep]
            log.warn(
                "More files in log than actual images, truncating loginfo")

    files = files[skip_first:last]

    if saved is not None:
        files = [
            f for f in files if utils.getBasename(f) not in saved["files"]
        ]
    log.info("Will do azimuthal integration for %d files" % (len(files)))

    files = np.asarray(files)
    basenames = np.asarray([utils.getBasename(file) for file in files])

    if len(files) > 0:
        # which poni file to use:
        ai = getAI(poni, folder)
        _msg = "could not interpret poni info or find poni file"
        if ai is None: raise ValueError(_msg)

        shape = read(files[0]).shape
        mask = interpretMasks(mask, shape)

        data = np.empty((len(files), nQ))
        err = np.empty((len(files), nQ))
        pbar = utils.progressBar(len(files))
        for ifname, fname in enumerate(files):
            img = read(fname)
            q, i, e = do1d(ai,
                           img,
                           mask=mask,
                           npt_radial=nQ,
                           dark=dark,
                           dezinger=dezinger,
                           azimuth_range=azimuth_range)
            data[ifname] = i
            err[ifname] = e
            if saveChi:
                chi_fname = utils.removeExt(fname) + ".chi"
                utils.saveTxt(chi_fname,
                              q,
                              np.vstack((i, e)),
                              info=ai_as_str(ai),
                              overwrite=True)
            pbar.update(ifname + 1)
        pbar.finish()
        if saved is not None:
            files = np.concatenate((saved.orig.files, basenames))
            data = np.concatenate((saved.orig.data, data))
            err = np.concatenate((saved.orig.err, err))
        else:
            files = basenames
        twotheta_rad = utils.qToTwoTheta(q, wavelength=ai.wavelength * 1e10)
        twotheta_deg = utils.qToTwoTheta(q,
                                         wavelength=ai.wavelength * 1e10,
                                         asDeg=True)
        orig = dict(data=data.copy(),
                    err=err.copy(),
                    q=q.copy(),
                    twotheta_deg=twotheta_deg,
                    twotheta_rad=twotheta_rad,
                    files=files)
        ret = dict(folder=folder,
                   files=files,
                   orig=orig,
                   pyfai=ai_as_dict(ai),
                   pyfai_info=ai_as_str(ai),
                   mask=mask,
                   args=args)
        if not save_pyfai:
            ret['pyfai']['chia'] = None
            ret['pyfai']['dssa'] = None
            ret['pyfai']['qa'] = None
            ret['pyfai']['ttha'] = None

        ret = DataStorage(ret)

    else:
        ret = saved

    if ret is None: return None

    if qlims is not None:
        idx = (ret.orig.q >= qlims[0]) & (ret.orig.q <= qlims[1])
    else:
        idx = np.ones_like(ret.orig.q, dtype=bool)

    ret.orig.twotheta_deg = utils.qToTwoTheta(ret.orig.q,
                                              wavelength=ai.wavelength * 1e10,
                                              asDeg=True)
    ret.orig.twotheta_rad = utils.qToTwoTheta(ret.orig.q,
                                              wavelength=ai.wavelength * 1e10)

    ret.data = ret.orig.data[:, idx]
    ret.err = ret.orig.err[:, idx]
    ret.q = ret.orig.q[idx]

    ret.twotheta_rad = ret.orig.twotheta_rad[idx]
    ret.twotheta_deg = ret.orig.twotheta_deg[idx]

    if isinstance(monitor, str):
        if monitor == 'auto':
            monitor = ret.data.mean(1)
        else:
            raise ValueError(
                "'monitor' must be ndarray, 2-D tuple/list, 'auto' or None.")
    elif isinstance(monitor, (tuple, list)):
        if len(monitor) == 2:
            idx_norm = (ret.q >= monitor[0]) & (ret.q <= monitor[1])
            monitor = ret.data[:, idx_norm].mean(1)
        else:
            raise ValueError(
                "'monitor' must be ndarray, 2-D tuple/list, 'auto' or None.")
    elif not isinstance(monitor, np.ndarray) and monitor is not None:
        raise ValueError(
            "'monitor' must be ndarray, 2-D tuple/list, 'auto' or None.")

    if monitor is not None:
        ret["data_norm"] = ret.data / monitor[:, np.newaxis]
        ret["err_norm"] = ret.err / monitor[:, np.newaxis]
        ret["monitor"] = monitor[:, np.newaxis]
    else:
        ret["data_norm"] = None
        ret["err_norm"] = None
        ret["monitor"] = None

    # add info from logDict if provided
    if logDict is not None: ret['log'] = logDict
    # sometime saving is not necessary (if one has to do it after subtracting background
    if storageFile is not None and save: ret.save(storageFile)

    return ret