def test_pca_mssa_rec(self): data = setup_data2(nx=30, ny=20) span = Analyzer(data) rec = span.mssa_rec(nmssa=10) self.assertTrue(npy.allclose( rec[2,:2,:2].compressed(), npy.array([ 0.36712853, 0.28310489, 0.27881545])))
def test_mssa_eof(self): data = setup_data2(nx=3, ny=2) span = Analyzer(data) eof = span.mssa_eof(nmssa=4) self.assertTrue(npy.allclose( eof[1,1].compressed(), npy.array([ 0.10764546, -0.0763087 , -0.07211439, 0.05060925, 0.128224 ])))
def test_mssa_eof_restack(self): data = setup_data2(nx=3, ny=2) span = Analyzer(data) steof = span.mssa_eof(nmssa=4) stacked_eof = npy.ascontiguousarray(span.restack(steof, scale=False)) self.assertTrue(npy.allclose(steof[1,1].compressed(), stacked_eof[:,1,1])) raw_eof = span._mssa_raw_eof.reshape(stacked_eof.shape) self.assertTrue(npy.allclose(stacked_eof, raw_eof))
def test_pca_ev(self): A = Analyzer(setup_data1(nt=70, nx=150)) ev = A.pca_ev() self.assertTrue(npy.allclose(ev, npy.array( [ 24.98560392, 18.04025736, 3.59796042, 1.44110266, 0.88847967, 0.45467402, 0.24646941, 0.22392288, 0.16899249, 0.14435935] )))
def test_mssa_phases(self): nt = 100. nx = 5 data = npy.cos(npy.linspace(0, 5*6.2, nt*nx)*npy.pi*2).reshape((nx, nt)).T data += npy.cos(npy.linspace(0, 5*3.4, nt*nx)*npy.pi*2).reshape((nx, nt)).T span = Analyzer(data) rec = span.mssa_rec(imodes=[-1], phases=8) self.assertTrue(npy.allclose( rec[:2,:2].ravel(), npy.array([ 0.5525298 , 0.32848071, 0.15829024, 1.19338772])))
def test_pca_eof(self): A = Analyzer(setup_data1(nt=70, nx=150)) raw_eof = A.pca_eof(raw=True) np_eof, np_pc, np_ev = pca_numpy(A.stacked_data.T, A.npca) self.assertTrue(npy.allclose(npy.abs(raw_eof[:2,:2]), npy.abs(npy.array( [[ 0.15702435, 0.05325682], [ 0.14647224, 0.05082613]])))) eof = A.pca_eof() self.assertTrue(npy.allclose(npy.abs(eof[:2,:3].filled(0.)), npy.abs(npy.array( [[ 0.15702435, 0., 0.14647224], [ 0.05325682, 0., 0.05082613]]))))
def test_mssa_rec(self): data = setup_data2(nx=3, ny=2, xyfact=0) span = Analyzer(data) rec = span.mssa_rec(nmssa=20) print span.mssa_ev(cumsum=True, relative=True) # import pylab as P;P.subplot(211);P.pcolormesh(data.reshape(-1, 6));P.subplot(212);P.pcolormesh(rec.reshape(-1, 6));P.show() print rec[50].compressed() print data[50].compressed() print (rec-data).var()/data.var() self.assertTrue(npy.allclose( rec[50].compressed(), npy.array([0.30848007, 0.8622368, 0.96028445, 0.8236466, 0.56329503 ])))
def test_ssa(self): data = setup_data0() span = Analyzer(data) span.mssa(nmssa=4) self.assertTrue(npy.allclose( span._mssa_raw_ev[0], npy.array([ 20.25648796, 16.57547325, 16.51572589, 16.31088181]))) self.assertTrue(npy.allclose( span._mssa_raw_pc[0][:2,:2], npy.array([[ 6.9105607 , 0.32054539], [ 5.77266597, 2.92740569]]))) self.assertTrue(npy.allclose( span._mssa_raw_eof[0][:2,:2], npy.array([[ 0.13637961, -0.12778613], [ 0.13417614, 0.13187513]])))
def test_mssa(self): data = setup_data2(nx=3, ny=2) span = Analyzer(data) span.mssa(nmssa=4) print data.reshape(120, 6).mask # self.assertTrue(npy.allclose( # span._mssa_raw_ev, # npy.array([ 84.65284564, 39.94531834, 36.70936929, 21.49023578]))) print span._mssa_raw_pc[:2,:2] self.assertTrue(npy.allclose( span._mssa_raw_pc[:2,:2], npy.array([[-13.66377591, 2.62887011], [-13.28622293, 2.74443558]]))) self.assertTrue(npy.allclose( span._mssa_raw_eof[:2,:2], npy.array([[ 0.05067246, 0.10596479], [ 0.05300994, 0.10764546]])))
def test_pca_mssa_ec(self): data = setup_data2(nx=30, ny=20) span = Analyzer(data) span.mssa(nmssa=4) ec = span.mssa_ec(xeof=span.mssa_eof(), xdata=data) pc = span.mssa_pc() self.assertTrue(npy.allclose(ec, pc))
def test_pca(self): # spanlib A = Analyzer(setup_data1(nt=70, nx=50)) sp_eof = A.pca_eof(raw=True) sp_pc = A.pca_pc(raw=True) sp_ev = A.pca_ev(raw=True) # numpy np_eof, np_pc, np_ev = pca_numpy(A.stacked_data.T, A.npca) # checks self.assertTrue(npy.allclose(sp_ev, np_ev)) signs = npy.sign(sp_eof[0])*npy.sign(np_eof[0]) np_eof *= npy.resize(signs, np_eof.shape) np_pc *= npy.resize(signs, np_pc.shape) self.assertTrue(npy.allclose(sp_eof, np_eof)) self.assertTrue(npy.allclose(sp_pc, np_pc)) self.assertTrue(npy.allclose(sp_ev, np_ev))
def test_pca_rec(self): """Test a PCA analysis and reconstruction with a gap inside""" # Init data = gensin2d(xnper=5, xper=30, ynper=5, yper=20) data[0:30, 0:55] = npy.ma.masked # Fill A = Analyzer(data, npca=10) rec = A.pca_rec() # Check # import pylab as P # P.subplot(211) # P.pcolor(data, vmin=data.min(), vmax=data.max()) # P.subplot(212) # P.pcolor(rec, vmin=data.min(), vmax=data.max()) # P.show() self.assertAlmostEqual(((data-rec)**2).sum()/(data**2).sum()*100, 0.064630381956367611)
def __init__(self, ldataset, rdataset, lweights=None, rweights=None, lnorms=None, rnorms=None, lminvalid=None, rminvalid=None, logger=None, loglevel=None, zerofill=0, **kwargs): # Loggers Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_')) self._quiet = False # Left and right Analyzer instances if zerofill == 2: kwargs['zerofill'] = 2 kwleft, kwright = self._dict_filter_lr_(kwargs) kwargs['logger'] = self.logger self.lspan = Analyzer(ldataset, weights=lweights, norms=lnorms, minvalid=lminvalid, **kwargs) self.rspan = Analyzer(rdataset, weights=rweights, norms=rnorms, minvalid=rminvalid, **kwargs) # Init results self.clean(pca=False) # Check and save parameters self.update_params(None, **kwargs)
def analyze(self, data=None, npca=15, nmssa=15, **kwargs): # Setup analyzer if data is None: data = self._data kwargs['keep_invalids'] = True kwargs.setdefault('nvalid', 1) kwargs.setdefault('quiet', True) npca = kwargs.pop('npcamax', npca) nmssa = kwargs.pop('nmssamax', nmssa) kwargs.setdefault('window', kwargs.pop('win', None)) span = self.span = Analyzer(data, logger=self.logger, **kwargs) span.update_params() self.mask = span.invalids return span
def __init__(self, ldataset, rdataset, lweights=None, rweights=None, lnorms=None, rnorms=None, lminvalid=None, rminvalid=None, logger=None, loglevel=None, zerofill=0, **kwargs): # Loggers Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_')) self._quiet=False # Left and right Analyzer instances if zerofill==2: kwargs['zerofill'] = 2 kwleft, kwright = self._dict_filter_lr_(kwargs) kwargs['logger'] = self.logger self.lspan = Analyzer(ldataset, weights=lweights, norms=lnorms, minvalid=lminvalid, **kwargs) self.rspan = Analyzer(rdataset, weights=rweights, norms=rnorms, minvalid=rminvalid, **kwargs) # Init results self.clean(pca=False) # Check and save parameters self.update_params(None, **kwargs)
def test_pca_xrec(self): var = setup_data1() A = Analyzer(var) rec = A.pca_rec() xeof = A.pca_eof() xpc = A.pca_pc() xrec = A.pca_rec(xeof=xeof, xpc=xpc) self.assertTrue(npy.allclose(rec, xrec))
def test_pca_mssa_xrec(self): data = setup_data2(nx=30, ny=20) span = Analyzer(data) rec = span.mssa_rec(nmssa=4) xeof = span.mssa_eof() xpc = span.mssa_pc() xrec = span.mssa_rec(xeof=xeof, xpc=xpc) self.assertTrue(npy.allclose(rec, xrec))
def test_ssa_xrec(self): data = setup_data0() span = Analyzer(data, nmssa=6) rec = span.mssa_rec() eof = span.mssa_eof() pc = span.mssa_pc() xrec = span.mssa_rec(xeof=eof, xpc=pc) self.assertTrue(npy.allclose(rec,xrec))
def test_pca_mssa(self): data = setup_data2(nx=30, ny=20) span = Analyzer(data, nmssa=2) self.assertTrue(npy.allclose( span.mssa_ev(), npy.array([ 2482.65882191, 2355.53413819]))) pc = span.mssa_pc() self.assertTrue(npy.allclose( span.mssa_pc()[:,:2].ravel(), npy.array([-19.47701848, -20.97653195, 2.99480916, -0.20444392]))) self.assertAlmostEqual(span.mssa_eof()[1,2,3,4], 0.0041007559488027926)
def clean(self): Analyzer.clean(self) self._regre_ = None gc.collect()
class DualAnalyzer(_BasicAnalyzer_, Logger): _nsvd_max = 100 _nsvd_default = 10 _svd_params = ['nsvd'] _params = dict(svd=_svd_params) _all_params = _svd_params _int_params = ['nsvd'] def __init__(self, ldataset, rdataset, lweights=None, rweights=None, lnorms=None, rnorms=None, lminvalid=None, rminvalid=None, logger=None, loglevel=None, zerofill=0, **kwargs): # Loggers Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_')) self._quiet = False # Left and right Analyzer instances if zerofill == 2: kwargs['zerofill'] = 2 kwleft, kwright = self._dict_filter_lr_(kwargs) kwargs['logger'] = self.logger self.lspan = Analyzer(ldataset, weights=lweights, norms=lnorms, minvalid=lminvalid, **kwargs) self.rspan = Analyzer(rdataset, weights=rweights, norms=rnorms, minvalid=rminvalid, **kwargs) # Init results self.clean(pca=False) # Check and save parameters self.update_params(None, **kwargs) def __getitem__(self, key): return (self.lspan, self.rspan)[key] def __len__(self): return 2 @staticmethod def _dict_filter_lr_(kwargs): return dict_filter(kwargs, 'l'), dict_filter(kwargs, 'r') def update_params(self, anatype=None, **kwargs): """Initialize, update and check statistical paremeters. A value of ``None`` is converted to an optimal value. Analyses are re-ran if needed by checking dependencies. :Params: - **anatype**, optional: current analysis type. - ``None``: Simple initialization with ``None`` - ``"svd"``: Check SVD parameters. If different from ``None``, analysis may be ran again if parameters are changed. - Other keywords are interpreted as analysis parameters. :Output: A dictionary of (param name, change status) items. """ # Update left and right params first kwleft, kwright = self._dict_filter_lr_(kwargs) reran = (self.lspan.update_params(**kwleft), self.rspan.update_params(**kwright)) # Initialize old values and defaults changed to False old = {} for param in self._all_params: old[param] = getattr(self, '_' + param, None) setattr(self, '_' + param, kwargs.pop(param, old[param])) # Number of SVD modes if self._nsvd is None: # Initialization self._nsvd = self._nsvd_default # Default value for iset in range(2): # Check values if self[iset].prepca: nchanmax = self[iset].prepca # Input channels are from pre-PCA else: nchanmax = self[iset].ns # Input channels are from real space self._nsvd = npy.clip(self._nsvd, 1, min(self._nsvd_max, nchanmax)) # Max # Re-run analyses when needed rerun = dict( svd = self._has_run_('svd') and (self._has_changed_(old, 'nsvd')>0 or \ (self[0].prepca and reran[0]['pca']) or \ (self[1].prepca and reran[1]['pca'])), ) # - SVD if rerun['svd']: self.debug('Re-running SVD because some parameters changed') self.svd() # Inform what has reran return rerun def _has_run_(self, anatype=None, iset=0): """Check if an analysis has already run""" if anatype is None: return False getattr(self, '_%s_raw_eof' % anatype) is None def _svd_channel_axis_(self, iset): """Get the SVD channel axis for one dataset""" return _channel_axis_(iset, 'svd', svd=True) def get_nsvd(self): """Get :attr:`nsvd` :Returns: integer """ return self._nsvd def set_nsvd(self, nsvd): """Set :attr:`nsvd` and update analyzes""" self.update(nsvd=nsvd) nsvd = property(get_nsvd, set_nsvd, doc="Number of SVD modes") def has_cdat(self): return self[0].has_cdat() or self[1].has_cdat() def clean(self, svd=True, pca=True): """(Re-)Initialization""" anatypes = [] self.lspan.clean(pca=pca) self.rspan.clean(pca=pca) if pca: svd = True if svd: anatypes.append('svd') # Register what to clean nones = [] for aa in anatypes: nones.append('_%s_ev_sum' % aa) for bb in 'raw', 'fmt': for cc in 'eof', 'pc', 'ev': nones.append('_%s_%s_%s' % (aa, bb, cc)) if svd: nones.extend(['_svd_channel_axes']) # lists = ['_mssa_pairs','_nt','_ns','_ndata','_pdata'] dicts = ['_mode_axes'] # Clean now using values or functions for ll, init in [(nones, None), (dicts, dict)]: #,(lists,list): for att in ll: self._cleanattr_(att, init) # Integers self._last_anatype = None gc.collect() def _dual_(self, data): """Makes sure to have 2 elements""" if not isinstance(data, (list, tuple)): data = [data] if len(data) != 2: data = broadcast(data, 2) return data def _isets_(self, iset): """Dataset indices""" if iset is None: return [0, 1] if iset == 0 or str(iset).lower() == 'left': return [0] if iset == 1 or str(iset).lower() == 'right': return [1] raise DualSpanLibError( "Wrong dataset choice. Choose one of None, 0/'left' or 1/'right'") ################################################################# ## SVD ################################################################# @_filldocs_ def svd(self, usecorr=False, largematrix=False, **kwargs): """ Singular Value Decomposition (SVD) It is called everytime needed by :meth:`svd_eof`, :meth:`svd_pc`, :meth:`svd_ev` and :meth:`svd_rec`. Thus, since results are stored in cache, it not necessary call it explicitly. :Parameters: %(nsvd)s %(prepca)s - *usecorr*: bool Use correlations instead of covariances. """ # Parameters self.update_params('svd', **kwargs) # Loop on first two datasets (left and right) for iset in range(2): # Check if old results can be used when nsvd is lower if self._svd_raw_pc is not None and \ self._svd_raw_pc[iset].shape[-1] > self._nsvd: break # Remove old results for att in 'raw_eof', 'raw_pc', 'raw_ev', 'ev_sum': dic = getattr(self, '_svd_' + att) if isinstance(dic, dict) and iset in dic: del dic[iset] # Prepare input to SVD raw_input = self[iset].preproc_raw_output() if iset == 0: left = raw_input # lweights = raw_input[:, 0]*0.+1 else: right = raw_input # rweights = raw_input[:, 0]*0.+1 # if self._prepca[iset]: # Pre-PCA case # # # Pre-PCA # if not self._pca_raw_pc.has_key(iset): self.pca(iset=iset) # # # svd # data = self._pca_raw_pc[iset][:, :self._prepca[iset]].transpose() # if iset == 0: # left = data # lweights = data[:, 0]*0.+1 # else: # right = data # rweights = data[:, 0]*0.+1 # # else: # Direct svd case # weights = self[iset].stacked_weights # pdata = self[iset].stacked_data # if iset == 0: # left = pdata # lweights = weights # # else: # right = pdata # rweights = weights # Compute SVD left = npy.asfortranarray(left, 'd') right = npy.asfortranarray(right, 'd') raw_eof_left, raw_eof_right, raw_pc_left, raw_pc_right, raw_ev, ev_sum, errmsg = \ _core.svd(left, right, self._nsvd, int(usecorr), default_missing_value) self.check_fortran_errmsg(errmsg) # Save results self._svd_raw_pc = [ npy.ascontiguousarray(raw_pc_left), npy.ascontiguousarray(raw_pc_right) ] self._svd_raw_eof = [ npy.ascontiguousarray(raw_eof_left), npy.ascontiguousarray(raw_eof_right) ] self._svd_raw_ev = raw_ev self._svd_ev_sum = ev_sum self._last_anatype = 'svd' gc.collect() @_filldocs_ def svd_eof(self, scale=False, raw=False, unmap=True, format=True, **kwargs): """Get EOFs from SVD analysis If SVD was not performed, it is done with all parameters sent to :meth:`svd` :Parameters: %(scale)s %(raw)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Arrays with shape ``(nsvd,...)`` """ # Update params self.update_params('svd', **kwargs) # No analyses performed? if not self._has_run_('svd'): self.svd() # Of, let's format the variable fmt_eof = [] for iset in range(2): # (window*nchan,nsvd) # # EOF already available # if not raw and self._svd_fmt_eof is not None: # fmt_eof[iset] = self._svd_fmt_eof[iset] # continue # Raw data raw_eof = self._svd_raw_eof[iset][:, :self._nsvd] nl, nm = raw_eof.shape if raw: # Do not go back to physical space fmt_eof.append([npy.ascontiguousarray(raw_eof.T)]) if format and self[iset].has_cdat(): fmt_eof[iset][0] = cdms2.createVariable(fmt_eof[iset]) fmt_eof[iset][0].setAxisList([ self._mode_axis_('svd'), self._svd_channel_axis_(iset) ]) else: # Get raw data back to physical space raw_eof = npy.ascontiguousarray(raw_eof) raw_eof.shape = (nl, nm) firstaxes = self._mode_axis_('svd') if not self[iset].prepca: # No pre-PCA performed fmt_eof.append(self[iset].unstack(raw_eof, rescale=False, format=1, firstaxes=firstaxes)) else: proj_eof, smodes = self[iset]._raw_rec_( self[iset]._pca_raw_eof, raw_eof.T) fmt_eof.append(self[iset].unstack(proj_eof, rescale=False, format=1, firstaxes=firstaxes)) # Set attributes for idata, eof in enumerate(fmt_eof[iset]): # Attributes if format and cdms2_isVariable(eof): if not raw and not self[iset][idata].id.find('variable_'): eof.id = self[iset][idata].id + '_svd_eof' else: eof.id = 'svd_eof' eof.name = eof.id eof.standard_name = 'empirical_orthogonal_functions_of_svd' eof.long_name = 'SVD empirical orthogonal functions' if not raw: atts = self[iset][idata].atts if atts.has_key('long_name'): eof.long_name += ' of ' + atts['long_name'] if scale is True and atts.has_key('units'): eof.units = atts['units'] # Scaling if scale: if scale is True: scale = npy.sqrt(self._svd_raw_ev[iset]) * nl eof[:] *= scale if raw: unmap = False if not unmap: return fmt_eof return self.unmap(fmt_eof) @_filldocs_ def svd_pc(self, raw=False, unmap=True, format=True, **kwargs): """Get PCs from SVD analysis If SVD was not performed, it is done with all parameters sent to :meth:`svd` :Parameters: %(raw)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Arrays with shape ``(nsvd,nt)`` """ # Update params self.update_params('svd', **kwargs) # No analyses performed? if not self._svd_raw_pc: self.svd() # Check cache # Of, let's format the variable fmt_pc = [] for iset in range(2): # Raw? raw_pc = self._svd_raw_pc[iset][:, :self._nsvd] if raw: fmt_pc.append(raw_pc) continue # Format the variable idata = 0 # Reference is first data pc = npy.ascontiguousarray(raw_pc).T if self[iset].has_cdat(): pc = cdms2.createVariable(pc) pc.setAxis(0, self._mode_axis_('svd')) pc.setAxis(1, self[iset].get_time(idata=idata)) pc.id = pc.name = 'svd_pc' # pc.standard_name = 'principal_components_of_svd' pc.long_name = 'SVD principal components' atts = self[iset][idata].atts if atts.has_key('long_name'): pc.long_name += ' of ' + atts['long_name'] # if atts.has_key('units'): pc.units = atts['units'] fmt_pc.append(pc) # self._check_dataset_tag_('_svd_fmt_pc', iset, svd=True) # self._svd_fmt_pc = fmt_pc return fmt_pc # if not unmap: return fmt_pc # return self.unmap(fmt_pc) def svd_ec(self, iset=None, xdata=None, xeof=None, xraw=False, raw=False, unmap=True, format=True, **kwargs): """Get expansion coefficients from SVD analysis :Parameters: %(raw)s :SVD parameters: %(nsvd)s %(window)s %(prepca)s :Returns: Arrays with the shape ``(nsvd,nt)`` """ # Update params self.update_params('svd', **kwargs) # Dual input isets = self._isets_(iset) if len(isets) > 1: xeof = self._dual_(xeof) xdata = self._dual_(xdata) # Loop on left and right fmt_ec = [] for iset in isets: # SVD EOFs used for projection if xeof[iset] is None: if self._has_run_('svd'): self.svd() raw_eof = self[iset]._svd_raw_eof elif xraw: raw_eof = xeof[iset] elif self[iset].prepca: # After PCA raw_eof = self[iset].pca_ec(xdata=xeof, raw=True, xscale=False, unmap=False)[iset].T raw_eof.shape = -1, self._nsvd else: eofs = self[iset].remap(xeof[iset]) raw_eof = npy.ascontiguousarray(self[iset].restack( eofs, scale=False)) del eofs # Data to project on SVD EOFs if xdata[iset] is None: # From input raw_data = self[i].preproc_raw_output() # if not self[iset]._prepca: # Input data # raw_data = self[iset].stacked_data # else: # After PCA # raw_data = self._pca_raw_pc[iset].T elif xraw: # Direct use raw_data = xdata[iset] elif self[iset].prepca: # After PCA raw_data = self[iset].pca_ec(xdata=xdata, raw=True, unmap=False)[iset].T else: data = self[iset].remap(xdata[iset]) raw_data = self[iset].restack(data, scale=True) # Projection raw_data = npy.asfortranarray(raw_data) raw_eof = npy.asfortranarray(raw_eof[:, :self._nsvd]) raw_weights = npy.ones(raw_data.shape[0]) raw_ec = _core.pca_getec(raw_data, raw_eof, raw_weights, default_missing_value) if raw: fmt_ec[iset] = raw_ec continue ec = npy.ascontiguousarray(raw_ec.T) ec = npy.ma.masked_values(ec, default_missing_value, copy=False) # Format the variable if format and self[iset].has_cdat(): ec = cdms2.createVariable(ec) ec.setAxis(0, self._mode_axis_('svd')) # ec.setAxis(1,self._svd_pctime_axis_(iset)) ec.id = ec.name = 'svd_ec' # ec.standard_name = 'expansion coefficient_of_svd of ' ec.long_name = 'SVD principal components' atts = self[iset][0].atts if len(self[iset]) == 1 and atts.has_key('long_name'): ec.long_name += atts['long_name'] # else: # ec.long_name += 'of dataset %i'%iset #if (len(self[iset])==1 or npy.allclose(self.norms(iset), 1.)) and atts.has_key('units'): # ec.units = atts['units'] fmt_ec[iset] = ec # FIXME: _check_dataset_tag_ # self._check_dataset_tag_('_svd_fmt_ec', iset) if len(isets) == 1: fmt_ec = fmt_ec[0] return fmt_ec # if not unmap: return fmt_ec # return self.unmap(fmt_ec) @_filldocs_ def svd_ev(self, relative=False, sum=False, cumsum=False, **kwargs): """Get eigen values from SVD analysis :Parameters: %(relative)s %(sum)s %(cumsum)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Array with shape ``(nsvd,)`` or a float """ # Update params self.update_params('svd', **kwargs) # No analyses performed? if self._svd_raw_eof is None: self.svd() # We only want the sum if sum: return self._svd_ev_sum # Format the variable id = 'svd_ev' long_name = [] ev = self._svd_raw_ev[:self._nsvd] if cumsum: ev = ev.cumsum() id += '_cumsum' long_name.append('cumulative') if relative: ev = 100. * ev / self._svd_ev_sum id += '_rel' long_name.append('relative') # Format the variables if self.has_cdat(): ev = cdms2.createVariable(ev) ev.id = ev.name = id long_name = ['SVD eigen values'] ev.long_name = ' '.join(long_name).title() ev.setAxis(0, self._mode_axis_('svd')) # ev.standard_name = 'eigen_values_of_svd' if cumsum: ev.id += '_cumsum' long_name.append('cumulative') if relative: ev.id += '_rel' ev.units = '% of total variance' long_name.append('cumulative') ev.long_name = ' '.join(long_name).title() return ev @_filldocs_ def svd_rec(self, modes=None, raw=False, xpc=None, xeof=None, xraw=False, rescale=True, unmap=True, format=2, **kwargs): """Reconstruction of SVD modes :Parameters: %(modes)s %(raw)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Arrays with the same shape as input arrays. """ # Update params self.update_params('svd', **kwargs) # Dual input xeof = self._dual_(xeof) xpc = self._dual_(xpc) # Loop on left and right fmt_rec = [] for iset in range(2): # ST-EOFs used for projection if xeof[iset] is None: if self._has_run_('svd'): self.svd() raw_eof = self._svd_raw_eof[iset] elif xraw: raw_eof = xeof[iset] elif self[iset].prepca: # After PCA raw_eof = self[iset].pca_ec(xdata=xeof, raw=True, xscale=False, unmap=False)[iset].T else: eofs = self[iset].remap(xeof[iset]) raw_eof = npy.ascontiguousarray(self[iset].restack( eofs, scale=False)) del eofs raw_eof = raw_eof[:, :self._nsvd] # PCs if xpc[iset] is None: # From PCA if not self._has_run_('svd'): self.svd() raw_pc = self._svd_raw_pc[iset] elif xraw: # Direct use raw_pc = xpc[iset] elif npy.ma.isMA(xpc[iset]): # formatted -> pure numpy raw_pc = xpc[iset].filled(default_missing_value).T else: raw_pc = xpc[iset].T raw_pc = raw_pc[:, :self._nsvd] # Projection raw_rec, smodes = self[0]._raw_rec_(raw_eof, raw_pc, modes) # Get raw data back to physical space (nchan,nt) taxis = self[iset].get_time() if not self[iset].prepca: # No pre-PCA performed fmt_rec.append(self[iset].unstack(raw_rec, rescale=rescale, format=format)) elif raw: # Force direct result from svd if format and self[iset].has_cdat(): fmt_rec.append([cdms2.createVariable(raw_rec.T)]) fmt_rec[iset][0].setAxisList( 0, [taxis, self._mode_axis_('svd')]) else: fmt_rec[iset] = raw_rec.T else: # With pre-pca proj_rec, spcamodes = self[iset]._raw_rec_( self[iset]._pca_raw_eof, raw_rec.T) fmt_rec.append(self[iset].unstack(proj_rec, rescale=rescale, format=format)) del raw_rec # Set attributes if format: for idata, rec in enumerate(fmt_rec[iset]): if not cdms2_isVariable(rec): continue if not self[iset][idata].id.startswith('variable_'): rec.id = self[iset][idata].id + '_svd_rec' else: rec.id = 'svd_rec' # if modes is not None: rec.id += smodes #FIXME: do we keep it? rec.modes = smodes # else: # rec.modes = '1-%i'%self._nsvd[iset] rec.standard_name = 'recontruction_of_svd_modes' rec.long_name = 'Reconstruction of SVD modes' atts = self[iset][idata].atts if atts.has_key('long_name'): rec.long_name += ' of ' + atts['long_name'] if raw: unmap is False if not unmap: return fmt_rec return self.unmap(fmt_rec) def unmap(self, data, **kwargs): return self[0].unmap(data[0], **kwargs), self[1].unmap(data[1], **kwargs)
def clean(self): Analyzer.clean(self) self._regre_ = None gc.collect()
def setup_Analyzer3(self): var1 = setup_data1() var2 = setup_data1(nx=45, tfreq=4, xfreq=5) var3 = setup_data1(nt=120, nx=20) return Analyzer([[var1, var2], var3])
def test_pca_ndim3(self): A = Analyzer(setup_data2()) A.pca() self.assertTrue(npy.allclose( npy.abs(A.pca_eof(raw=True)[190:193,5]), npy.abs(npy.array([-0.03623882, -0.02312414, -0.00942417]))))
def test_pca_pc(self): A = Analyzer(setup_data1(nt=70, nx=150)) pc = A.pca_pc() self.assertTrue(npy.allclose(pc[:2,:2], npy.array( [[ 2.75532971, 2.96740389], [-8.78119373, -8.49616261]])))
def test_ssa_rec(self): data = setup_data0() span = Analyzer(data) rec = span.mssa_rec(nmssa=6) self.assertTrue(npy.allclose(rec[:4], npy.array([ 2.14699609, 1.3861899 , 2.70336958, 0.70994501])))
def test_pca_ec(self): data = setup_data1(nt=70, nx=150) A = Analyzer(data) A.pca_eof() ec = A.pca_ec() pc = A.pca_pc() self.assertTrue(npy.allclose(ec, pc)) ece = A.pca_ec(xeof=A.pca_eof(raw=True), xraw=True) self.assertTrue(npy.allclose(ece, pc)) ecd = A.pca_ec(xdata=data) self.assertTrue(npy.allclose(ecd, pc)) ecrd = A.pca_ec(xdata=A.stacked_data, xraw=True) self.assertTrue(npy.allclose(ecrd, pc)) ecre = A.pca_ec(xeof=A.pca_eof(raw=True), xraw=True) self.assertTrue(npy.allclose(ecrd, pc))
def test_pca_rec(self): A = Analyzer(setup_data1()) rec1 = A.pca_rec() self.assertAlmostEqual(rec1.sum(), 18464.31988812385)
class DualAnalyzer(_BasicAnalyzer_, Logger): _nsvd_max = 100 _nsvd_default = 10 _svd_params = ['nsvd'] _params = dict(svd=_svd_params) _all_params = _svd_params _int_params = ['nsvd'] def __init__(self, ldataset, rdataset, lweights=None, rweights=None, lnorms=None, rnorms=None, lminvalid=None, rminvalid=None, logger=None, loglevel=None, zerofill=0, **kwargs): # Loggers Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_')) self._quiet=False # Left and right Analyzer instances if zerofill==2: kwargs['zerofill'] = 2 kwleft, kwright = self._dict_filter_lr_(kwargs) kwargs['logger'] = self.logger self.lspan = Analyzer(ldataset, weights=lweights, norms=lnorms, minvalid=lminvalid, **kwargs) self.rspan = Analyzer(rdataset, weights=rweights, norms=rnorms, minvalid=rminvalid, **kwargs) # Init results self.clean(pca=False) # Check and save parameters self.update_params(None, **kwargs) def __getitem__(self, key): return (self.lspan, self.rspan)[key] def __len__(self): return 2 @staticmethod def _dict_filter_lr_(kwargs): return dict_filter(kwargs, 'l'), dict_filter(kwargs, 'r') def update_params(self, anatype=None, **kwargs): """Initialize, update and check statistical paremeters. A value of ``None`` is converted to an optimal value. Analyses are re-ran if needed by checking dependencies. :Params: - **anatype**, optional: current analysis type. - ``None``: Simple initialization with ``None`` - ``"svd"``: Check SVD parameters. If different from ``None``, analysis may be ran again if parameters are changed. - Other keywords are interpreted as analysis parameters. :Output: A dictionary of (param name, change status) items. """ # Update left and right params first kwleft, kwright = self._dict_filter_lr_(kwargs) reran = (self.lspan.update_params(**kwleft), self.rspan.update_params(**kwright)) # Initialize old values and defaults changed to False old = {} for param in self._all_params: old[param] = getattr(self,'_'+param, None) setattr(self, '_'+param, kwargs.pop(param, old[param])) # Number of SVD modes if self._nsvd is None: # Initialization self._nsvd = self._nsvd_default # Default value for iset in range(2): # Check values if self[iset].prepca: nchanmax = self[iset].prepca # Input channels are from pre-PCA else: nchanmax = self[iset].ns # Input channels are from real space self._nsvd = npy.clip(self._nsvd, 1, min(self._nsvd_max, nchanmax)) # Max # Re-run analyses when needed rerun = dict( svd = self._has_run_('svd') and (self._has_changed_(old, 'nsvd')>0 or \ (self[0].prepca and reran[0]['pca']) or \ (self[1].prepca and reran[1]['pca'])), ) # - SVD if rerun['svd']: self.debug('Re-running SVD because some parameters changed') self.svd() # Inform what has reran return rerun def _has_run_(self, anatype=None, iset=0): """Check if an analysis has already run""" if anatype is None: return False getattr(self, '_%s_raw_eof'%anatype) is None def _svd_channel_axis_(self,iset): """Get the SVD channel axis for one dataset""" return _channel_axis_(iset, 'svd', svd=True) def get_nsvd(self): """Get :attr:`nsvd` :Returns: integer """ return self._nsvd def set_nsvd(self, nsvd): """Set :attr:`nsvd` and update analyzes""" self.update(nsvd=nsvd) nsvd = property(get_nsvd, set_nsvd, doc="Number of SVD modes") def has_cdat(self): return self[0].has_cdat() or self[1].has_cdat() def clean(self, svd=True, pca=True): """(Re-)Initialization""" anatypes = [] self.lspan.clean(pca=pca) self.rspan.clean(pca=pca) if pca: svd = True if svd: anatypes.append('svd') # Register what to clean nones = [] for aa in anatypes: nones.append('_%s_ev_sum'%aa) for bb in 'raw','fmt': for cc in 'eof','pc','ev': nones.append('_%s_%s_%s'%(aa,bb,cc)) if svd: nones.extend(['_svd_channel_axes']) # lists = ['_mssa_pairs','_nt','_ns','_ndata','_pdata'] dicts = ['_mode_axes'] # Clean now using values or functions for ll,init in [(nones, None), (dicts, dict)]:#,(lists,list): for att in ll: self._cleanattr_(att, init) # Integers self._last_anatype = None gc.collect() def _dual_(self, data): """Makes sure to have 2 elements""" if not isinstance(data, (list, tuple)): data = [data] if len(data)!=2: data = broadcast(data, 2) return data def _isets_(self, iset): """Dataset indices""" if iset is None: return [0, 1] if iset==0 or str(iset).lower()=='left': return [0] if iset==1 or str(iset).lower()=='right': return [1] raise DualSpanLibError("Wrong dataset choice. Choose one of None, 0/'left' or 1/'right'") ################################################################# ## SVD ################################################################# @_filldocs_ def svd(self, usecorr=False, largematrix=False, **kwargs): """ Singular Value Decomposition (SVD) It is called everytime needed by :meth:`svd_eof`, :meth:`svd_pc`, :meth:`svd_ev` and :meth:`svd_rec`. Thus, since results are stored in cache, it not necessary call it explicitly. :Parameters: %(nsvd)s %(prepca)s - *usecorr*: bool Use correlations instead of covariances. """ # Parameters self.update_params('svd', **kwargs) # Loop on first two datasets (left and right) for iset in range(2): # Check if old results can be used when nsvd is lower if self._svd_raw_pc is not None and \ self._svd_raw_pc[iset].shape[-1] > self._nsvd: break # Remove old results for att in 'raw_eof','raw_pc','raw_ev','ev_sum': dic = getattr(self,'_svd_'+att) if isinstance(dic, dict) and iset in dic: del dic[iset] # Prepare input to SVD raw_input = self[iset].preproc_raw_output() if iset==0: left = raw_input # lweights = raw_input[:, 0]*0.+1 else: right = raw_input # rweights = raw_input[:, 0]*0.+1 # if self._prepca[iset]: # Pre-PCA case # # # Pre-PCA # if not self._pca_raw_pc.has_key(iset): self.pca(iset=iset) # # # svd # data = self._pca_raw_pc[iset][:, :self._prepca[iset]].transpose() # if iset == 0: # left = data # lweights = data[:, 0]*0.+1 # else: # right = data # rweights = data[:, 0]*0.+1 # # else: # Direct svd case # weights = self[iset].stacked_weights # pdata = self[iset].stacked_data # if iset == 0: # left = pdata # lweights = weights # # else: # right = pdata # rweights = weights # Compute SVD left = npy.asfortranarray(left, 'd') right = npy.asfortranarray(right, 'd') raw_eof_left, raw_eof_right, raw_pc_left, raw_pc_right, raw_ev, ev_sum, errmsg = \ _core.svd(left, right, self._nsvd, int(usecorr), default_missing_value) self.check_fortran_errmsg(errmsg) # Save results self._svd_raw_pc = [npy.ascontiguousarray(raw_pc_left), npy.ascontiguousarray(raw_pc_right)] self._svd_raw_eof = [npy.ascontiguousarray(raw_eof_left), npy.ascontiguousarray(raw_eof_right)] self._svd_raw_ev = raw_ev self._svd_ev_sum = ev_sum self._last_anatype = 'svd' gc.collect() @_filldocs_ def svd_eof(self, scale=False, raw=False, unmap=True, format=True, **kwargs): """Get EOFs from SVD analysis If SVD was not performed, it is done with all parameters sent to :meth:`svd` :Parameters: %(scale)s %(raw)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Arrays with shape ``(nsvd,...)`` """ # Update params self.update_params('svd', **kwargs) # No analyses performed? if not self._has_run_('svd'): self.svd() # Of, let's format the variable fmt_eof = [] for iset in range(2): # (window*nchan,nsvd) # # EOF already available # if not raw and self._svd_fmt_eof is not None: # fmt_eof[iset] = self._svd_fmt_eof[iset] # continue # Raw data raw_eof = self._svd_raw_eof[iset][:, :self._nsvd] nl, nm = raw_eof.shape if raw: # Do not go back to physical space fmt_eof.append([npy.ascontiguousarray(raw_eof.T)]) if format and self[iset].has_cdat(): fmt_eof[iset][0] = cdms2.createVariable(fmt_eof[iset]) fmt_eof[iset][0].setAxisList( [self._mode_axis_('svd'), self._svd_channel_axis_(iset)]) else: # Get raw data back to physical space raw_eof = npy.ascontiguousarray(raw_eof) raw_eof.shape = (nl, nm) firstaxes = self._mode_axis_('svd') if not self[iset].prepca: # No pre-PCA performed fmt_eof.append(self[iset].unstack(raw_eof, rescale=False, format=1, firstaxes=firstaxes)) else: proj_eof, smodes = self[iset]._raw_rec_(self[iset]._pca_raw_eof, raw_eof.T) fmt_eof.append(self[iset].unstack(proj_eof, rescale=False, format=1, firstaxes = firstaxes)) # Set attributes for idata,eof in enumerate(fmt_eof[iset]): # Attributes if format and cdms2_isVariable(eof): if not raw and not self[iset][idata].id.find('variable_'): eof.id = self[iset][idata].id+'_svd_eof' else: eof.id = 'svd_eof' eof.name = eof.id eof.standard_name = 'empirical_orthogonal_functions_of_svd' eof.long_name = 'SVD empirical orthogonal functions' if not raw: atts = self[iset][idata].atts if atts.has_key('long_name'): eof.long_name += ' of '+atts['long_name'] if scale is True and atts.has_key('units'): eof.units = atts['units'] # Scaling if scale: if scale is True: scale = npy.sqrt(self._svd_raw_ev[iset])*nl eof[:] *= scale if raw: unmap = False if not unmap: return fmt_eof return self.unmap(fmt_eof) @_filldocs_ def svd_pc(self, raw=False, unmap=True, format=True, **kwargs): """Get PCs from SVD analysis If SVD was not performed, it is done with all parameters sent to :meth:`svd` :Parameters: %(raw)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Arrays with shape ``(nsvd,nt)`` """ # Update params self.update_params('svd', **kwargs) # No analyses performed? if not self._svd_raw_pc: self.svd() # Check cache # Of, let's format the variable fmt_pc = [] for iset in range(2): # Raw? raw_pc = self._svd_raw_pc[iset][:,:self._nsvd] if raw: fmt_pc.append(raw_pc) continue # Format the variable idata = 0 # Reference is first data pc = npy.ascontiguousarray(raw_pc).T if self[iset].has_cdat(): pc = cdms2.createVariable(pc) pc.setAxis(0,self._mode_axis_('svd')) pc.setAxis(1,self[iset].get_time(idata=idata)) pc.id = pc.name = 'svd_pc' # pc.standard_name = 'principal_components_of_svd' pc.long_name = 'SVD principal components' atts = self[iset][idata].atts if atts.has_key('long_name'): pc.long_name += ' of '+atts['long_name'] # if atts.has_key('units'): pc.units = atts['units'] fmt_pc.append(pc) # self._check_dataset_tag_('_svd_fmt_pc', iset, svd=True) # self._svd_fmt_pc = fmt_pc return fmt_pc # if not unmap: return fmt_pc # return self.unmap(fmt_pc) def svd_ec(self, iset=None, xdata=None, xeof=None, xraw=False, raw=False, unmap=True, format=True, **kwargs): """Get expansion coefficients from SVD analysis :Parameters: %(raw)s :SVD parameters: %(nsvd)s %(window)s %(prepca)s :Returns: Arrays with the shape ``(nsvd,nt)`` """ # Update params self.update_params('svd', **kwargs) # Dual input isets = self._isets_(iset) if len(isets)>1: xeof = self._dual_(xeof) xdata = self._dual_(xdata) # Loop on left and right fmt_ec = [] for iset in isets: # SVD EOFs used for projection if xeof[iset] is None: if self._has_run_('svd'): self.svd() raw_eof = self[iset]._svd_raw_eof elif xraw: raw_eof = xeof[iset] elif self[iset].prepca: # After PCA raw_eof = self[iset].pca_ec(xdata=xeof, raw=True, xscale=False, unmap=False)[iset].T raw_eof.shape = -1, self._nsvd else: eofs = self[iset].remap(xeof[iset]) raw_eof = npy.ascontiguousarray(self[iset].restack(eofs, scale=False)) del eofs # Data to project on SVD EOFs if xdata[iset] is None: # From input raw_data = self[i].preproc_raw_output() # if not self[iset]._prepca: # Input data # raw_data = self[iset].stacked_data # else: # After PCA # raw_data = self._pca_raw_pc[iset].T elif xraw: # Direct use raw_data = xdata[iset] elif self [iset].prepca: # After PCA raw_data = self[iset].pca_ec(xdata=xdata, raw=True, unmap=False)[iset].T else: data = self[iset].remap(xdata[iset]) raw_data = self[iset].restack(data, scale=True) # Projection raw_data = npy.asfortranarray(raw_data) raw_eof = npy.asfortranarray(raw_eof[:, :self._nsvd]) raw_weights = npy.ones(raw_data.shape[0]) raw_ec = _core.pca_getec(raw_data, raw_eof, raw_weights, default_missing_value) if raw: fmt_ec[iset] = raw_ec continue ec = npy.ascontiguousarray(raw_ec.T) ec = npy.ma.masked_values(ec, default_missing_value, copy=False) # Format the variable if format and self[iset].has_cdat(): ec = cdms2.createVariable(ec) ec.setAxis(0,self._mode_axis_('svd')) # ec.setAxis(1,self._svd_pctime_axis_(iset)) ec.id = ec.name = 'svd_ec' # ec.standard_name = 'expansion coefficient_of_svd of ' ec.long_name = 'SVD principal components' atts = self[iset][0].atts if len(self[iset])==1 and atts.has_key('long_name'): ec.long_name += atts['long_name'] # else: # ec.long_name += 'of dataset %i'%iset #if (len(self[iset])==1 or npy.allclose(self.norms(iset), 1.)) and atts.has_key('units'): # ec.units = atts['units'] fmt_ec[iset] = ec # FIXME: _check_dataset_tag_ # self._check_dataset_tag_('_svd_fmt_ec', iset) if len(isets)==1: fmt_ec = fmt_ec[0] return fmt_ec # if not unmap: return fmt_ec # return self.unmap(fmt_ec) @_filldocs_ def svd_ev(self,relative=False,sum=False,cumsum=False,**kwargs): """Get eigen values from SVD analysis :Parameters: %(relative)s %(sum)s %(cumsum)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Array with shape ``(nsvd,)`` or a float """ # Update params self.update_params('svd', **kwargs) # No analyses performed? if self._svd_raw_eof is None: self.svd() # We only want the sum if sum: return self._svd_ev_sum # Format the variable id = 'svd_ev' long_name = [] ev = self._svd_raw_ev[:self._nsvd] if cumsum: ev = ev.cumsum() id += '_cumsum' long_name.append('cumulative') if relative: ev = 100.*ev/self._svd_ev_sum id += '_rel' long_name.append('relative') # Format the variables if self.has_cdat(): ev = cdms2.createVariable(ev) ev.id = ev.name = id long_name = ['SVD eigen values'] ev.long_name = ' '.join(long_name).title() ev.setAxis(0, self._mode_axis_('svd')) # ev.standard_name = 'eigen_values_of_svd' if cumsum: ev.id += '_cumsum' long_name.append('cumulative') if relative: ev.id += '_rel' ev.units = '% of total variance' long_name.append('cumulative') ev.long_name = ' '.join(long_name).title() return ev @_filldocs_ def svd_rec(self, modes=None, raw=False, xpc=None, xeof=None, xraw=False, rescale=True, unmap=True, format=2, **kwargs): """Reconstruction of SVD modes :Parameters: %(modes)s %(raw)s :SVD parameters: %(nsvd)s %(prepca)s :Returns: Arrays with the same shape as input arrays. """ # Update params self.update_params('svd', **kwargs) # Dual input xeof = self._dual_(xeof) xpc = self._dual_(xpc) # Loop on left and right fmt_rec = [] for iset in range(2): # ST-EOFs used for projection if xeof[iset] is None: if self._has_run_('svd'): self.svd() raw_eof = self._svd_raw_eof[iset] elif xraw: raw_eof = xeof[iset] elif self[iset].prepca: # After PCA raw_eof = self[iset].pca_ec(xdata=xeof, raw=True, xscale=False, unmap=False)[iset].T else: eofs = self[iset].remap(xeof[iset]) raw_eof = npy.ascontiguousarray(self[iset].restack(eofs, scale=False)) del eofs raw_eof = raw_eof[:, :self._nsvd] # PCs if xpc[iset] is None: # From PCA if not self._has_run_('svd'): self.svd() raw_pc = self._svd_raw_pc[iset] elif xraw: # Direct use raw_pc = xpc[iset] elif npy.ma.isMA(xpc[iset]): # formatted -> pure numpy raw_pc = xpc[iset].filled(default_missing_value).T else: raw_pc = xpc[iset].T raw_pc = raw_pc[:, :self._nsvd] # Projection raw_rec,smodes = self[0]._raw_rec_(raw_eof, raw_pc, modes) # Get raw data back to physical space (nchan,nt) taxis = self[iset].get_time() if not self[iset].prepca: # No pre-PCA performed fmt_rec.append(self[iset].unstack(raw_rec, rescale=rescale, format=format)) elif raw: # Force direct result from svd if format and self[iset].has_cdat(): fmt_rec.append([cdms2.createVariable(raw_rec.T)]) fmt_rec[iset][0].setAxisList(0, [taxis,self._mode_axis_('svd')]) else: fmt_rec[iset] = raw_rec.T else: # With pre-pca proj_rec, spcamodes = self[iset]._raw_rec_(self[iset]._pca_raw_eof, raw_rec.T) fmt_rec.append(self[iset].unstack(proj_rec, rescale=rescale, format=format)) del raw_rec # Set attributes if format: for idata,rec in enumerate(fmt_rec[iset]): if not cdms2_isVariable(rec): continue if not self[iset][idata].id.startswith('variable_'): rec.id = self[iset][idata].id+'_svd_rec' else: rec.id = 'svd_rec' # if modes is not None: rec.id += smodes #FIXME: do we keep it? rec.modes = smodes # else: # rec.modes = '1-%i'%self._nsvd[iset] rec.standard_name = 'recontruction_of_svd_modes' rec.long_name = 'Reconstruction of SVD modes' atts = self[iset][idata].atts if atts.has_key('long_name'): rec.long_name += ' of '+atts['long_name'] if raw: unmap is False if not unmap: return fmt_rec return self.unmap(fmt_rec) def unmap(self, data, **kwargs): return self[0].unmap(data[0], **kwargs), self[1].unmap(data[1], **kwargs)