Exemplo n.º 1
0
def addDistFit(ensemble=None, lfit=True, lflatten=None, lrescale=False, reference=None, target=None,  
               lbootstrap=False, nbs=30, sample_axis=None, lglobalScale=False, lsourceScale=False, 
               lcrossval=False, ncv=0.2, dist=None, dist_args=None, load_list=None, lproduct='outer', **kwargs): 
  ''' add distribution fits to ensemble; optionally also rescale; kwargs are necessary for correct list expansion '''
  
  # find appropriate sample axis
  if lflatten: 
    if sample_axis is not None: raise ArgumentError(sample_axis)
  elif sample_axis is None: # auto-detect
    for saxis in ('time','year'):
      if all([all(ens.hasAxis(saxis)) for ens in ensemble]): 
        sample_axis = saxis; break
    if sample_axis is None: raise AxisError("No sample axis detected") 
  else:
    if isinstance(sample_axis,str):
      if not all([all(ens.hasAxis(sample_axis)) for ens in ensemble]): raise AxisError(sample_axis) 
    elif isinstance(sample_axis,(tuple,list)):
      # check that axes are there
      for ax in sample_axis: 
        if not all([all(ens.hasAxis(ax)) for ens in ensemble]): raise AxisError(ax)
      # merge axes
      ensemble = [ens.mergeAxes(axes=sample_axis, new_axis='sample', asVar=True, linplace=False, \
                              lcheckAxis=False) for ens in ensemble]
      sample_axis = 'sample'
    else: raise AxisError(sample_axis)
  # perform fit or return dummy
  if dist_args is None: dist_args = dict()
  if lfit: fitens = [ens.fitDist(lflatten=lflatten, axis=sample_axis, lcrossval=lcrossval, ncv=ncv,
                                 lignoreParams=True, lbootstrap=lbootstrap, nbs=nbs,
                                 dist=dist, **dist_args) for ens in ensemble]
  else: fitens = [None]*len(ensemble)
  
  # rescale fitted distribution (according to certain rules)
  if lrescale:
    if not reference: raise ArgumentError(str(reference))
    # expand target list
    if isinstance(target, (list,tuple)):  
      expand_list = load_list[:]
      if 'names' in expand_list: expand_list[expand_list.index('names')] = 'target' 
      kwarg_list = expandArgumentList(target=target, expand_list=expand_list, lproduct=lproduct, **kwargs)
      targets = [kwarg['target'] for kwarg in kwarg_list]
    else: targets = [target]*len(fitens)
    if isinstance(reference, (list,tuple)): raise NotImplementedError # don't expand reference list
    # use global reference, if necessary
    if isinstance(reference,str) and not all(reference in fit for fit in fitens):
      i = 0 
      while i < len(fitens) and reference not in fitens[i]: i += 1
      if i >= len(fitens): raise ArgumentError("Reference {:s} not found in any dataset!".format(reference))
      reference = fitens[i][reference]
    sclens = [rescaleDistributions(fit, samples=ens, reference=reference, target=tgt, 
                                   lglobal=lglobalScale, lsourceScale=lsourceScale) for fit,ens,tgt in zip(fitens,ensemble,targets)]
  else: sclens = [None]*len(ensemble)
  
  # return results
  return fitens, sclens
Exemplo n.º 2
0
 def testExpArgList(self):
   ''' test function to expand argument lists '''    
   from datasets.common import expandArgumentList
   # test arguments
   args1 = [0,1,2]; args2 = ['0','1','2']; args3 = ['test']*3; arg4 = 'static1'; arg5 = 'static2' 
   explist = ['arg1','arg2','arg3']
   # test inner product expansion
   arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
                                 expand_list=explist, lproduct='inner')
   assert len(arg_list) == len(args1) and len(arg_list) == len(args2)
   for args,arg1,arg2,arg3 in zip(arg_list,args1,args2,args3):
     assert args['arg1'] == arg1
     assert args['arg2'] == arg2
     assert args['arg3'] == arg3
     assert args['arg4'] == arg4
     assert args['arg5'] == arg5
   # test outer product expansion
   arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
                                 expand_list=explist, lproduct='outer')
   assert len(arg_list) == len(args1) * len(args2) * len(args3)
   n = 0
   for arg1 in args1:
     for arg2 in args2:
       for arg3 in args3:
         args = arg_list[n]
         assert args['arg1'] == arg1
         assert args['arg2'] == arg2
         assert args['arg3'] == arg3
         assert args['arg4'] == arg4
         assert args['arg5'] == arg5
         n += 1
   assert n == len(arg_list)
   # test simultaneous inner and outer product expansion
   n1 = len(args2) * len(args3) / len(args1)
   tmp1 = args1*n1
   arg_list = expandArgumentList(arg1=tmp1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
                                 outer_list=['arg2','arg3'], inner_list=['arg1'])
   assert len(arg_list) == len(args2) * len(args3) == len(tmp1)
   n = 0
   for arg2 in args2:
     for arg3 in args3:
         args = arg_list[n]
         assert args['arg1'] == tmp1[n]
         assert args['arg2'] == arg2
         assert args['arg3'] == arg3
         assert args['arg4'] == arg4
         assert args['arg5'] == arg5
         n += 1
   assert n == len(arg_list)
   # test parallel outer product expansion
   assert len(args1) == len(args2) # necessary for test
   arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
                                 expand_list=[('arg1','arg2'),'arg3'], lproduct='outer')
   assert len(arg_list) == len(args1) * len(args3)
   n = 0
   for arg1,arg2 in zip(args1,args2):
     for arg3 in args3:
       args = arg_list[n]
       assert args['arg1'] == arg1
       assert args['arg2'] == arg2
       assert args['arg3'] == arg3
       assert args['arg4'] == arg4
       assert args['arg5'] == arg5
       n += 1
   assert n == len(arg_list)
Exemplo n.º 3
0
def addDistFit(ensemble=None,
               lfit=True,
               lflatten=None,
               lrescale=False,
               reference=None,
               target=None,
               lbootstrap=False,
               nbs=30,
               sample_axis=None,
               lglobalScale=False,
               lcrossval=False,
               ncv=0.2,
               dist=None,
               dist_args=None,
               load_list=None,
               lproduct='outer',
               **kwargs):
    ''' add distribution fits to ensemble; optionally also rescale; kwargs are necessary for correct list expansion '''

    # find appropriate sample axis
    if lflatten:
        if sample_axis is not None: raise ArgumentError, sample_axis
    elif sample_axis is None:  # auto-detect
        for saxis in ('time', 'year'):
            if all([all(ens.hasAxis(saxis)) for ens in ensemble]):
                sample_axis = saxis
                break
        if sample_axis is None: raise AxisError, "No sample axis detected"
    else:
        if isinstance(sample_axis, basestring):
            if not all([all(ens.hasAxis(sample_axis)) for ens in ensemble]):
                raise AxisError, sample_axis
        elif isinstance(sample_axis, (tuple, list)):
            # check that axes are there
            for ax in sample_axis:
                if not all([all(ens.hasAxis(ax)) for ens in ensemble]):
                    raise AxisError, ax
            # merge axes
            ensemble = [ens.mergeAxes(axes=sample_axis, new_axis='sample', asVar=True, linplace=False, \
                                    lcheckAxis=False) for ens in ensemble]
            sample_axis = 'sample'
        else:
            raise AxisError, sample_axis
    # perform fit or return dummy
    if dist_args is None: dist_args = dict()
    if lfit:
        fitens = [
            ens.fitDist(lflatten=lflatten,
                        axis=sample_axis,
                        lcrossval=lcrossval,
                        ncv=ncv,
                        lignoreParams=True,
                        lbootstrap=lbootstrap,
                        nbs=nbs,
                        dist=dist,
                        **dist_args) for ens in ensemble
        ]
    else:
        fitens = [None] * len(ensemble)

    # rescale fitted distribution (according to certain rules)
    if lrescale:
        if not reference: raise ArgumentError(str(reference))
        # expand target list
        if isinstance(target, (list, tuple)):
            expand_list = load_list[:]
            if 'names' in expand_list:
                expand_list[expand_list.index('names')] = 'target'
            kwarg_list = expandArgumentList(target=target,
                                            expand_list=expand_list,
                                            lproduct=lproduct,
                                            **kwargs)
            targets = [kwarg['target'] for kwarg in kwarg_list]
        else:
            targets = [target] * len(fitens)
        if isinstance(reference, (list, tuple)):
            raise NotImplementedError  # don't expand reference list
        # use global reference, if necessary
        if isinstance(reference, basestring) and not all(reference in fit
                                                         for fit in fitens):
            i = 0
            while i < len(fitens) and reference not in fitens[i]:
                i += 1
            if i >= len(fitens):
                raise ArgumentError, "Reference {:s} not found in any dataset!".format(
                    reference)
            reference = fitens[i][reference]
        sclens = [
            rescaleDistributions(fit,
                                 reference=reference,
                                 target=tgt,
                                 lglobal=lglobalScale)
            for fit, tgt in zip(fitens, targets)
        ]
    else:
        sclens = [None] * len(ensemble)

    # return results
    return fitens, sclens
Exemplo n.º 4
0
 def testExpArgList(self):
     ''' test function to expand argument lists '''
     from datasets.common import expandArgumentList
     # test arguments
     args1 = [0, 1, 2]
     args2 = ['0', '1', '2']
     args3 = ['test'] * 3
     arg4 = 'static1'
     arg5 = 'static2'
     explist = ['arg1', 'arg2', 'arg3']
     # test inner product expansion
     arg_list = expandArgumentList(arg1=args1,
                                   arg2=args2,
                                   arg3=args3,
                                   arg4=arg4,
                                   arg5=arg5,
                                   expand_list=explist,
                                   lproduct='inner')
     assert len(arg_list) == len(args1) and len(arg_list) == len(args2)
     for args, arg1, arg2, arg3 in zip(arg_list, args1, args2, args3):
         assert args['arg1'] == arg1
         assert args['arg2'] == arg2
         assert args['arg3'] == arg3
         assert args['arg4'] == arg4
         assert args['arg5'] == arg5
     # test outer product expansion
     arg_list = expandArgumentList(arg1=args1,
                                   arg2=args2,
                                   arg3=args3,
                                   arg4=arg4,
                                   arg5=arg5,
                                   expand_list=explist,
                                   lproduct='outer')
     assert len(arg_list) == len(args1) * len(args2) * len(args3)
     n = 0
     for arg1 in args1:
         for arg2 in args2:
             for arg3 in args3:
                 args = arg_list[n]
                 assert args['arg1'] == arg1
                 assert args['arg2'] == arg2
                 assert args['arg3'] == arg3
                 assert args['arg4'] == arg4
                 assert args['arg5'] == arg5
                 n += 1
     assert n == len(arg_list)
     # test simultaneous inner and outer product expansion
     n1 = len(args2) * len(args3) / len(args1)
     tmp1 = args1 * n1
     arg_list = expandArgumentList(arg1=tmp1,
                                   arg2=args2,
                                   arg3=args3,
                                   arg4=arg4,
                                   arg5=arg5,
                                   outer_list=['arg2', 'arg3'],
                                   inner_list=['arg1'])
     assert len(arg_list) == len(args2) * len(args3) == len(tmp1)
     n = 0
     for arg2 in args2:
         for arg3 in args3:
             args = arg_list[n]
             assert args['arg1'] == tmp1[n]
             assert args['arg2'] == arg2
             assert args['arg3'] == arg3
             assert args['arg4'] == arg4
             assert args['arg5'] == arg5
             n += 1
     assert n == len(arg_list)
     # test parallel outer product expansion
     assert len(args1) == len(args2)  # necessary for test
     arg_list = expandArgumentList(arg1=args1,
                                   arg2=args2,
                                   arg3=args3,
                                   arg4=arg4,
                                   arg5=arg5,
                                   expand_list=[('arg1', 'arg2'), 'arg3'],
                                   lproduct='outer')
     assert len(arg_list) == len(args1) * len(args3)
     n = 0
     for arg1, arg2 in zip(args1, args2):
         for arg3 in args3:
             args = arg_list[n]
             assert args['arg1'] == arg1
             assert args['arg2'] == arg2
             assert args['arg3'] == arg3
             assert args['arg4'] == arg4
             assert args['arg5'] == arg5
             n += 1
     assert n == len(arg_list)