def addDistFit(ensemble=None, lfit=True, lflatten=None, lrescale=False, reference=None, target=None, lbootstrap=False, nbs=30, sample_axis=None, lglobalScale=False, lsourceScale=False, lcrossval=False, ncv=0.2, dist=None, dist_args=None, load_list=None, lproduct='outer', **kwargs): ''' add distribution fits to ensemble; optionally also rescale; kwargs are necessary for correct list expansion ''' # find appropriate sample axis if lflatten: if sample_axis is not None: raise ArgumentError(sample_axis) elif sample_axis is None: # auto-detect for saxis in ('time','year'): if all([all(ens.hasAxis(saxis)) for ens in ensemble]): sample_axis = saxis; break if sample_axis is None: raise AxisError("No sample axis detected") else: if isinstance(sample_axis,str): if not all([all(ens.hasAxis(sample_axis)) for ens in ensemble]): raise AxisError(sample_axis) elif isinstance(sample_axis,(tuple,list)): # check that axes are there for ax in sample_axis: if not all([all(ens.hasAxis(ax)) for ens in ensemble]): raise AxisError(ax) # merge axes ensemble = [ens.mergeAxes(axes=sample_axis, new_axis='sample', asVar=True, linplace=False, \ lcheckAxis=False) for ens in ensemble] sample_axis = 'sample' else: raise AxisError(sample_axis) # perform fit or return dummy if dist_args is None: dist_args = dict() if lfit: fitens = [ens.fitDist(lflatten=lflatten, axis=sample_axis, lcrossval=lcrossval, ncv=ncv, lignoreParams=True, lbootstrap=lbootstrap, nbs=nbs, dist=dist, **dist_args) for ens in ensemble] else: fitens = [None]*len(ensemble) # rescale fitted distribution (according to certain rules) if lrescale: if not reference: raise ArgumentError(str(reference)) # expand target list if isinstance(target, (list,tuple)): expand_list = load_list[:] if 'names' in expand_list: expand_list[expand_list.index('names')] = 'target' kwarg_list = expandArgumentList(target=target, expand_list=expand_list, lproduct=lproduct, **kwargs) targets = [kwarg['target'] for kwarg in kwarg_list] else: targets = [target]*len(fitens) if isinstance(reference, (list,tuple)): raise NotImplementedError # don't expand reference list # use global reference, if necessary if isinstance(reference,str) and not all(reference in fit for fit in fitens): i = 0 while i < len(fitens) and reference not in fitens[i]: i += 1 if i >= len(fitens): raise ArgumentError("Reference {:s} not found in any dataset!".format(reference)) reference = fitens[i][reference] sclens = [rescaleDistributions(fit, samples=ens, reference=reference, target=tgt, lglobal=lglobalScale, lsourceScale=lsourceScale) for fit,ens,tgt in zip(fitens,ensemble,targets)] else: sclens = [None]*len(ensemble) # return results return fitens, sclens
def testExpArgList(self): ''' test function to expand argument lists ''' from datasets.common import expandArgumentList # test arguments args1 = [0,1,2]; args2 = ['0','1','2']; args3 = ['test']*3; arg4 = 'static1'; arg5 = 'static2' explist = ['arg1','arg2','arg3'] # test inner product expansion arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, expand_list=explist, lproduct='inner') assert len(arg_list) == len(args1) and len(arg_list) == len(args2) for args,arg1,arg2,arg3 in zip(arg_list,args1,args2,args3): assert args['arg1'] == arg1 assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 # test outer product expansion arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, expand_list=explist, lproduct='outer') assert len(arg_list) == len(args1) * len(args2) * len(args3) n = 0 for arg1 in args1: for arg2 in args2: for arg3 in args3: args = arg_list[n] assert args['arg1'] == arg1 assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 n += 1 assert n == len(arg_list) # test simultaneous inner and outer product expansion n1 = len(args2) * len(args3) / len(args1) tmp1 = args1*n1 arg_list = expandArgumentList(arg1=tmp1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, outer_list=['arg2','arg3'], inner_list=['arg1']) assert len(arg_list) == len(args2) * len(args3) == len(tmp1) n = 0 for arg2 in args2: for arg3 in args3: args = arg_list[n] assert args['arg1'] == tmp1[n] assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 n += 1 assert n == len(arg_list) # test parallel outer product expansion assert len(args1) == len(args2) # necessary for test arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, expand_list=[('arg1','arg2'),'arg3'], lproduct='outer') assert len(arg_list) == len(args1) * len(args3) n = 0 for arg1,arg2 in zip(args1,args2): for arg3 in args3: args = arg_list[n] assert args['arg1'] == arg1 assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 n += 1 assert n == len(arg_list)
def addDistFit(ensemble=None, lfit=True, lflatten=None, lrescale=False, reference=None, target=None, lbootstrap=False, nbs=30, sample_axis=None, lglobalScale=False, lcrossval=False, ncv=0.2, dist=None, dist_args=None, load_list=None, lproduct='outer', **kwargs): ''' add distribution fits to ensemble; optionally also rescale; kwargs are necessary for correct list expansion ''' # find appropriate sample axis if lflatten: if sample_axis is not None: raise ArgumentError, sample_axis elif sample_axis is None: # auto-detect for saxis in ('time', 'year'): if all([all(ens.hasAxis(saxis)) for ens in ensemble]): sample_axis = saxis break if sample_axis is None: raise AxisError, "No sample axis detected" else: if isinstance(sample_axis, basestring): if not all([all(ens.hasAxis(sample_axis)) for ens in ensemble]): raise AxisError, sample_axis elif isinstance(sample_axis, (tuple, list)): # check that axes are there for ax in sample_axis: if not all([all(ens.hasAxis(ax)) for ens in ensemble]): raise AxisError, ax # merge axes ensemble = [ens.mergeAxes(axes=sample_axis, new_axis='sample', asVar=True, linplace=False, \ lcheckAxis=False) for ens in ensemble] sample_axis = 'sample' else: raise AxisError, sample_axis # perform fit or return dummy if dist_args is None: dist_args = dict() if lfit: fitens = [ ens.fitDist(lflatten=lflatten, axis=sample_axis, lcrossval=lcrossval, ncv=ncv, lignoreParams=True, lbootstrap=lbootstrap, nbs=nbs, dist=dist, **dist_args) for ens in ensemble ] else: fitens = [None] * len(ensemble) # rescale fitted distribution (according to certain rules) if lrescale: if not reference: raise ArgumentError(str(reference)) # expand target list if isinstance(target, (list, tuple)): expand_list = load_list[:] if 'names' in expand_list: expand_list[expand_list.index('names')] = 'target' kwarg_list = expandArgumentList(target=target, expand_list=expand_list, lproduct=lproduct, **kwargs) targets = [kwarg['target'] for kwarg in kwarg_list] else: targets = [target] * len(fitens) if isinstance(reference, (list, tuple)): raise NotImplementedError # don't expand reference list # use global reference, if necessary if isinstance(reference, basestring) and not all(reference in fit for fit in fitens): i = 0 while i < len(fitens) and reference not in fitens[i]: i += 1 if i >= len(fitens): raise ArgumentError, "Reference {:s} not found in any dataset!".format( reference) reference = fitens[i][reference] sclens = [ rescaleDistributions(fit, reference=reference, target=tgt, lglobal=lglobalScale) for fit, tgt in zip(fitens, targets) ] else: sclens = [None] * len(ensemble) # return results return fitens, sclens
def testExpArgList(self): ''' test function to expand argument lists ''' from datasets.common import expandArgumentList # test arguments args1 = [0, 1, 2] args2 = ['0', '1', '2'] args3 = ['test'] * 3 arg4 = 'static1' arg5 = 'static2' explist = ['arg1', 'arg2', 'arg3'] # test inner product expansion arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, expand_list=explist, lproduct='inner') assert len(arg_list) == len(args1) and len(arg_list) == len(args2) for args, arg1, arg2, arg3 in zip(arg_list, args1, args2, args3): assert args['arg1'] == arg1 assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 # test outer product expansion arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, expand_list=explist, lproduct='outer') assert len(arg_list) == len(args1) * len(args2) * len(args3) n = 0 for arg1 in args1: for arg2 in args2: for arg3 in args3: args = arg_list[n] assert args['arg1'] == arg1 assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 n += 1 assert n == len(arg_list) # test simultaneous inner and outer product expansion n1 = len(args2) * len(args3) / len(args1) tmp1 = args1 * n1 arg_list = expandArgumentList(arg1=tmp1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, outer_list=['arg2', 'arg3'], inner_list=['arg1']) assert len(arg_list) == len(args2) * len(args3) == len(tmp1) n = 0 for arg2 in args2: for arg3 in args3: args = arg_list[n] assert args['arg1'] == tmp1[n] assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 n += 1 assert n == len(arg_list) # test parallel outer product expansion assert len(args1) == len(args2) # necessary for test arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5, expand_list=[('arg1', 'arg2'), 'arg3'], lproduct='outer') assert len(arg_list) == len(args1) * len(args3) n = 0 for arg1, arg2 in zip(args1, args2): for arg3 in args3: args = arg_list[n] assert args['arg1'] == arg1 assert args['arg2'] == arg2 assert args['arg3'] == arg3 assert args['arg4'] == arg4 assert args['arg5'] == arg5 n += 1 assert n == len(arg_list)