def addDistFit(ensemble=None, lfit=True, lflatten=None, lrescale=False, reference=None, target=None,
lbootstrap=False, nbs=30, sample_axis=None, lglobalScale=False, lsourceScale=False,
lcrossval=False, ncv=0.2, dist=None, dist_args=None, load_list=None, lproduct='outer', **kwargs):
''' add distribution fits to ensemble; optionally also rescale; kwargs are necessary for correct list expansion '''
# find appropriate sample axis
if lflatten:
if sample_axis is not None: raise ArgumentError(sample_axis)
elif sample_axis is None: # auto-detect
for saxis in ('time','year'):
if all([all(ens.hasAxis(saxis)) for ens in ensemble]):
sample_axis = saxis; break
if sample_axis is None: raise AxisError("No sample axis detected")
else:
if isinstance(sample_axis,str):
if not all([all(ens.hasAxis(sample_axis)) for ens in ensemble]): raise AxisError(sample_axis)
elif isinstance(sample_axis,(tuple,list)):
# check that axes are there
for ax in sample_axis:
if not all([all(ens.hasAxis(ax)) for ens in ensemble]): raise AxisError(ax)
# merge axes
ensemble = [ens.mergeAxes(axes=sample_axis, new_axis='sample', asVar=True, linplace=False, \
lcheckAxis=False) for ens in ensemble]
sample_axis = 'sample'
else: raise AxisError(sample_axis)
# perform fit or return dummy
if dist_args is None: dist_args = dict()
if lfit: fitens = [ens.fitDist(lflatten=lflatten, axis=sample_axis, lcrossval=lcrossval, ncv=ncv,
lignoreParams=True, lbootstrap=lbootstrap, nbs=nbs,
dist=dist, **dist_args) for ens in ensemble]
else: fitens = [None]*len(ensemble)
# rescale fitted distribution (according to certain rules)
if lrescale:
if not reference: raise ArgumentError(str(reference))
# expand target list
if isinstance(target, (list,tuple)):
expand_list = load_list[:]
if 'names' in expand_list: expand_list[expand_list.index('names')] = 'target'
kwarg_list = expandArgumentList(target=target, expand_list=expand_list, lproduct=lproduct, **kwargs)
targets = [kwarg['target'] for kwarg in kwarg_list]
else: targets = [target]*len(fitens)
if isinstance(reference, (list,tuple)): raise NotImplementedError # don't expand reference list
# use global reference, if necessary
if isinstance(reference,str) and not all(reference in fit for fit in fitens):
i = 0
while i < len(fitens) and reference not in fitens[i]: i += 1
if i >= len(fitens): raise ArgumentError("Reference {:s} not found in any dataset!".format(reference))
reference = fitens[i][reference]
sclens = [rescaleDistributions(fit, samples=ens, reference=reference, target=tgt,
lglobal=lglobalScale, lsourceScale=lsourceScale) for fit,ens,tgt in zip(fitens,ensemble,targets)]
else: sclens = [None]*len(ensemble)
# return results
return fitens, sclens
def testExpArgList(self):
''' test function to expand argument lists '''
from datasets.common import expandArgumentList
# test arguments
args1 = [0,1,2]; args2 = ['0','1','2']; args3 = ['test']*3; arg4 = 'static1'; arg5 = 'static2'
explist = ['arg1','arg2','arg3']
# test inner product expansion
arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
expand_list=explist, lproduct='inner')
assert len(arg_list) == len(args1) and len(arg_list) == len(args2)
for args,arg1,arg2,arg3 in zip(arg_list,args1,args2,args3):
assert args['arg1'] == arg1
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
# test outer product expansion
arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
expand_list=explist, lproduct='outer')
assert len(arg_list) == len(args1) * len(args2) * len(args3)
n = 0
for arg1 in args1:
for arg2 in args2:
for arg3 in args3:
args = arg_list[n]
assert args['arg1'] == arg1
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
n += 1
assert n == len(arg_list)
# test simultaneous inner and outer product expansion
n1 = len(args2) * len(args3) / len(args1)
tmp1 = args1*n1
arg_list = expandArgumentList(arg1=tmp1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
outer_list=['arg2','arg3'], inner_list=['arg1'])
assert len(arg_list) == len(args2) * len(args3) == len(tmp1)
n = 0
for arg2 in args2:
for arg3 in args3:
args = arg_list[n]
assert args['arg1'] == tmp1[n]
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
n += 1
assert n == len(arg_list)
# test parallel outer product expansion
assert len(args1) == len(args2) # necessary for test
arg_list = expandArgumentList(arg1=args1, arg2=args2, arg3=args3, arg4=arg4, arg5=arg5,
expand_list=[('arg1','arg2'),'arg3'], lproduct='outer')
assert len(arg_list) == len(args1) * len(args3)
n = 0
for arg1,arg2 in zip(args1,args2):
for arg3 in args3:
args = arg_list[n]
assert args['arg1'] == arg1
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
n += 1
assert n == len(arg_list)
def addDistFit(ensemble=None,
lfit=True,
lflatten=None,
lrescale=False,
reference=None,
target=None,
lbootstrap=False,
nbs=30,
sample_axis=None,
lglobalScale=False,
lcrossval=False,
ncv=0.2,
dist=None,
dist_args=None,
load_list=None,
lproduct='outer',
**kwargs):
''' add distribution fits to ensemble; optionally also rescale; kwargs are necessary for correct list expansion '''
# find appropriate sample axis
if lflatten:
if sample_axis is not None: raise ArgumentError, sample_axis
elif sample_axis is None: # auto-detect
for saxis in ('time', 'year'):
if all([all(ens.hasAxis(saxis)) for ens in ensemble]):
sample_axis = saxis
break
if sample_axis is None: raise AxisError, "No sample axis detected"
else:
if isinstance(sample_axis, basestring):
if not all([all(ens.hasAxis(sample_axis)) for ens in ensemble]):
raise AxisError, sample_axis
elif isinstance(sample_axis, (tuple, list)):
# check that axes are there
for ax in sample_axis:
if not all([all(ens.hasAxis(ax)) for ens in ensemble]):
raise AxisError, ax
# merge axes
ensemble = [ens.mergeAxes(axes=sample_axis, new_axis='sample', asVar=True, linplace=False, \
lcheckAxis=False) for ens in ensemble]
sample_axis = 'sample'
else:
raise AxisError, sample_axis
# perform fit or return dummy
if dist_args is None: dist_args = dict()
if lfit:
fitens = [
ens.fitDist(lflatten=lflatten,
axis=sample_axis,
lcrossval=lcrossval,
ncv=ncv,
lignoreParams=True,
lbootstrap=lbootstrap,
nbs=nbs,
dist=dist,
**dist_args) for ens in ensemble
]
else:
fitens = [None] * len(ensemble)
# rescale fitted distribution (according to certain rules)
if lrescale:
if not reference: raise ArgumentError(str(reference))
# expand target list
if isinstance(target, (list, tuple)):
expand_list = load_list[:]
if 'names' in expand_list:
expand_list[expand_list.index('names')] = 'target'
kwarg_list = expandArgumentList(target=target,
expand_list=expand_list,
lproduct=lproduct,
**kwargs)
targets = [kwarg['target'] for kwarg in kwarg_list]
else:
targets = [target] * len(fitens)
if isinstance(reference, (list, tuple)):
raise NotImplementedError # don't expand reference list
# use global reference, if necessary
if isinstance(reference, basestring) and not all(reference in fit
for fit in fitens):
i = 0
while i < len(fitens) and reference not in fitens[i]:
i += 1
if i >= len(fitens):
raise ArgumentError, "Reference {:s} not found in any dataset!".format(
reference)
reference = fitens[i][reference]
sclens = [
rescaleDistributions(fit,
reference=reference,
target=tgt,
lglobal=lglobalScale)
for fit, tgt in zip(fitens, targets)
]
else:
sclens = [None] * len(ensemble)
# return results
return fitens, sclens
def testExpArgList(self):
''' test function to expand argument lists '''
from datasets.common import expandArgumentList
# test arguments
args1 = [0, 1, 2]
args2 = ['0', '1', '2']
args3 = ['test'] * 3
arg4 = 'static1'
arg5 = 'static2'
explist = ['arg1', 'arg2', 'arg3']
# test inner product expansion
arg_list = expandArgumentList(arg1=args1,
arg2=args2,
arg3=args3,
arg4=arg4,
arg5=arg5,
expand_list=explist,
lproduct='inner')
assert len(arg_list) == len(args1) and len(arg_list) == len(args2)
for args, arg1, arg2, arg3 in zip(arg_list, args1, args2, args3):
assert args['arg1'] == arg1
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
# test outer product expansion
arg_list = expandArgumentList(arg1=args1,
arg2=args2,
arg3=args3,
arg4=arg4,
arg5=arg5,
expand_list=explist,
lproduct='outer')
assert len(arg_list) == len(args1) * len(args2) * len(args3)
n = 0
for arg1 in args1:
for arg2 in args2:
for arg3 in args3:
args = arg_list[n]
assert args['arg1'] == arg1
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
n += 1
assert n == len(arg_list)
# test simultaneous inner and outer product expansion
n1 = len(args2) * len(args3) / len(args1)
tmp1 = args1 * n1
arg_list = expandArgumentList(arg1=tmp1,
arg2=args2,
arg3=args3,
arg4=arg4,
arg5=arg5,
outer_list=['arg2', 'arg3'],
inner_list=['arg1'])
assert len(arg_list) == len(args2) * len(args3) == len(tmp1)
n = 0
for arg2 in args2:
for arg3 in args3:
args = arg_list[n]
assert args['arg1'] == tmp1[n]
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
n += 1
assert n == len(arg_list)
# test parallel outer product expansion
assert len(args1) == len(args2) # necessary for test
arg_list = expandArgumentList(arg1=args1,
arg2=args2,
arg3=args3,
arg4=arg4,
arg5=arg5,
expand_list=[('arg1', 'arg2'), 'arg3'],
lproduct='outer')
assert len(arg_list) == len(args1) * len(args3)
n = 0
for arg1, arg2 in zip(args1, args2):
for arg3 in args3:
args = arg_list[n]
assert args['arg1'] == arg1
assert args['arg2'] == arg2
assert args['arg3'] == arg3
assert args['arg4'] == arg4
assert args['arg5'] == arg5
n += 1
assert n == len(arg_list)