def simCDO_Sobol(cdo, rho, disc, paths, b):
means = np.zeros([b, np.shape(cdo.a)[0]])
ss = sobol(1,paths*b,0)
for i in range(b):
zs = norm.ppf(ss[0,i*paths:(i+1)*paths]).T
pv = np.zeros(np.shape(cdo.a)) # np.shape(cdo.a): number of tranches
for z in zs:
thisPV, _ = cdo.drawPV(z, rho, discf)
pv += thisPV
means[i,:] = pv/paths
return np.mean(means,0), np.std(means,0)
def _downsample_mask(X, pct):
""" Create a boolean mask indicating which subset of X should be
evaluated.
"""
if pct < 1.0:
Mask = np.zeros(X.shape, dtype=np.bool)
m = X.shape[-2]
n = X.shape[-1]
nToEval = np.round(pct*m*n).astype(np.int32)
idx = sobol(2, nToEval ,0)
idx[0] = np.floor(m*idx[0])
idx[1] = np.floor(n*idx[1])
idx = idx.astype(np.int32)
Mask[:,:,idx[0], idx[1]] = True
else:
Mask = np.ones(X.shape, dtype=np.bool)
return Mask
def _downsample_mask(X, pct):
""" Create a boolean mask indicating which subset of X should be
evaluated.
"""
if pct < 1.0:
Mask = np.zeros(X.shape, dtype=np.bool)
m = X.shape[-2]
n = X.shape[-1]
nToEval = np.round(pct * m * n).astype(np.int32)
idx = sobol(2, nToEval, 0)
idx[0] = np.floor(m * idx[0])
idx[1] = np.floor(n * idx[1])
idx = idx.astype(np.int32)
Mask[:, :, idx[0], idx[1]] = True
else:
Mask = np.ones(X.shape, dtype=np.bool)
return Mask
def __init__(self,
dist,
scheme=1,
box=True,
edge=False,
growth=0,
sparse=0):
"""
Parameters
----------
dist : Dist
The domain where the samples are to be generated
Optional parameters
box : int
0 : The samples are mapped onto the domain using an inverse
Rosenblatt transform.
1 : Will only use distribution bounds and evenly
distribute samples inbetween.
2 : Non transformation will be performed.
edge: bool
True : Will include the bounds of the domain in the samples.
If infinite domain, a resonable trunkation will be used.
growth : int
0 : Linear growth rule. Minimizes the number of samples.
1 : Exponential growth rule. Nested for schemes 1 and 2.
2 : Double the number of polynomial terms. Recommended for
schemes 4, 5 and 6.
sparse : int
Defines the sparseness of the samples.
sparse=len(dist) is equivalent to Smolyak sparse grid nodes.
0 : A full tensor product nodes will be used
scheme : int
0 : Gaussian quadrature will be used.
box and edge argument will be ignored.
1 : uniform distributed samples will be used.
2 : The roots of Chebishev polynomials will be used
(Clenshaw-Curtis and Fejer).
3 : Stroud's cubature rule.
Only order 2 and 3 valid,
edge, growth and sparse are ignored.
4 : Latin Hypercube sampling, no edge or sparse.
5 : Classical random sampling, no edge or sparse.
6 : Halton sampling, no edge or sparse
7 : Hammersley sampling, no edge or sparse
8 : Sobol sampling, no edge or sparse
9 : Korobov samples, no edge or sparse
"""
print "Warning: Sampler is depricated. Use samplegen instead"
self.dist = dist
self.scheme = scheme
self.sparse = sparse
# Gausian Quadrature
if scheme == 0:
segment = lambda n: _gw(n, dist)[0]
self.trans = lambda x: x
else:
if box == 0:
self.trans = lambda x: self.dist.inv(x.T).T
elif box == 1:
lo, up = dist.range().reshape(2, len(dist))
self.trans = lambda x: np.array(x) * (up - lo) + lo
elif box == 2:
self.trans = lambda x: x
if scheme == 1:
_segment = lambda n: np.arange(0, n + 1) * 1. / n
elif scheme == 2:
_segment = lambda n: \
.5*np.cos(np.arange(n,-1,-1)*np.pi/n) + .5
if scheme in (1, 2):
if edge:
if growth == 0:
segment = lambda n: _segment(n + 1)
elif growth == 1:
segment = lambda n: _segment(2**n)
elif growth == 2:
segment = lambda n: _segment(2 * be.terms(n, len(dist)))
else:
if growth == 0:
segment = lambda n: _segment(n + 2)[1:-1]
elif growth == 1:
segment = lambda n: _segment(2**(n + 1))[1:-1]
elif growth == 2:
segment = lambda n: _segment(2*be.terms(n, \
len(dist)))[1:-1]
elif scheme == 4:
if growth == 0:
segment = lambda n: latin_hypercube(n + 1, len(dist))
elif growth == 1:
segment = lambda n: latin_hypercube(2**n, len(dist))
elif growth == 2:
segment = lambda n: latin_hypercube(2*be.terms(n, len(dist)), \
len(dist))
elif scheme == 5:
if growth == 0:
segment = lambda n: np.random.random((n + 1, len(dist)))
elif growth == 1:
segment = lambda n: np.random.random((2**n, len(dist)))
elif growth == 2:
segment = lambda n: np.random.random(
(2 * be.terms(n + 1, len(dist)), len(dist)))
elif scheme == 6:
if growth == 0:
segment = lambda n: halton(n + 1, len(dist))
elif growth == 1:
segment = lambda n: halton(2**n, len(dist))
elif growth == 2:
segment = lambda n: halton(2*be.terms(n, len(dist)), \
len(dist))
elif scheme == 7:
if growth == 0:
segment = lambda n: hammersley(n + 1, len(dist))
elif growth == 1:
segment = lambda n: hammersley(2**n, len(dist))
elif growth == 2:
segment = lambda n: hammersley(2*be.terms(n, len(dist)), \
len(dist))
elif scheme == 8:
if growth == 0:
segment = lambda n: sobol(n + 1, len(dist))
elif growth == 1:
segment = lambda n: sobol(2**n, len(dist))
elif growth == 2:
segment = lambda n: sobol(2*be.terms(n, len(dist)), \
len(dist))
elif scheme == 9:
if growth == 0:
segment = lambda n: korobov(n + 1, len(dist))
elif growth == 1:
segment = lambda n: korobov(2**n, len(dist))
elif growth == 2:
segment = lambda n: korobov(2*be.terms(n, len(dist)), \
len(dist))
self.segment = segment
def samplegen(order, domain, rule="S", antithetic=None, verbose=False):
"""
Sample generator
Parameters
----------
order : int
Sample order
domain : Dist, int, array_like
Defines the space where the samples are generated.
Dist
Mapped to distribution domain using inverse Rosenblatt.
int
No mapping, but sets the number of dimension.
array_like
Stretch samples such that they are in [domain[0], domain[1]]
rule : str
rule for generating samples, where d is the number of
dimensions.
Key Name Nested
---- ---------------- ------
"C" Chebyshev nodes no
"NC" Nested Chebyshev yes
"G" Gaussian quadrature no
"K" Korobov no
"R" (Pseudo-)Random no
"RG" Regular grid no
"NG" Nested grid yes
"L" Latin hypercube no
"S" Sobol yes
"H" Halton yes
"M" Hammersley yes
antithetic : array_like, optional
List of bool. Represents the axes to mirror using antithetic
variable.
"""
rule = rule.upper()
if isinstance(domain, int):
dim = domain
trans = lambda x, verbose: x
elif isinstance(domain, (tuple, list, np.ndarray)):
domain = np.asfarray(domain)
if len(domain.shape) < 2:
dim = 1
else:
dim = len(domain[0])
lo, up = domain
trans = lambda x, verbose: ((up - lo) * x.T + lo).T
else:
dist = domain
dim = len(dist)
trans = dist.inv
if not (antithetic is None):
antithetic = np.array(antithetic, dtype=bool).flatten()
if antithetic.size == 1 and dim > 1:
antithetic = np.repeat(antithetic, dim)
N = np.sum(1 * np.array(antithetic))
order_, order = order, int(order * 2**-N + 1 * (order % 2 != 0))
trans_ = trans
trans = lambda X, verbose: \
trans_(antithetic_gen(X, antithetic)[:,:order_])
if rule == "C":
X = chebyshev(dim, order)
elif rule == "NC":
X = chebyshev_nested(dim, order)
elif rule == "G":
X = _gw(order - 1, dist)[0]
trans = lambda x, verbose: x
elif rule == "K":
X = korobov(dim, order)
elif rule == "R":
X = np.random.random((dim, order))
elif rule == "RG":
X = regular_grid(dim, order)
elif rule == "NG":
X = regular_grid_nested(dim, order)
elif rule == "L":
X = latin_hypercube(dim, order)
elif rule == "S":
X = sobol(dim, order)
elif rule == "H":
X = halton(dim, order)
elif rule == "M":
X = hammersley(dim, order)
else:
raise KeyError, "rule not recognised"
X = trans(X, verbose=verbose)
return X
def __init__(self, dist, scheme=1, box=True, edge=False,
growth=0, sparse=0):
"""
Parameters
----------
dist : Dist
The domain where the samples are to be generated
Optional parameters
box : int
0 : The samples are mapped onto the domain using an inverse
Rosenblatt transform.
1 : Will only use distribution bounds and evenly
distribute samples inbetween.
2 : Non transformation will be performed.
edge: bool
True : Will include the bounds of the domain in the samples.
If infinite domain, a resonable trunkation will be used.
growth : int
0 : Linear growth rule. Minimizes the number of samples.
1 : Exponential growth rule. Nested for schemes 1 and 2.
2 : Double the number of polynomial terms. Recommended for
schemes 4, 5 and 6.
sparse : int
Defines the sparseness of the samples.
sparse=len(dist) is equivalent to Smolyak sparse grid nodes.
0 : A full tensor product nodes will be used
scheme : int
0 : Gaussian quadrature will be used.
box and edge argument will be ignored.
1 : uniform distributed samples will be used.
2 : The roots of Chebishev polynomials will be used
(Clenshaw-Curtis and Fejer).
3 : Stroud's cubature rule.
Only order 2 and 3 valid,
edge, growth and sparse are ignored.
4 : Latin Hypercube sampling, no edge or sparse.
5 : Classical random sampling, no edge or sparse.
6 : Halton sampling, no edge or sparse
7 : Hammersley sampling, no edge or sparse
8 : Sobol sampling, no edge or sparse
9 : Korobov samples, no edge or sparse
"""
print "Warning: Sampler is depricated. Use samplegen instead"
self.dist = dist
self.scheme = scheme
self.sparse = sparse
# Gausian Quadrature
if scheme==0:
segment = lambda n: _gw(n,dist)[0]
self.trans = lambda x:x
else:
if box==0:
self.trans = lambda x: self.dist.inv(x.T).T
elif box==1:
lo, up = dist.range().reshape(2, len(dist))
self.trans = lambda x: np.array(x)*(up-lo) + lo
elif box==2:
self.trans = lambda x: x
if scheme==1:
_segment = lambda n: np.arange(0, n+1)*1./n
elif scheme==2:
_segment = lambda n: \
.5*np.cos(np.arange(n,-1,-1)*np.pi/n) + .5
if scheme in (1,2):
if edge:
if growth==0:
segment = lambda n: _segment(n+1)
elif growth==1:
segment = lambda n: _segment(2**n)
elif growth==2:
segment = lambda n: _segment(2*be.terms(n, len(dist)))
else:
if growth==0:
segment = lambda n: _segment(n+2)[1:-1]
elif growth==1:
segment = lambda n: _segment(2**(n+1))[1:-1]
elif growth==2:
segment = lambda n: _segment(2*be.terms(n, \
len(dist)))[1:-1]
elif scheme==4:
if growth==0:
segment = lambda n: latin_hypercube(n+1, len(dist))
elif growth==1:
segment = lambda n: latin_hypercube(2**n, len(dist))
elif growth==2:
segment = lambda n: latin_hypercube(2*be.terms(n, len(dist)), \
len(dist))
elif scheme==5:
if growth==0:
segment = lambda n: np.random.random((n+1, len(dist)))
elif growth==1:
segment = lambda n: np.random.random((2**n, len(dist)))
elif growth==2:
segment = lambda n: np.random.random((2*be.terms(n+1,
len(dist)), len(dist)))
elif scheme==6:
if growth==0:
segment = lambda n: halton(n+1, len(dist))
elif growth==1:
segment = lambda n: halton(2**n, len(dist))
elif growth==2:
segment = lambda n: halton(2*be.terms(n, len(dist)), \
len(dist))
elif scheme==7:
if growth==0:
segment = lambda n: hammersley(n+1, len(dist))
elif growth==1:
segment = lambda n: hammersley(2**n, len(dist))
elif growth==2:
segment = lambda n: hammersley(2*be.terms(n, len(dist)), \
len(dist))
elif scheme==8:
if growth==0:
segment = lambda n: sobol(n+1, len(dist))
elif growth==1:
segment = lambda n: sobol(2**n, len(dist))
elif growth==2:
segment = lambda n: sobol(2*be.terms(n, len(dist)), \
len(dist))
elif scheme==9:
if growth==0:
segment = lambda n: korobov(n+1, len(dist))
elif growth==1:
segment = lambda n: korobov(2**n, len(dist))
elif growth==2:
segment = lambda n: korobov(2*be.terms(n, len(dist)), \
len(dist))
self.segment = segment
def samplegen(order, domain, rule="S", antithetic=None,
verbose=False):
"""
Sample generator
Parameters
----------
order : int
Sample order
domain : Dist, int, array_like
Defines the space where the samples are generated.
Dist
Mapped to distribution domain using inverse Rosenblatt.
int
No mapping, but sets the number of dimension.
array_like
Stretch samples such that they are in [domain[0], domain[1]]
rule : str
rule for generating samples, where d is the number of
dimensions.
Key Name Nested
---- ---------------- ------
"C" Chebyshev nodes no
"NC" Nested Chebyshev yes
"G" Gaussian quadrature no
"K" Korobov no
"R" (Pseudo-)Random no
"RG" Regular grid no
"NG" Nested grid yes
"LH" Latin hypercube no
"S" Sobol yes
"H" Halton yes
"M" Hammersley yes
antithetic : array_like, optional
List of bool. Represents the axes to mirror using antithetic
variable.
"""
rule = rule.upper()
if isinstance(domain, int):
dim = domain
trans = lambda x, verbose:x
elif isinstance(domain, (tuple, list, np.ndarray)):
domain = np.asfarray(domain)
if len(domain.shape)<2:
dim = 1
else:
dim = len(domain[0])
lo,up = domain
trans = lambda x, verbose: ((up-lo)*x.T + lo).T
else:
dist = domain
dim = len(dist)
trans = dist.inv
if not (antithetic is None):
antithetic = np.array(antithetic, dtype=bool).flatten()
if antithetic.size==1 and dim>1:
antithetic = np.repeat(antithetic, dim)
N = np.sum(1*np.array(antithetic))
order_,order = order,int(order*2**-N+1*(order%2!=0))
trans_ = trans
trans = lambda X, verbose: \
trans_(antithetic_gen(X, antithetic)[:,:order_])
if rule=="C":
X = chebyshev(dim, order)
elif rule=="NC":
X = chebyshev_nested(dim, order)
elif rule=="G":
X = _gw(order-1,dist)[0]
trans = lambda x, verbose:x
elif rule=="K":
X = korobov(dim, order)
elif rule=="R":
X = np.random.random((dim,order))
elif rule=="RG":
X = regular_grid(dim, order)
elif rule=="NG":
X = regular_grid_nested(dim, order)
elif rule=="LH":
X = latin_hypercube(dim, order)
elif rule=="S":
X = sobol(dim, order)
elif rule=="H":
X = halton(dim, order)
elif rule=="M":
X = hammersley(dim,order)
else:
raise KeyError, "rule not recognised"
X = trans(X, verbose=verbose)
return X
def _deploy_network(args):
""" Runs Caffe in deploy mode (where there is no solver).
"""
#----------------------------------------
# parse information from the prototxt files
#----------------------------------------
netFn = str(args.network) # unicode->str to avoid caffe API problems
netParam = caffe_pb2.NetParameter()
text_format.Merge(open(netFn).read(), netParam)
batchDim = emlib.infer_data_dimensions(netFn)
assert(batchDim[2] == batchDim[3]) # tiles must be square
print('[emCNN]: batch shape: %s' % str(batchDim))
if args.outDir:
outDir = args.outDir # overrides default
else:
# there is no snapshot dir in a network file, so default is
# to just use the location of the network file.
outDir = os.path.dirname(args.network)
# Add a timestamped subdirectory.
ts = datetime.datetime.now()
subdir = "Deploy_%s_%02d:%02d" % (ts.date(), ts.hour, ts.minute)
outDir = os.path.join(outDir, subdir)
if not os.path.isdir(outDir):
os.makedirs(outDir)
print('[emCNN]: writing results to: %s' % outDir)
# save the parameters we're using
with open(os.path.join(outDir, 'params.txt'), 'w') as f:
pprint(args, stream=f)
#----------------------------------------
# Create the Caffe network
# Note this assumes a relatively recent PyCaffe
#----------------------------------------
phaseTest = 1 # 1 := test mode
net = caffe.Net(netFn, args.model, phaseTest)
#----------------------------------------
# Load data
#----------------------------------------
bs = border_size(batchDim)
print "[emCNN]: loading deploy data..."
Xdeploy = _load_data(args.emDeployFile,
None,
tileRadius=bs,
onlySlices=args.deploySlices)
print "[emCNN]: tile radius is: %d" % bs
# Create a mask volume (vs list of labels to omit) due to API of emlib
if args.evalPct < 1:
Mask = np.zeros(Xdeploy.shape, dtype=np.bool)
m = Xdeploy.shape[-2]
n = Xdeploy.shape[-1]
nToEval = np.round(args.evalPct*m*n).astype(np.int32)
idx = sobol(2, nToEval ,0)
idx[0] = np.floor(m*idx[0])
idx[1] = np.floor(n*idx[1])
idx = idx.astype(np.int32)
Mask[:,idx[0], idx[1]] = True
pct = 100.*np.sum(Mask) / Mask.size
print("[emCNN]: subsampling volume...%0.2f%% remains" % pct)
else:
Mask = np.ones(Xdeploy.shape, dtype=np.bool)
#----------------------------------------
# Do deployment & save results
#----------------------------------------
sys.stdout.flush()
if args.nMC < 0:
Prob = predict(net, Xdeploy, Mask, batchDim)
else:
Prob = predict(net, Xdeploy, Mask, batchDim, nMC=args.nMC)
# discard mirrored edges
Prob = prune_border_4d(Prob, bs)
net.save(str(os.path.join(outDir, 'final.caffemodel')))
np.save(os.path.join(outDir, 'YhatDeploy'), Prob)
scipy.io.savemat(os.path.join(outDir, 'YhatDeploy.mat'), {'Yhat' : Prob})
print('[emCNN]: deployment complete.')
