def simCDO_Sobol(cdo, rho, disc, paths, b): means = np.zeros([b, np.shape(cdo.a)[0]]) ss = sobol(1,paths*b,0) for i in range(b): zs = norm.ppf(ss[0,i*paths:(i+1)*paths]).T pv = np.zeros(np.shape(cdo.a)) # np.shape(cdo.a): number of tranches for z in zs: thisPV, _ = cdo.drawPV(z, rho, discf) pv += thisPV means[i,:] = pv/paths return np.mean(means,0), np.std(means,0)
def _downsample_mask(X, pct): """ Create a boolean mask indicating which subset of X should be evaluated. """ if pct < 1.0: Mask = np.zeros(X.shape, dtype=np.bool) m = X.shape[-2] n = X.shape[-1] nToEval = np.round(pct*m*n).astype(np.int32) idx = sobol(2, nToEval ,0) idx[0] = np.floor(m*idx[0]) idx[1] = np.floor(n*idx[1]) idx = idx.astype(np.int32) Mask[:,:,idx[0], idx[1]] = True else: Mask = np.ones(X.shape, dtype=np.bool) return Mask
def _downsample_mask(X, pct): """ Create a boolean mask indicating which subset of X should be evaluated. """ if pct < 1.0: Mask = np.zeros(X.shape, dtype=np.bool) m = X.shape[-2] n = X.shape[-1] nToEval = np.round(pct * m * n).astype(np.int32) idx = sobol(2, nToEval, 0) idx[0] = np.floor(m * idx[0]) idx[1] = np.floor(n * idx[1]) idx = idx.astype(np.int32) Mask[:, :, idx[0], idx[1]] = True else: Mask = np.ones(X.shape, dtype=np.bool) return Mask
def __init__(self, dist, scheme=1, box=True, edge=False, growth=0, sparse=0): """ Parameters ---------- dist : Dist The domain where the samples are to be generated Optional parameters box : int 0 : The samples are mapped onto the domain using an inverse Rosenblatt transform. 1 : Will only use distribution bounds and evenly distribute samples inbetween. 2 : Non transformation will be performed. edge: bool True : Will include the bounds of the domain in the samples. If infinite domain, a resonable trunkation will be used. growth : int 0 : Linear growth rule. Minimizes the number of samples. 1 : Exponential growth rule. Nested for schemes 1 and 2. 2 : Double the number of polynomial terms. Recommended for schemes 4, 5 and 6. sparse : int Defines the sparseness of the samples. sparse=len(dist) is equivalent to Smolyak sparse grid nodes. 0 : A full tensor product nodes will be used scheme : int 0 : Gaussian quadrature will be used. box and edge argument will be ignored. 1 : uniform distributed samples will be used. 2 : The roots of Chebishev polynomials will be used (Clenshaw-Curtis and Fejer). 3 : Stroud's cubature rule. Only order 2 and 3 valid, edge, growth and sparse are ignored. 4 : Latin Hypercube sampling, no edge or sparse. 5 : Classical random sampling, no edge or sparse. 6 : Halton sampling, no edge or sparse 7 : Hammersley sampling, no edge or sparse 8 : Sobol sampling, no edge or sparse 9 : Korobov samples, no edge or sparse """ print "Warning: Sampler is depricated. Use samplegen instead" self.dist = dist self.scheme = scheme self.sparse = sparse # Gausian Quadrature if scheme == 0: segment = lambda n: _gw(n, dist)[0] self.trans = lambda x: x else: if box == 0: self.trans = lambda x: self.dist.inv(x.T).T elif box == 1: lo, up = dist.range().reshape(2, len(dist)) self.trans = lambda x: np.array(x) * (up - lo) + lo elif box == 2: self.trans = lambda x: x if scheme == 1: _segment = lambda n: np.arange(0, n + 1) * 1. / n elif scheme == 2: _segment = lambda n: \ .5*np.cos(np.arange(n,-1,-1)*np.pi/n) + .5 if scheme in (1, 2): if edge: if growth == 0: segment = lambda n: _segment(n + 1) elif growth == 1: segment = lambda n: _segment(2**n) elif growth == 2: segment = lambda n: _segment(2 * be.terms(n, len(dist))) else: if growth == 0: segment = lambda n: _segment(n + 2)[1:-1] elif growth == 1: segment = lambda n: _segment(2**(n + 1))[1:-1] elif growth == 2: segment = lambda n: _segment(2*be.terms(n, \ len(dist)))[1:-1] elif scheme == 4: if growth == 0: segment = lambda n: latin_hypercube(n + 1, len(dist)) elif growth == 1: segment = lambda n: latin_hypercube(2**n, len(dist)) elif growth == 2: segment = lambda n: latin_hypercube(2*be.terms(n, len(dist)), \ len(dist)) elif scheme == 5: if growth == 0: segment = lambda n: np.random.random((n + 1, len(dist))) elif growth == 1: segment = lambda n: np.random.random((2**n, len(dist))) elif growth == 2: segment = lambda n: np.random.random( (2 * be.terms(n + 1, len(dist)), len(dist))) elif scheme == 6: if growth == 0: segment = lambda n: halton(n + 1, len(dist)) elif growth == 1: segment = lambda n: halton(2**n, len(dist)) elif growth == 2: segment = lambda n: halton(2*be.terms(n, len(dist)), \ len(dist)) elif scheme == 7: if growth == 0: segment = lambda n: hammersley(n + 1, len(dist)) elif growth == 1: segment = lambda n: hammersley(2**n, len(dist)) elif growth == 2: segment = lambda n: hammersley(2*be.terms(n, len(dist)), \ len(dist)) elif scheme == 8: if growth == 0: segment = lambda n: sobol(n + 1, len(dist)) elif growth == 1: segment = lambda n: sobol(2**n, len(dist)) elif growth == 2: segment = lambda n: sobol(2*be.terms(n, len(dist)), \ len(dist)) elif scheme == 9: if growth == 0: segment = lambda n: korobov(n + 1, len(dist)) elif growth == 1: segment = lambda n: korobov(2**n, len(dist)) elif growth == 2: segment = lambda n: korobov(2*be.terms(n, len(dist)), \ len(dist)) self.segment = segment
def samplegen(order, domain, rule="S", antithetic=None, verbose=False): """ Sample generator Parameters ---------- order : int Sample order domain : Dist, int, array_like Defines the space where the samples are generated. Dist Mapped to distribution domain using inverse Rosenblatt. int No mapping, but sets the number of dimension. array_like Stretch samples such that they are in [domain[0], domain[1]] rule : str rule for generating samples, where d is the number of dimensions. Key Name Nested ---- ---------------- ------ "C" Chebyshev nodes no "NC" Nested Chebyshev yes "G" Gaussian quadrature no "K" Korobov no "R" (Pseudo-)Random no "RG" Regular grid no "NG" Nested grid yes "L" Latin hypercube no "S" Sobol yes "H" Halton yes "M" Hammersley yes antithetic : array_like, optional List of bool. Represents the axes to mirror using antithetic variable. """ rule = rule.upper() if isinstance(domain, int): dim = domain trans = lambda x, verbose: x elif isinstance(domain, (tuple, list, np.ndarray)): domain = np.asfarray(domain) if len(domain.shape) < 2: dim = 1 else: dim = len(domain[0]) lo, up = domain trans = lambda x, verbose: ((up - lo) * x.T + lo).T else: dist = domain dim = len(dist) trans = dist.inv if not (antithetic is None): antithetic = np.array(antithetic, dtype=bool).flatten() if antithetic.size == 1 and dim > 1: antithetic = np.repeat(antithetic, dim) N = np.sum(1 * np.array(antithetic)) order_, order = order, int(order * 2**-N + 1 * (order % 2 != 0)) trans_ = trans trans = lambda X, verbose: \ trans_(antithetic_gen(X, antithetic)[:,:order_]) if rule == "C": X = chebyshev(dim, order) elif rule == "NC": X = chebyshev_nested(dim, order) elif rule == "G": X = _gw(order - 1, dist)[0] trans = lambda x, verbose: x elif rule == "K": X = korobov(dim, order) elif rule == "R": X = np.random.random((dim, order)) elif rule == "RG": X = regular_grid(dim, order) elif rule == "NG": X = regular_grid_nested(dim, order) elif rule == "L": X = latin_hypercube(dim, order) elif rule == "S": X = sobol(dim, order) elif rule == "H": X = halton(dim, order) elif rule == "M": X = hammersley(dim, order) else: raise KeyError, "rule not recognised" X = trans(X, verbose=verbose) return X
def __init__(self, dist, scheme=1, box=True, edge=False, growth=0, sparse=0): """ Parameters ---------- dist : Dist The domain where the samples are to be generated Optional parameters box : int 0 : The samples are mapped onto the domain using an inverse Rosenblatt transform. 1 : Will only use distribution bounds and evenly distribute samples inbetween. 2 : Non transformation will be performed. edge: bool True : Will include the bounds of the domain in the samples. If infinite domain, a resonable trunkation will be used. growth : int 0 : Linear growth rule. Minimizes the number of samples. 1 : Exponential growth rule. Nested for schemes 1 and 2. 2 : Double the number of polynomial terms. Recommended for schemes 4, 5 and 6. sparse : int Defines the sparseness of the samples. sparse=len(dist) is equivalent to Smolyak sparse grid nodes. 0 : A full tensor product nodes will be used scheme : int 0 : Gaussian quadrature will be used. box and edge argument will be ignored. 1 : uniform distributed samples will be used. 2 : The roots of Chebishev polynomials will be used (Clenshaw-Curtis and Fejer). 3 : Stroud's cubature rule. Only order 2 and 3 valid, edge, growth and sparse are ignored. 4 : Latin Hypercube sampling, no edge or sparse. 5 : Classical random sampling, no edge or sparse. 6 : Halton sampling, no edge or sparse 7 : Hammersley sampling, no edge or sparse 8 : Sobol sampling, no edge or sparse 9 : Korobov samples, no edge or sparse """ print "Warning: Sampler is depricated. Use samplegen instead" self.dist = dist self.scheme = scheme self.sparse = sparse # Gausian Quadrature if scheme==0: segment = lambda n: _gw(n,dist)[0] self.trans = lambda x:x else: if box==0: self.trans = lambda x: self.dist.inv(x.T).T elif box==1: lo, up = dist.range().reshape(2, len(dist)) self.trans = lambda x: np.array(x)*(up-lo) + lo elif box==2: self.trans = lambda x: x if scheme==1: _segment = lambda n: np.arange(0, n+1)*1./n elif scheme==2: _segment = lambda n: \ .5*np.cos(np.arange(n,-1,-1)*np.pi/n) + .5 if scheme in (1,2): if edge: if growth==0: segment = lambda n: _segment(n+1) elif growth==1: segment = lambda n: _segment(2**n) elif growth==2: segment = lambda n: _segment(2*be.terms(n, len(dist))) else: if growth==0: segment = lambda n: _segment(n+2)[1:-1] elif growth==1: segment = lambda n: _segment(2**(n+1))[1:-1] elif growth==2: segment = lambda n: _segment(2*be.terms(n, \ len(dist)))[1:-1] elif scheme==4: if growth==0: segment = lambda n: latin_hypercube(n+1, len(dist)) elif growth==1: segment = lambda n: latin_hypercube(2**n, len(dist)) elif growth==2: segment = lambda n: latin_hypercube(2*be.terms(n, len(dist)), \ len(dist)) elif scheme==5: if growth==0: segment = lambda n: np.random.random((n+1, len(dist))) elif growth==1: segment = lambda n: np.random.random((2**n, len(dist))) elif growth==2: segment = lambda n: np.random.random((2*be.terms(n+1, len(dist)), len(dist))) elif scheme==6: if growth==0: segment = lambda n: halton(n+1, len(dist)) elif growth==1: segment = lambda n: halton(2**n, len(dist)) elif growth==2: segment = lambda n: halton(2*be.terms(n, len(dist)), \ len(dist)) elif scheme==7: if growth==0: segment = lambda n: hammersley(n+1, len(dist)) elif growth==1: segment = lambda n: hammersley(2**n, len(dist)) elif growth==2: segment = lambda n: hammersley(2*be.terms(n, len(dist)), \ len(dist)) elif scheme==8: if growth==0: segment = lambda n: sobol(n+1, len(dist)) elif growth==1: segment = lambda n: sobol(2**n, len(dist)) elif growth==2: segment = lambda n: sobol(2*be.terms(n, len(dist)), \ len(dist)) elif scheme==9: if growth==0: segment = lambda n: korobov(n+1, len(dist)) elif growth==1: segment = lambda n: korobov(2**n, len(dist)) elif growth==2: segment = lambda n: korobov(2*be.terms(n, len(dist)), \ len(dist)) self.segment = segment
def samplegen(order, domain, rule="S", antithetic=None, verbose=False): """ Sample generator Parameters ---------- order : int Sample order domain : Dist, int, array_like Defines the space where the samples are generated. Dist Mapped to distribution domain using inverse Rosenblatt. int No mapping, but sets the number of dimension. array_like Stretch samples such that they are in [domain[0], domain[1]] rule : str rule for generating samples, where d is the number of dimensions. Key Name Nested ---- ---------------- ------ "C" Chebyshev nodes no "NC" Nested Chebyshev yes "G" Gaussian quadrature no "K" Korobov no "R" (Pseudo-)Random no "RG" Regular grid no "NG" Nested grid yes "LH" Latin hypercube no "S" Sobol yes "H" Halton yes "M" Hammersley yes antithetic : array_like, optional List of bool. Represents the axes to mirror using antithetic variable. """ rule = rule.upper() if isinstance(domain, int): dim = domain trans = lambda x, verbose:x elif isinstance(domain, (tuple, list, np.ndarray)): domain = np.asfarray(domain) if len(domain.shape)<2: dim = 1 else: dim = len(domain[0]) lo,up = domain trans = lambda x, verbose: ((up-lo)*x.T + lo).T else: dist = domain dim = len(dist) trans = dist.inv if not (antithetic is None): antithetic = np.array(antithetic, dtype=bool).flatten() if antithetic.size==1 and dim>1: antithetic = np.repeat(antithetic, dim) N = np.sum(1*np.array(antithetic)) order_,order = order,int(order*2**-N+1*(order%2!=0)) trans_ = trans trans = lambda X, verbose: \ trans_(antithetic_gen(X, antithetic)[:,:order_]) if rule=="C": X = chebyshev(dim, order) elif rule=="NC": X = chebyshev_nested(dim, order) elif rule=="G": X = _gw(order-1,dist)[0] trans = lambda x, verbose:x elif rule=="K": X = korobov(dim, order) elif rule=="R": X = np.random.random((dim,order)) elif rule=="RG": X = regular_grid(dim, order) elif rule=="NG": X = regular_grid_nested(dim, order) elif rule=="LH": X = latin_hypercube(dim, order) elif rule=="S": X = sobol(dim, order) elif rule=="H": X = halton(dim, order) elif rule=="M": X = hammersley(dim,order) else: raise KeyError, "rule not recognised" X = trans(X, verbose=verbose) return X
def _deploy_network(args): """ Runs Caffe in deploy mode (where there is no solver). """ #---------------------------------------- # parse information from the prototxt files #---------------------------------------- netFn = str(args.network) # unicode->str to avoid caffe API problems netParam = caffe_pb2.NetParameter() text_format.Merge(open(netFn).read(), netParam) batchDim = emlib.infer_data_dimensions(netFn) assert(batchDim[2] == batchDim[3]) # tiles must be square print('[emCNN]: batch shape: %s' % str(batchDim)) if args.outDir: outDir = args.outDir # overrides default else: # there is no snapshot dir in a network file, so default is # to just use the location of the network file. outDir = os.path.dirname(args.network) # Add a timestamped subdirectory. ts = datetime.datetime.now() subdir = "Deploy_%s_%02d:%02d" % (ts.date(), ts.hour, ts.minute) outDir = os.path.join(outDir, subdir) if not os.path.isdir(outDir): os.makedirs(outDir) print('[emCNN]: writing results to: %s' % outDir) # save the parameters we're using with open(os.path.join(outDir, 'params.txt'), 'w') as f: pprint(args, stream=f) #---------------------------------------- # Create the Caffe network # Note this assumes a relatively recent PyCaffe #---------------------------------------- phaseTest = 1 # 1 := test mode net = caffe.Net(netFn, args.model, phaseTest) #---------------------------------------- # Load data #---------------------------------------- bs = border_size(batchDim) print "[emCNN]: loading deploy data..." Xdeploy = _load_data(args.emDeployFile, None, tileRadius=bs, onlySlices=args.deploySlices) print "[emCNN]: tile radius is: %d" % bs # Create a mask volume (vs list of labels to omit) due to API of emlib if args.evalPct < 1: Mask = np.zeros(Xdeploy.shape, dtype=np.bool) m = Xdeploy.shape[-2] n = Xdeploy.shape[-1] nToEval = np.round(args.evalPct*m*n).astype(np.int32) idx = sobol(2, nToEval ,0) idx[0] = np.floor(m*idx[0]) idx[1] = np.floor(n*idx[1]) idx = idx.astype(np.int32) Mask[:,idx[0], idx[1]] = True pct = 100.*np.sum(Mask) / Mask.size print("[emCNN]: subsampling volume...%0.2f%% remains" % pct) else: Mask = np.ones(Xdeploy.shape, dtype=np.bool) #---------------------------------------- # Do deployment & save results #---------------------------------------- sys.stdout.flush() if args.nMC < 0: Prob = predict(net, Xdeploy, Mask, batchDim) else: Prob = predict(net, Xdeploy, Mask, batchDim, nMC=args.nMC) # discard mirrored edges Prob = prune_border_4d(Prob, bs) net.save(str(os.path.join(outDir, 'final.caffemodel'))) np.save(os.path.join(outDir, 'YhatDeploy'), Prob) scipy.io.savemat(os.path.join(outDir, 'YhatDeploy.mat'), {'Yhat' : Prob}) print('[emCNN]: deployment complete.')