def generate_rand_obs(nExp, nObs, domainBounds, sigvar=None, rng=None): # create random sets of observations for each experiment if not sigvar: sigvar = 1. if not rng: rng = RandomState() domainBounds = npa(domainBounds) dimX = domainBounds.shape[0] minX = domainBounds[:, 0] maxX = domainBounds[:, 1] rangeX = maxX - minX xObs = rng.uniform(size=(nExp, nObs, dimX)) xObs *= rangeX xObs += minX yObs = empty(shape=(nExp, nObs)) for iexp in xrange(nExp): good = False while not good: yObs0 = rng.normal(size=(nObs)) if yObs0.max() > 0: good = True yObs[iexp, :] = yObs0 # yObs = rng.normal(size=(nExp, nObs, 1)) yObs *= sigvar return {'x': xObs, 'y': yObs}
def test_param_cov(self): """ Tests that the 'param_cov' fit_info entry gets the right answer for *linear* least squares, where the answer is exact """ rs = RandomState(1234567890) a = 2 b = 100 x = np.linspace(0, 1, 100) # y scatter is amplitude ~1 to make sure covarience is non-negligible y = x*a + b + rs.randn(len(x)) #first compute the ordinary least squares covariance matrix X = np.matrix(np.vstack([x, np.ones(len(x))]).T) beta = np.linalg.inv(X.T * X) * X.T * np.matrix(y).T s2 = np.sum((y - (X * beta).A.ravel())**2) / (len(y) - len(beta)) olscov = np.linalg.inv(X.T * X) * s2 #now do the non-linear least squares fit mod = models.Linear1D(a, b) fitter = fitting.NonLinearLSQFitter() fmod = fitter(mod, x, y) utils.assert_allclose(fmod.parameters, beta.A.ravel()) utils.assert_allclose(olscov, fitter.fit_info['param_cov'])
def split(dataset, test_size=0.5, random_state=None): if random_state is None: random_state = np.random.randint(0, 999999) nb = dataset.X.shape[0] nb_test = int(nb * test_size) nb_train = nb - nb_test rng = RandomState(random_state) indices = np.arange(0, nb) rng.shuffle(indices) indices_train = indices[0:nb_train] indices_test = indices[nb_train:] X = dataset.X[indices_train] if hasattr(dataset, 'y') and dataset.y is not None: y = dataset.y[indices_train] else: y = None dataset_train = Manual(X, y) if hasattr(dataset, "img_dim"): dataset_train.img_dim = dataset.img_dim if hasattr(dataset, "output_dim"): dataset_train.output_dim = dataset.output_dim X = dataset.X[indices_test] if hasattr(dataset, 'y') and dataset.y is not None: y = dataset.y[indices_test] else: y = None dataset_test = Manual(X, y) if hasattr(dataset, "img_dim"): dataset_test.img_dim = dataset.img_dim if hasattr(dataset, "output_dim"): dataset_test.output_dim = dataset.output_dim return dataset_train, dataset_test
def test_get_random_state(): prng1 = RandomState(42) prng2 = get_prng(42) prng3 = get_prng(prng1) prng4 = get_prng(prng2) prng5 = get_prng() prng6 = get_prng(None) prng7 = get_prng(np.random) assert(isinstance(prng1, RandomState)) assert(isinstance(prng3, RandomState)) assert(isinstance(prng5, RandomState)) assert(isinstance(prng6, RandomState)) assert(isinstance(prng7, RandomState)) x1 = prng1.randint(5, size=10) x2 = prng2.randint(5, size=10) x3 = prng3.randint(5, size=10) x4 = prng4.randint(5, size=10) x5 = prng5.randint(5, size=10) x6 = prng6.randint(5, size=10) x7 = prng7.randint(5, size=10) assert_equal(x1, x2) assert_equal(x3, x4) assert_equal(len(x5), 10) assert_equal(len(x6), 10) assert_equal(len(x7), 10)
def test_online_predictor(): """ Test whether predictions are done at correct timepoints. Model actually just returns input """ rng = RandomState(3904890384) n_samples_in_buffer = 1000 dataset = rng.rand(n_samples_in_buffer*2,5).astype(np.float32) markers = np.ones((n_samples_in_buffer*2,1)).astype(np.float32) set_and_markers = np.concatenate((dataset, markers), axis=1) factor_new=0.001 n_stride = 10 pred_freq = 11 standardized = exponential_running_standardize(dataset, factor_new=factor_new, init_block_size=n_stride) model = InputLayer([1,1,1,1]) processor = StandardizeProcessor(factor_new=factor_new, n_samples_in_buffer=n_samples_in_buffer) online_model = OnlineModel(model) online_pred = OnlineCoordinator(processor, online_model, pred_freq=pred_freq, trainer=NoTrainer()) online_pred.initialize(n_chans=dataset.shape[1]) all_preds = [] for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride): online_pred.receive_samples(set_and_markers[i_start_sample:i_start_sample+n_stride]) if online_pred.has_new_prediction(): pred, _ = online_pred.pop_last_prediction_and_sample_ind() all_preds.append(pred) assert np.array_equal(np.array(all_preds).squeeze(), standardized[10::pred_freq])
def test_sum_prediction(): """ Test with a model that predicts sum over four samples """ rng = RandomState(3904890384) n_samples_in_buffer = 1000 dataset = rng.rand(n_samples_in_buffer*2,5).astype(np.float32) markers = np.ones((n_samples_in_buffer*2,1)).astype(np.float32) set_and_markers = np.concatenate((dataset, markers), axis=1) factor_new=0.001 n_stride = 10 pred_freq = 11 standardized = exponential_running_standardize(dataset, factor_new=factor_new, init_block_size=n_stride) model = InputLayer([1,1,4,1]) model = GlobalPoolLayer(model,pool_function=T.sum) expected = [np.sum(standardized[stop-4:stop], axis=0) for stop in xrange(11, dataset.shape[0], 11)] expected = np.array(expected) processor = StandardizeProcessor(factor_new=factor_new, n_samples_in_buffer=n_samples_in_buffer) online_model = OnlineModel(model) online_pred = OnlineCoordinator(processor, online_model, pred_freq=pred_freq, trainer=NoTrainer()) online_pred.initialize(n_chans=dataset.shape[1]) all_preds = [] for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride): online_pred.receive_samples(set_and_markers[i_start_sample:i_start_sample+n_stride]) if online_pred.has_new_prediction(): pred, _ = online_pred.pop_last_prediction_and_sample_ind() all_preds.append(pred) assert np.allclose(np.array(all_preds).squeeze(), expected, rtol=1e-3)
def train(self, X, XY, V, VY, count_dict, word_dict, embed_map): """ Trains the LBL """ self.start = self.seed self.init_params(embed_map, count_dict, XY) inds = np.arange(len(X)) numbatches = len(inds) / self.batchsize curr = 1e20 counter = 0 target=None num = 15000 # Main loop stop.display_phase(1) for epoch in range(self.maxepoch): self.epoch = epoch tic = time.time() prng = RandomState(self.seed + epoch + 1) prng.shuffle(inds) for minibatch in range(numbatches): batchX = X[inds[minibatch::numbatches]] batchY = XY[inds[minibatch::numbatches]] (words, acts, preds) = self.forward(batchX) self.backward(batchY, preds, acts, words, batchX) self.update_params(batchX) self.update_hyperparams() toc = time.time() # Results and stopping criteria obj = self.compute_obj(X[:num], XY[:num]) obj_val = self.compute_obj(V[:num], VY[:num]) if self.verbose > 0: stop.display_results(epoch, toc-tic, obj, obj_val) (curr, counter) = stop.update_result(curr, obj_val, counter) if counter == 0: stop.save_model(self, self.loc) stopping_target = obj if stop.criteria_complete(self, epoch, curr, obj, counter, self.k, obj_val, target): if self.criteria == 'maxepoch': break elif self.criteria == 'validation_pp': self = stop.load_model(self.loc) counter = 0 X = np.r_[X, V] XY = vstack([XY, VY]).tocsr() self.criteria = 'll_train_heldout' target = stopping_target #obj stop.display_phase(2) inds = range(X.shape[0]) prng.shuffle(inds) numbatches = len(inds) / self.batchsize elif self.criteria == 'll_train_heldout': break
def cmc(g, xdists, u_to_x, T, seed, maxitr): """ Crude Monte Carlo simulation. """ # Seed the random number generator if required if seed == -1: prng = RandomState() else: prng = RandomState(seed) # Generate standard normal samples centered at the origin u0 = zeros(len(xdists)) covmat = eye(len(xdists)) u = prng.multivariate_normal(u0, covmat, size=maxitr).T g_mc = g(u_to_x(u, xdists, T)) # Convert g-function output to pass/fail indicator function and estimate pf g_mc[g_mc>0] = 0 g_mc[g_mc<0] = 1 mu_pf = g_mc.mean() beta = -norm.ppf(mu_pf) if mu_pf < 0.5 else norm.ppf(mu_pf) # Convergence metrics (standard deviation, standard error, CoV of s.e.) std_pf = g_mc.std(ddof=1) # Calculate sample standard deviation se_pf = std_pf/sqrt(maxitr) cv_pf = se_pf/mu_pf return {'vars': xdists, 'beta': beta, 'Pf': mu_pf, 'stderr': se_pf, 'stdcv': cv_pf}
def corr(x, y, reps=10**4, prng=None): """ Simulate permutation p-value for Spearman correlation coefficient Parameters ---------- x : array-like y : array-like reps : int prng : RandomState instance or None, optional (default=None) If RandomState instance, prng is the pseudorandom number generator; If None, the pseudorandom number generator is the RandomState instance used by `np.random`. Returns ------- tuple Returns test statistic, left-sided p-value, right-sided p-value, two-sided p-value, simulated distribution """ if prng is None: prng = RandomState() tst = np.corrcoef(x, y)[0, 1] sims = [np.corrcoef(prng.permutation(x), y)[0, 1] for i in range(reps)] left_pv = np.sum(sims <= tst)/reps right_pv = np.sum(sims >= tst)/reps two_sided_pv = np.sum(np.abs(sims) >= np.abs(tst))/reps return tst, left_pv, right_pv, two_sided_pv, sims
def permute_within_groups(x, group, prng=None): """ Permutation of condition within each group. Parameters ---------- x : array-like A 1-d array indicating treatment. group : array-like A 1-d array indicating group membership prng : RandomState instance or None, optional (default=None) If RandomState instance, prng is the pseudorandom number generator; If None, the pseudorandom number generator is the RandomState instance used by `np.random`. Returns ------- permuted : array-like The within group permutation of x. """ permuted = x.copy() if prng is None: prng = RandomState() # (avoid additional flops) -- maybe memoize for g in np.unique(group): gg = group == g permuted[gg] = prng.permutation(permuted[gg]) return permuted
def assign_random_gt(input_vcf, output, sample_name="HG", default_af=0.01, seed=None): vcf_pointer = pysam.VariantFile(filename=input_vcf) new_header = vcf_pointer.header.copy() if "GT" not in new_header.formats: new_header.formats.add("GT", "1", "String", "Consensus Genotype across all datasets with called genotype") new_header.samples.add(sample_name) output.write(str(new_header)) default_probs = [1 - default_af * (1 + default_af), default_af/2, default_af/2, default_af * default_af] rng = RandomState(seed) previous_pos = 0 for rec in vcf_pointer.fetch(): rec_copy = rec.copy() if "GT" not in rec_copy.format.keys(): if rec_copy.pos == previous_pos: c = "0|0" else: if "AF" not in rec_copy.info.keys(): gt_probs = default_probs else: af = rec_copy.info["AF"] gt_probs = [1 - af * (1 + af), af/2, af/2, af * af] c = rng.choice(["0|0", "0|1", "1|0", "1|1"], p=gt_probs) output.write("\t".join([str(rec_copy)[:-1], "GT", c]) + "\n") previous_pos = rec_copy.pos vcf_pointer.close()
def test_experiment_sample_windows(): data_rng = RandomState(398765905) rand_topo = data_rng.rand(200,10,10,3).astype(np.float32) rand_y = np.int32(data_rng.rand(200) > 0.5) rand_topo[rand_y == 1] += 0.1 rand_set = DenseDesignMatrixWrapper(topo_view=rand_topo, y=rand_y) lasagne.random.set_rng(RandomState(9859295)) in_layer = InputLayer(shape= [None, 10,5,3]) network = DenseLayer(incoming=in_layer, name='softmax', num_units=2, nonlinearity=lasagne.nonlinearities.softmax) updates_modifier = MaxNormConstraint({'softmax': 0.5}) dataset = rand_set dataset_iterator = WindowsIterator(n_samples_per_window=5, batch_size=60) preprocessor = OnlineAxiswiseStandardize(axis=['c', 1]) dataset_splitter=FixedTrialSplitter(n_train_trials=150, valid_set_fraction=0.1) updates_var_func=lasagne.updates.adam loss_var_func= lasagne.objectives.categorical_crossentropy monitors=[braindecode.veganlasagne.monitors.LossMonitor (), braindecode.veganlasagne.monitors.WindowMisclassMonitor(), braindecode.veganlasagne.monitors.RuntimeMonitor()] stop_criterion= braindecode.veganlasagne.stopping.MaxEpochs(num_epochs=5) exp = Experiment(network, dataset, dataset_splitter, preprocessor, dataset_iterator, loss_var_func, updates_var_func, updates_modifier, monitors, stop_criterion, remember_best_chan='valid_misclass', run_after_early_stop=True) exp.setup() exp.run() assert np.allclose( [0.629630,0.140741,0.029630,0.022222,0.000000,0.000000,0.000000], exp.monitor_chans['train_misclass'], rtol=1e-4, atol=1e-4) assert np.allclose( [0.400000,0.133333,0.066667,0.000000,0.000000,0.000000,0.000000], exp.monitor_chans['valid_misclass'], rtol=1e-4, atol=1e-4) assert np.allclose( [0.560000,0.060000,0.000000,0.000000,0.000000,0.000000,0.000000], exp.monitor_chans['test_misclass'], rtol=1e-4, atol=1e-4) assert np.allclose( [1.180485, 0.574264, 0.420023, 0.330909, 0.278569, 0.245692, 0.242845], exp.monitor_chans['train_loss'], rtol=1e-4, atol=1e-4) assert np.allclose( [1.016782, 0.514049, 0.370485, 0.288948, 0.240913, 0.211189, 0.215967], exp.monitor_chans['valid_loss'], rtol=1e-4, atol=1e-4) assert np.allclose( [1.031832, 0.504570, 0.352317, 0.269810, 0.223904, 0.196681, 0.197899], exp.monitor_chans['test_loss'], rtol=1e-4, atol=1e-4)
def _smepdpsolve_single_trajectory(L, dt, tlist, N_store, N_substeps, rho_t, c_ops, e_ops, data): """ Internal function. """ states_list = [] rho_t = np.copy(rho_t) prng = RandomState() # todo: seed it r_jump, r_op = prng.rand(2) jump_times = [] jump_op_idx = [] for t_idx, t in enumerate(tlist): if e_ops: for e_idx, e in enumerate(e_ops): data.expect[e_idx, t_idx] += expect_rho_vec(e, rho_t) else: states_list.append(Qobj(vec2mat(rho_t))) for j in range(N_substeps): if expect_rho_vec(d_op, sigma_t) < r_jump: # jump occurs p = np.array([rho_expect(c.dag() * c, rho_t) for c in c_ops]) p = np.cumsum(p / np.sum(p)) n = np.where(p >= r_op)[0][0] # apply jump rho_t = c_ops[n] * psi_t * c_ops[n].dag() rho_t /= rho_expect(c.dag() * c, rho_t) rho_t = np.copy(rho_t) # store info about jump jump_times.append(tlist[t_idx] + dt * j) jump_op_idx.append(n) # get new random numbers for next jump r_jump, r_op = prng.rand(2) # deterministic evolution wihtout correction for norm decay dsigma_t = spmv(L.data.data, L.data.indices, L.data.indptr, sigma_t) * dt # deterministic evolution with correction for norm decay drho_t = spmv(L.data.data, L.data.indices, L.data.indptr, rho_t) * dt rho_t += drho_t # increment density matrices sigma_t += dsigma_t rho_t += drho_t return states_list, jump_times, jump_op_idx
def test_one_sample(): prng = RandomState(42) x = np.array(range(5)) y = x-1 # case 1: one sample only res = one_sample(x, seed = 42, reps = 100) np.testing.assert_almost_equal(res[0], 0.05999999) np.testing.assert_equal(res[1], 2) # case 2: paired sample res = one_sample(x, y, seed = 42, reps = 100) np.testing.assert_equal(res[0], 0.02) np.testing.assert_equal(res[1], 1) # case 3: break it - supply x and y, but not paired y = np.append(y, 10) assert_raises(ValueError, one_sample, x, y) # case 4: say keep_dist=True res = one_sample(x, seed = 42, reps = 100, keep_dist=True) np.testing.assert_almost_equal(res[0], 0.05999999) np.testing.assert_equal(res[1], 2) np.testing.assert_equal(min(res[2]), -2) np.testing.assert_equal(max(res[2]), 2) np.testing.assert_equal(np.median(res[2]), 0) # case 5: use t as test statistic y = x + prng.normal(size=5) res = one_sample(x, y, seed = 42, reps = 100, stat = "t", alternative = "less") np.testing.assert_equal(res[0], 0.07) np.testing.assert_almost_equal(res[1], 1.4491883)
def test_minp_one_pvalue(): prng = RandomState(55) pvalues = np.array([1]) distr = prng.uniform(low=0, high=10, size=20).reshape(20, 1) npc(pvalues, distr, "fisher", "greater") # TODO: more fwer_minp tests
def transpose_characters(token, index_to_char, n=1, char_pool=None, seed=17): if isinstance(seed, RandomState): rng = seed else: rng = RandomState(seed) chars = set(token) if len(chars) == 1: return token new_token = token for i in six.moves.range(n): idx = max(1, rng.randint(len(new_token))) neighbor = 0 if idx == 0: neighbor == 1 elif idx == len(new_token) - 1: neighbor = len(new_token) - 2 else: if rng.uniform() > 0.5: neighbor = idx + 1 else: neighbor = idx - 1 left = min(idx, neighbor) right = max(idx, neighbor) new_token = unicode(new_token[0:left] + new_token[right] + new_token[left] + new_token[right+1:]) return new_token
def test_pad_and_unpad_equal_1d(): 'gridder.pad_array and subsequent .unpad_array gives original array: 1D' prng = RandomState(12345) x = prng.rand(21) xpad, nps = gridder.pad_array(x) xunpad = gridder.unpad_array(xpad, nps) assert_almost(xunpad, x)
def test_qtl_binomial_scan_covariate_redundance(): random = RandomState(9) N = 200 G = random.randn(N, N + 100) G = stdnorm(G, 0) G /= sqrt(G.shape[1]) p = 2 X = random.randn(N, p) X = stdnorm(X, 0) X /= sqrt(X.shape[1]) ntrials = random.randint(1, 50, N) nsuccesses = binomial( ntrials, -0.1, G, causal_variants=X, causal_variance=0.1, random_state=random) X[:] = 1 qtl = scan(BinomialPhenotype(nsuccesses, ntrials), X, G=G, progress=False, fast=False) assert_allclose(qtl.pvalues(), [1] * p, rtol=1e-4)
def test_qtl_fast_binomial_scan(): random = RandomState(9) N = 200 G = random.randn(N, N + 100) G = stdnorm(G, 0) G /= sqrt(G.shape[1]) p = 2 X = random.randn(N, p) X = stdnorm(X, 0) X /= sqrt(X.shape[1]) ntrials = random.randint(1, 50, N) nsuccesses = binomial( ntrials, -0.1, G, causal_variants=X, causal_variance=0.1, random_state=random) qtl = scan(BinomialPhenotype(nsuccesses, ntrials), X, G=G, progress=False, fast=True) assert_allclose( qtl.pvalues(), [ 0.698565827403, 0.443299805368 ], rtol=1e-4)
def createGraph(self): """ W -> Adjacency Matrix For test cases, this method is modified to generate random weight matrix with zero weights for no edges. Graph is undirected with only positive weights @prng: pseudo random number generator """ ''' _ = 0 # no edge # a b c d W = [[_, 1, 3, 4], # a [1, _, 2, 1], # b [_, 1, _, 1], # c [5, 1, 2, _]] # d #print W ''' prng = RandomState() # alternative to random.seed w = prng.randint(0, 6, size=16) # inflating 1D array to 2D square matrix W = w.reshape(4, 4) #pprint(W) #print W[a][d] W = np.array(W) W_symm = (W + W.T)/2 # making the matrix symmetric np.fill_diagonal(W_symm, 0) return W_symm
def align_converge(y_LR,size=64): """iterate until offsets converge""" (h,w) = y_LR.shape # split image y_L = y_LR[:,:w/2] y_R = y_LR[:,w/2:] (h,w) = y_L.shape s = size / 2 # now find n offsets rand = RandomState(0) prev_dx, prev_dy = 0, 0 series = [] while True: # at a random locations in y_L y = rand.randint(h/4,h*3/4) x = rand.randint(w/4,w*3/4) it = y_L[y:y+s,x:x+s] # take an s x s chunk there tm = match_template(y_R,it) # match it against y_R ry, rx = maximum_position(tm) # max value is location series += [((y-ry), (x-rx))] # accumulatea print series n = len(series) if n % 2 == 0: # take the median dy, dx = np.median(np.asarray(series),axis=0).astype(int) if n > 100 or (abs(dy-prev_dy) == 0 and abs(dx-prev_dx) == 0): return dy, dx prev_dy, prev_dx = dy, dx
class RandomGenerator(object): def __init__(self, seed=None): self._random = RandomState(seed=seed) def random(self): return self._random.rand() def randint(self, a, b=None): if b is None: b = a a = 0 r = self._random.randint(a, high=b, size=1) return r[0] def sample(self, population, k): if k == 0: return [] return self._random.choice(population, size=k, replace=False) def __getattr__(self, attr): return getattr(self._random, attr) def __getstate__(self): return {'_random': self._random} def __setstate__(self, d): self._random = d['_random']
def mkeindseed(self,desc_in_ind=5): if self.mkeindseed.count<=100: prng=RandomState(self.seed+self.mkeindseed.count) if self.mkeindseed.count>100: prng=RandomState(self.seed+(self.mkeindseed.count%100)) smple=prng.choice(self.basetable.columns,size=desc_in_ind, replace=False) return list(smple)
def setupSeed(hoursBetweenTimestepInROMSFiles,startTime,endTime,startSpawningTime,endSpawningTime,releaseParticles): ################################################## # Create seed variation as function of day ################################################## # Make datetime array from start to end at 3 hour interval difference=endTime-startTime hoursOfSimulation=divmod(difference.total_seconds(), 3600) difference=endSpawningTime-startSpawningTime hoursOfSpawning=divmod(difference.total_seconds(), 3600) timeStepsSimulation=int(int(hoursOfSimulation[0])/hoursBetweenTimestepInROMSFiles) print "\nKINO TIME EVOLUTION:" print "=>SIMULATION: Drift simulation will run for %s simulation hours" %(timeStepsSimulation) print "=>SPAWNING: Simulated spawning will run for %s simulation hours\n initiated on %s and ending on %s"%(timeStepsSimulation,startSpawningTime,endSpawningTime) interval = timedelta(hours=24) hoursPerSpawning=divmod(interval.total_seconds(), 3600) #hours per spawning event timeStepsSpawning=int(int(hoursOfSpawning[0])/int(hoursPerSpawning[0])) #number of spawning timesteps spawningTimes = [startSpawningTime + interval*n for n in range(timeStepsSpawning)] #times of spawning # Normal distribution around 0.5 mu, sigma = 0.5, 0.1 # mean and standard deviation prng = RandomState(1) # random number generator (specify number to ensure the same sequence each time) s = prng.normal(mu, sigma, len(spawningTimes)) # random distribution num=(s*releaseParticles).astype(int) # number of particles released per spawning event print num #timeStepsSpawning * releaseParticles * mu gives approx. number of particles released num=np.sort(num) #sort particles in increasing order num=np.concatenate((num[len(num)%2::2],num[::-2]),axis=0) #release the highest number of particles at the midpoint of the spawning period print "SPAWNING: Simulated spawning will release %s eggs"%(np.sum(num)) return num, spawningTimes
def randomRotations(D, rng=None): from math import pi, sin, cos, sqrt from numpy.random import RandomState from itertools import product as cartesian, repeat from .util import frange if not isinstance(rng, RandomState): rng = RandomState(rng) if D == 2: while True: yield (rng.uniform(-pi, pi),) elif D == 3: # Ken Shoemake # Graphics Gems III, pp 124-132 from .quaternion import Quaternion while True: X = rng.uniform(0, 1) theta = rng.uniform(0, 2*pi), rng.uniform(0, 2*pi) R = (sqrt(1-X), sqrt(X)) yield Quaternion(sin(theta[0]) * R[0], cos(theta[0]) * R[0], sin(theta[1]) * R[1], cos(theta[1]) * R[1]).axis_angle else: raise NotImplementedError("Only defined for D in [2..3], not {}" .format(D))
class GaussianY(PhaseSpace): """Vertical Gaussian particle phase space distribution.""" def __init__(self, sigma_y, sigma_yp, generator_seed=None): """Initiates the vertical beam coordinates to the given Gaussian shape. """ self.sigma_y = sigma_y self.sigma_yp = sigma_yp self.random_state = RandomState() self.random_state.seed(generator_seed) @classmethod def from_optics(cls, alpha_y, beta_y, epsn_y, betagamma, generator_seed=None): """Initialise GaussianY from the given optics functions. beta_y is given in meters and epsn_y in micrometers. """ sigma_y = np.sqrt(beta_y * epsn_y * 1e-6 / betagamma) sigma_yp = sigma_y / beta_y return cls(sigma_y, sigma_yp, generator_seed) def generate(self, beam): beam.y = self.sigma_y * self.random_state.randn(beam.n_macroparticles) beam.yp = self.sigma_yp * self.random_state.randn(beam.n_macroparticles)
def stitch(targets,images): mask = rois_mask(targets) # True where image data is gaps_mask = mask==False # True where infill needs to go # compute bounds relative to the camera field (x,y,w,h) = stitched_box(targets) uroi = img_as_float(stitch_raw(targets,images,(x,y,w,h))) # stitch with black infill # step 1: sparsely sample background mostly ignoring blob # compute gradient on both axes k = [[-3,-1,0,1,3], [-3,-1,0,1,3], [-3,-1,0,1,3], [-3,-1,0,1,3]] gy = convolve(uroi,k) gx = convolve(uroi,np.rot90(k)) # ignore all but low-gradient areas bg = (abs(gy+gx) < 0.2) & mask # step 2: remove less contiguous areas filter_size = max(2,int(max(h,w)/200)) mf = minimum_filter(bg*1,filter_size) # step 3: interpolate between samples z = inpaint(uroi*mf,mf==False) # step 4: subsample and re-interpolate to degrade artifacts in fill region random = RandomState(0) (h,w)=z.shape ng = random.rand(h,w) < 0.01 z2 = inpaint(z*ng,ng==False) # step 5: final composite roi = (z2 * gaps_mask) + uroi return (roi * 255).astype(np.uint8), mask
def make_trial(): try: # t0 = time() # args: # lenscale # nPassiveObs # rngstate params = request.json # works when ajax request contentType specified as "applications/json" # unpack random number generator rng = RandomState() rngstate = unpack_rngstate(params['rngstate']) rng.set_state(rngstate) lenscale = float(params['lenscale']) nPassiveObs = int(params['nPassiveObs']) thisTri = boe.make_trial(nPassiveObs, DOMAIN, lenscale, SIGVAR, NOISEVAR2, XSAM_BOUNDS, rng) resp = {'sample': thisTri['sample'].tolist(), 'xObs': thisTri['xObs'].flatten().tolist(), 'yObs': thisTri['yObs'].tolist(), 'iObs': thisTri['iObs'].tolist(), 'rngstate': pack_rngstate(rng.get_state())} except: raise ExperimentError('improper_inputs') # i don't like returning HTML to JSON requests... maybe should change this return jsonify(**resp)
class GaussianX(PhaseSpace): """Horizontal Gaussian particle phase space distribution.""" def __init__(self, sigma_x, sigma_xp, generator_seed=None): """Initiates the horizontal beam coordinates to the given Gaussian shape. """ self.sigma_x = sigma_x self.sigma_xp = sigma_xp self.random_state = RandomState() self.random_state.seed(generator_seed) @classmethod def from_optics(cls, alpha_x, beta_x, epsn_x, betagamma, generator_seed=None): """Initialise GaussianX from the given optics functions. beta_x is given in meters and epsn_x in micrometers. """ sigma_x = np.sqrt(beta_x * epsn_x * 1e-6 / betagamma) sigma_xp = sigma_x / beta_x return cls(sigma_x, sigma_xp, generator_seed) def generate(self, beam): beam.x = self.sigma_x * self.random_state.randn(beam.n_macroparticles) beam.xp = self.sigma_xp * self.random_state.randn(beam.n_macroparticles)
def shared_normal(shape, scale=1, rng=None, name=None): """Initialize a matrix shared variable with normally distributed elements.""" if rng is None: rng = RandomState(seed=np.random.randint(1 << 30)) return theano.shared(rng.normal( scale=scale, size=shape).astype(theano.config.floatX), name=name)
y_predict = tf.matmul(x, w1) # 自定义损失函数 loss_less = 10 loss_more = 1 loss = tf.reduce_sum( tf.where(tf.greater(y, y_predict), (y - y_predict) * loss_more, (y_predict - y) * loss_less)) # 优化 trian_step = tf.train.AdamOptimizer(0.0001).minimize(loss) # 随机生成模拟数据集 rdm = RandomState(1) data_size = 128 X = rdm.rand(data_size, 2) # 128个样本 # 加入噪音 Y = [[x1 + x2 + rdm.rand() / 10.0 - 0.05] for (x1, x2) in X] # 训练 with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) for i in range(5000): start = (i * batch_size) % data_size end = min(start + batch_size, data_size)
from utils.loading import get_preproc_data, get_training_data from utils.postprocess import scale_to_priors from lasagne.layers import InputLayer, DenseLayer, DropoutLayer from lasagne.objectives import Objective from lasagne.updates import nesterov_momentum from lasagne.init import HeUniform, Constant from lasagne.nonlinearities import LeakyRectify, softmax from functools import partial from validation.crossvalidate import SampleCrossValidator from validation.optimize_parallel import ParallelGridOptimizer train_data, true_labels = get_training_data()[:2] true_labels -= 1 cpus = max(cpu_count() - 1, 1) random = RandomState() def train_test(train, labels, test, weight_decay): net = NeuralNet( layers=[ ('input', InputLayer), ('dropout0', DropoutLayer), ('dense1', DenseLayer), ('dropout1', DropoutLayer), ('dense2', DenseLayer), ('dropout2', DropoutLayer), ('dense3', DenseLayer), ('dropout3', DropoutLayer), ('output', DenseLayer), ],
def extract_f0_func(gender): floor_sp, ceil_sp = -80, 30 mel_basis = mel(16000, 1024, fmin=90, fmax=7600, n_mels=80).T min_level = np.exp(-100 / 20 * np.log(10)) b, a = butter_highpass(30, 16000, order=5) # Set the directory you want to start from ROOT = r'E:\Dataset\VCTK\test_audio' rootDir = os.path.join(ROOT, 'audio') targetDir_f0 = os.path.join(ROOT, 'f0') targetDir = os.path.join(ROOT, 'mel-sp') pt = glob.glob1(rootDir, '*') cep_all = [] dirName, subdirList, _ = next(os.walk(rootDir)) print('Found directory: %s' % dirName) for subdir in sorted(pt): print(subdir) if not os.path.exists(os.path.join(targetDir, subdir)): os.makedirs(os.path.join(targetDir, subdir)) if not os.path.exists(os.path.join(targetDir_f0, subdir)): os.makedirs(os.path.join(targetDir_f0, subdir)) _, _, fileList = next(os.walk(os.path.join(dirName, subdir))) if gender == 'M': lo, hi = 50, 250 elif gender == 'F': lo, hi = 100, 600 else: raise ValueError prng = RandomState(0) for fileName in sorted(fileList): print(subdir, fileName) x, fs = sf.read(os.path.join(dirName, subdir, fileName)) if (len(x.shape) >= 2): x = x[:, 0] if x.shape[0] % 256 == 0: x = np.concatenate((x, np.array([1e-06])), axis=0) y = signal.filtfilt(b, a, x) wav = y * 0.95 + (prng.rand(y.shape[0]) - 0.5) * 1e-06 D = pySTFT(wav).T D_mel = np.dot(D, mel_basis) D_db = 20 * np.log10(np.maximum(min_level, D_mel)) - 16 S = (D_db + 100) / 100 f0_rapt = sptk.rapt(wav.astype(np.float32) * 32768, fs, 256, min=lo, max=hi, otype=2) index_nonzero = (f0_rapt != -1e10) tmp = f0_rapt[index_nonzero] mean_f0, std_f0 = np.mean(tmp), np.std(tmp) f0_norm = speaker_normalization(f0_rapt, index_nonzero, mean_f0, std_f0) if len(S) != len(f0_norm): pdb.set_trace() np.save(os.path.join(targetDir, subdir, fileName[:-4]), S.astype(np.float32), allow_pickle=False) np.save(os.path.join(targetDir_f0, subdir, fileName[:-4]), f0_norm.astype(np.float32), allow_pickle=False) print(S.shape) print(f0_norm.shape)
import numpy as np from electromorpho.core.gaussian import sample_from_gn from electromorpho.metrics.score import BGe from numpy.random import RandomState from electromorpho.structure.graph_generation import random_dag from electromorpho.mcmc.graphs.state_space import RestrictionViolation from electromorpho.mcmc.graphs.checks import check_distribution from electromorpho.mcmc.graphs.proposal import get_parent_set_distributions n_variables = 15 seeds = list(range(101, 200)) rng = RandomState(19023) variables = list(range(n_variables)) n_samples = 200 # Data generation parameters gen_mean = np.zeros(n_variables) gen_var = np.zeros(n_variables) + 0.2 gen_weight = 2 # Generate some data form a GN graph = random_dag(variables, rng=rng) beta = graph.A.T * gen_weight sample_seed = rng.randint(0, 2**32 - 1) data_gn = sample_from_gn(graph, gen_mean, gen_var, beta, n_samples, sample_seed) # Fit the score and create the parent set distributions fan_in = 5
def __init__(self, bullet_client, offset=(0, 0, 0), sim_time=0., scale=1., parameter_distributions=None): self.logger = logging.Logger(f"robot:panda:{bullet_client}") if parameter_distributions is not None: logging.warning("Domain randomization not implemented for IIWA") raise NotImplementedError() self.time_step = bullet_client.getPhysicsEngineParameters( )["fixedTimeStep"] if not sim_time: sim_time = self.time_step self.scale = scale self.max_steps = int(sim_time / self.time_step) self.offset = offset self.bullet_client = bullet_client self.random = RandomState( int.from_bytes(os.urandom(4), byteorder='little')) # load robot in simulation urdf_file = os.path.join(str(Path(__file__).absolute().parent), "iiwa.urdf") self.model_id = bullet_client.loadURDF( urdf_file, np.array([0, 0, 0]) + self.offset, useFixedBase=True, flags=p.URDF_USE_SELF_COLLISION | p.URDF_MAINTAIN_LINK_ORDER) self.joint_name2id = {} for jj in range(self.bullet_client.getNumJoints(self.model_id)): jointInfo = self.bullet_client.getJointInfo(self.model_id, jj) self.joint_name2id[jointInfo[1].decode("utf-8")] = jointInfo[0] print(jointInfo[1].decode("utf-8")) Joint = namedtuple( "Joint", ["id", "initial_position", "limits", "max_velocity", "max_torque"]) # self.joints = { # 0: Joint(0, (-2.96705972839, 2.96705972839), 1.71042266695, 320), # 1: Joint(0, (-2.09439510239, 2.09439510239), 1.71042266695, 320), # 2: Joint(0, (-2.96705972839, 2.96705972839), 1.74532925199, 176), # 3: Joint(0, (-2.09439510239, 2.09439510239), 2.26892802759, 176), # 4: Joint(0, (-2.96705972839, 2.96705972839), 2.44346095279, 110), # 5: Joint(0, (-2.09439510239, 2.09439510239), 3.14159265359, 40), # 6: Joint(0, (-3.05432619099, 3.05432619099), 3.14159265359, 40), self.joints_arm = { "lbr_iiwa_joint_1": Joint(self.joint_name2id["lbr_iiwa_joint_1"], 0, (-2.96705972839, 2.96705972839), 1.71042266695, 320), "lbr_iiwa_joint_2": Joint(self.joint_name2id["lbr_iiwa_joint_2"], 0, (-2.09439510239, 2.09439510239), 1.71042266695, 320), "lbr_iiwa_joint_3": Joint(self.joint_name2id["lbr_iiwa_joint_3"], 0, (-2.96705972839, 2.96705972839), 1.74532925199, 176), "lbr_iiwa_joint_4": Joint(self.joint_name2id["lbr_iiwa_joint_4"], 0, (-2.09439510239, 2.09439510239), 2.26892802759, 176), "lbr_iiwa_joint_5": Joint(self.joint_name2id["lbr_iiwa_joint_5"], 0, (-2.96705972839, 2.96705972839), 2.44346095279, 110), "lbr_iiwa_joint_6": Joint(self.joint_name2id["lbr_iiwa_joint_6"], 0, (-2.09439510239, 2.09439510239), 3.14159265359, 40), "lbr_iiwa_joint_7": Joint(self.joint_name2id["lbr_iiwa_joint_7"], 0, (-3.05432619099, 3.05432619099), 3.14159265359, 40), } self.joints_hand = { # hand "left_outer_knuckle_joint": Joint(self.joint_name2id["left_outer_knuckle_joint"], 0.3, (0.0, 0.725), 2., 15), "left_inner_knuckle_joint": Joint(self.joint_name2id["left_inner_knuckle_joint"], 0.3, (0.0, 0.8757), 2., 15), "right_outer_knuckle_joint": Joint(self.joint_name2id["right_outer_knuckle_joint"], 0.3, (0.0, 0.725), 2., 15), "right_inner_knuckle_joint": Joint(self.joint_name2id["right_inner_knuckle_joint"], 0.3, (0.0, 0.8757), 2., 15), } self.joints_passive = { # hand "left_outer_finger_joint": Joint(self.joint_name2id["left_outer_finger_joint"], 0.3, (0.0, 0.725), 2., 20), "left_inner_finger_joint": Joint(self.joint_name2id["left_inner_finger_joint"], 0.3, (0.0, 0.8757), 2., 20), "right_outer_finger_joint": Joint(self.joint_name2id["right_outer_finger_joint"], 0.3, (0.0, 0.725), 2., 20), "right_inner_finger_joint": Joint(self.joint_name2id["right_inner_finger_joint"], 0.3, (0.0, 0.8757), 2., 20), } constraint = self.bullet_client.createConstraint( self.model_id, self.joint_name2id["left_inner_finger_loop"], self.model_id, self.joint_name2id["left_outer_finger_loop"], self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0], [0, 0, 0]) self.bullet_client.changeConstraint(constraint, maxForce=1e4) constraint = self.bullet_client.createConstraint( self.model_id, self.joint_name2id["right_inner_finger_loop"], self.model_id, self.joint_name2id["right_outer_finger_loop"], self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0], [0, 0, 0]) self.bullet_client.changeConstraint(constraint, maxForce=1e4) from itertools import chain, permutations for linkA, linkB in permutations( chain(self.joints_hand.keys(), self.joints_passive.keys()), 2): self.bullet_client.setCollisionFilterPair( self.model_id, self.model_id, self.joint_name2id[linkA], self.joint_name2id[linkB], False) self.bullet_client.setCollisionFilterPair( self.model_id, self.model_id, self.joint_name2id["left_inner_finger_joint"], self.joint_name2id["right_inner_finger_joint"], False) # todo introduce friction # self.bullet_client.setJointMotorControlArray(self.model_id, # [joint.id for _, joint in self.joints_arm.items()], # p.VELOCITY_CONTROL, # forces=[0.1 * joint.max_torque # for _, joint in self.joints_arm.items()]) self.bullet_client.setJointMotorControlArray( self.model_id, [joint.id for _, joint in self.joints_hand.items()], p.VELOCITY_CONTROL, forces=[ 0 * joint.max_torque for _, joint in self.joints_hand.items() ]) self.bullet_client.setJointMotorControlArray( self.model_id, [joint.id for _, joint in self.joints_passive.items()], p.VELOCITY_CONTROL, forces=[ 0 * joint.max_torque for _, joint in self.joints_passive.items() ]) self.bullet_client.stepSimulation() # define spaces self.action_space = spaces.Box(-1., 1., shape=(len(self.joints_arm) + 1, )) self.observation_space = spaces.Dict({ "joint_positions": spaces.Box(-1., 1., shape=(len(self.joints_arm), )), "joint_velocities": spaces.Box(-1., 1., shape=(len(self.joints_arm), )), "tcp_position": spaces.Box(-1., 1., shape=(3, )), # "tcp_velocity": spaces.Box(-1., 1., shape=(3,)), }) # reset to initial position self.reset()
class IIWA: def __init__(self, bullet_client, offset=(0, 0, 0), sim_time=0., scale=1., parameter_distributions=None): self.logger = logging.Logger(f"robot:panda:{bullet_client}") if parameter_distributions is not None: logging.warning("Domain randomization not implemented for IIWA") raise NotImplementedError() self.time_step = bullet_client.getPhysicsEngineParameters( )["fixedTimeStep"] if not sim_time: sim_time = self.time_step self.scale = scale self.max_steps = int(sim_time / self.time_step) self.offset = offset self.bullet_client = bullet_client self.random = RandomState( int.from_bytes(os.urandom(4), byteorder='little')) # load robot in simulation urdf_file = os.path.join(str(Path(__file__).absolute().parent), "iiwa.urdf") self.model_id = bullet_client.loadURDF( urdf_file, np.array([0, 0, 0]) + self.offset, useFixedBase=True, flags=p.URDF_USE_SELF_COLLISION | p.URDF_MAINTAIN_LINK_ORDER) self.joint_name2id = {} for jj in range(self.bullet_client.getNumJoints(self.model_id)): jointInfo = self.bullet_client.getJointInfo(self.model_id, jj) self.joint_name2id[jointInfo[1].decode("utf-8")] = jointInfo[0] print(jointInfo[1].decode("utf-8")) Joint = namedtuple( "Joint", ["id", "initial_position", "limits", "max_velocity", "max_torque"]) # self.joints = { # 0: Joint(0, (-2.96705972839, 2.96705972839), 1.71042266695, 320), # 1: Joint(0, (-2.09439510239, 2.09439510239), 1.71042266695, 320), # 2: Joint(0, (-2.96705972839, 2.96705972839), 1.74532925199, 176), # 3: Joint(0, (-2.09439510239, 2.09439510239), 2.26892802759, 176), # 4: Joint(0, (-2.96705972839, 2.96705972839), 2.44346095279, 110), # 5: Joint(0, (-2.09439510239, 2.09439510239), 3.14159265359, 40), # 6: Joint(0, (-3.05432619099, 3.05432619099), 3.14159265359, 40), self.joints_arm = { "lbr_iiwa_joint_1": Joint(self.joint_name2id["lbr_iiwa_joint_1"], 0, (-2.96705972839, 2.96705972839), 1.71042266695, 320), "lbr_iiwa_joint_2": Joint(self.joint_name2id["lbr_iiwa_joint_2"], 0, (-2.09439510239, 2.09439510239), 1.71042266695, 320), "lbr_iiwa_joint_3": Joint(self.joint_name2id["lbr_iiwa_joint_3"], 0, (-2.96705972839, 2.96705972839), 1.74532925199, 176), "lbr_iiwa_joint_4": Joint(self.joint_name2id["lbr_iiwa_joint_4"], 0, (-2.09439510239, 2.09439510239), 2.26892802759, 176), "lbr_iiwa_joint_5": Joint(self.joint_name2id["lbr_iiwa_joint_5"], 0, (-2.96705972839, 2.96705972839), 2.44346095279, 110), "lbr_iiwa_joint_6": Joint(self.joint_name2id["lbr_iiwa_joint_6"], 0, (-2.09439510239, 2.09439510239), 3.14159265359, 40), "lbr_iiwa_joint_7": Joint(self.joint_name2id["lbr_iiwa_joint_7"], 0, (-3.05432619099, 3.05432619099), 3.14159265359, 40), } self.joints_hand = { # hand "left_outer_knuckle_joint": Joint(self.joint_name2id["left_outer_knuckle_joint"], 0.3, (0.0, 0.725), 2., 15), "left_inner_knuckle_joint": Joint(self.joint_name2id["left_inner_knuckle_joint"], 0.3, (0.0, 0.8757), 2., 15), "right_outer_knuckle_joint": Joint(self.joint_name2id["right_outer_knuckle_joint"], 0.3, (0.0, 0.725), 2., 15), "right_inner_knuckle_joint": Joint(self.joint_name2id["right_inner_knuckle_joint"], 0.3, (0.0, 0.8757), 2., 15), } self.joints_passive = { # hand "left_outer_finger_joint": Joint(self.joint_name2id["left_outer_finger_joint"], 0.3, (0.0, 0.725), 2., 20), "left_inner_finger_joint": Joint(self.joint_name2id["left_inner_finger_joint"], 0.3, (0.0, 0.8757), 2., 20), "right_outer_finger_joint": Joint(self.joint_name2id["right_outer_finger_joint"], 0.3, (0.0, 0.725), 2., 20), "right_inner_finger_joint": Joint(self.joint_name2id["right_inner_finger_joint"], 0.3, (0.0, 0.8757), 2., 20), } constraint = self.bullet_client.createConstraint( self.model_id, self.joint_name2id["left_inner_finger_loop"], self.model_id, self.joint_name2id["left_outer_finger_loop"], self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0], [0, 0, 0]) self.bullet_client.changeConstraint(constraint, maxForce=1e4) constraint = self.bullet_client.createConstraint( self.model_id, self.joint_name2id["right_inner_finger_loop"], self.model_id, self.joint_name2id["right_outer_finger_loop"], self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0], [0, 0, 0]) self.bullet_client.changeConstraint(constraint, maxForce=1e4) from itertools import chain, permutations for linkA, linkB in permutations( chain(self.joints_hand.keys(), self.joints_passive.keys()), 2): self.bullet_client.setCollisionFilterPair( self.model_id, self.model_id, self.joint_name2id[linkA], self.joint_name2id[linkB], False) self.bullet_client.setCollisionFilterPair( self.model_id, self.model_id, self.joint_name2id["left_inner_finger_joint"], self.joint_name2id["right_inner_finger_joint"], False) # todo introduce friction # self.bullet_client.setJointMotorControlArray(self.model_id, # [joint.id for _, joint in self.joints_arm.items()], # p.VELOCITY_CONTROL, # forces=[0.1 * joint.max_torque # for _, joint in self.joints_arm.items()]) self.bullet_client.setJointMotorControlArray( self.model_id, [joint.id for _, joint in self.joints_hand.items()], p.VELOCITY_CONTROL, forces=[ 0 * joint.max_torque for _, joint in self.joints_hand.items() ]) self.bullet_client.setJointMotorControlArray( self.model_id, [joint.id for _, joint in self.joints_passive.items()], p.VELOCITY_CONTROL, forces=[ 0 * joint.max_torque for _, joint in self.joints_passive.items() ]) self.bullet_client.stepSimulation() # define spaces self.action_space = spaces.Box(-1., 1., shape=(len(self.joints_arm) + 1, )) self.observation_space = spaces.Dict({ "joint_positions": spaces.Box(-1., 1., shape=(len(self.joints_arm), )), "joint_velocities": spaces.Box(-1., 1., shape=(len(self.joints_arm), )), "tcp_position": spaces.Box(-1., 1., shape=(3, )), # "tcp_velocity": spaces.Box(-1., 1., shape=(3,)), }) # reset to initial position self.reset() def reset(self, desired_state=None): """Reset robot to initial pose and return new state.""" contact_points = True if desired_state is not None: desired_state = list(desired_state) for _, joint in self.joints_arm.items(): joint_position = np.interp(desired_state.pop(0), [-1, 1], joint.limits) self.bullet_client.resetJointState(self.model_id, joint.id, joint_position) self.bullet_client.stepSimulation() contact_points = self.bullet_client.getContactPoints( self.model_id, self.model_id) # reset until state is valid while contact_points: for _, joint in self.joints_hand.items(): joint_position = self.random.uniform(*joint.limits) self.bullet_client.resetJointState(self.model_id, joint.id, joint_position) self.bullet_client.stepSimulation() contact_points = self.bullet_client.getContactPoints( self.model_id, self.model_id) observation = self.get_observation() return observation def step(self, action: np.ndarray): assert self.action_space.contains(action), f"{action}" action_arm = action[:6] action_hand = action[-1] action_arm = list(action_arm * self.scale) # / self.max_steps) joint_ids = [] target_arm_positions = [] maxVelocities = [] torques_arm = [] for (_, joint), action_joint in zip(self.joints_arm.items(), action_arm): position, _, _, _ = self.bullet_client.getJointState( self.model_id, joint.id) normalized_joint_position = np.interp(position, joint.limits, [-1, 1]) normalized_target_joint_position = np.clip( normalized_joint_position + action_joint, -1, 1) target_joint_position = np.interp(normalized_target_joint_position, [-1, 1], joint.limits) joint_ids.append(joint.id) target_arm_positions.append(target_joint_position) torques_arm.append(joint.max_torque) maxVelocities.append(joint.max_velocity) # if joint.id == 2: # print(position, target_joint_position) # self.bullet_client.setJointMotorControl2(self.model_id, # joint.id, # p.POSITION_CONTROL, # targetPosition=target_joint_position, # maxVelocity=joint.max_velocity, # force=joint.max_torque # ) print(target_arm_positions) self.bullet_client.setJointMotorControlArray( self.model_id, joint_ids, p.POSITION_CONTROL, targetPositions=target_arm_positions, # maxVelocities=maxVelocities, forces=torques_arm) # joint_ids = [] # target_hand_positions = [] # torques_hand = [] # # for _, joint in self.joints_hand.items(): # # position, _, _, _ = self.bullet_client.getJointState(self.model_id, joint.id) # # normalized_joint_position = np.interp(position, joint.limits, [-1, 1]) # normalized_target_joint_position = np.clip(normalized_joint_position + action_hand, -1, 1) # target_joint_position = np.interp(normalized_target_joint_position, [-1, 1], joint.limits) # # joint_ids.append(joint.id) # target_hand_positions.append(target_joint_position) # torques_hand.append(joint.max_torque) # # self.bullet_client.setJointMotorControlArray(self.model_id, # joint_ids, # p.POSITION_CONTROL, # targetPositions=target_hand_positions, # forces=torques_hand # ) joint_ids = [joint.id for _, joint in self.joints_hand.items()] torques_fingers = [ action_hand * joint.max_torque for _, joint in self.joints_hand.items() ] for step in range(self.max_steps): self.bullet_client.setJointMotorControlArray( self.model_id, joint_ids, p.TORQUE_CONTROL, forces=torques_fingers) self.bullet_client.stepSimulation() if self.bullet_client.getConnectionInfo( )["connectionMethod"] == p.GUI: time.sleep(self.time_step) observation = self.get_observation() return observation def get_observation(self): joint_positions, joint_velocities = [], [] for _, joint in self.joints_arm.items(): joint_position, joint_velocity, _, _ = self.bullet_client.getJointState( self.model_id, joint.id) joint_positions.append( np.interp(joint_position, joint.limits, [-1, 1])) joint_velocities.append( np.interp(joint_velocity, [-joint.max_velocity, joint.max_velocity], [-1, 1])) tcp_position, _, _, _, _, _, tcp_velocity, _ = \ self.bullet_client.getLinkState(self.model_id, self.joint_name2id["tcp"], computeLinkVelocity=True) joint_positions = np.array(joint_positions) joint_velocities = np.array(joint_velocities) tcp_position = np.array(tcp_position) # tcp_velocity = np.array(tcp_velocity) observation = { "joint_positions": joint_positions, "joint_velocities": joint_velocities, "tcp_position": tcp_position, # "tcp_velocity": tcp_velocity } for key in observation: observation[key] = observation[key].clip( self.observation_space[key].low, self.observation_space[key].high) return observation
def main_box_plot(num_dim = 1, num_neurons = 2, analyze_sensors = True, analyze_brain = True, analyze_motors = False, combined_complexity = True, only_part_n1n2 = True, input_dir = 'data', csv_dir = None, plot_dir = None): """ Calculates neural complexity for different conditions, saves it to CSV and outputs boxplots. """ rs = RandomState(1) selected_nodes_str_list = [ n for n, b in zip( ['sensors', 'brain', 'motors'], [analyze_sensors, analyze_brain, analyze_motors] ) if b ] all_NC = [] if combined_complexity: dir_pop_index = [ (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_switch', 0), (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual', 0), ] x_labels = ['gen', 'spec'] else: dir_pop_index = [ (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill/', 0), (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_switch', 0), (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual', 0), (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual', 1) ] x_labels = ['iso', 'gen', 'spec-left', 'spec-right'] for in_dir, pop_index in dir_pop_index: SEEDS, GEN, BP, NC = get_seeds_generations_complexities( in_dir, analyze_sensors, analyze_brain, analyze_motors, pop_index, only_last_generation=True, filter_performance_threshold=20.0, combined_complexity=combined_complexity, only_part_n1n2=only_part_n1n2, rs=rs) NC = np.squeeze(NC) # print(NC) # print(NC.shape) all_NC.append(NC) all_NC = np.array(all_NC) # 4 x 20 print(all_NC.shape) selected_nodes_file_str = '_'.join([x[:3] for x in selected_nodes_str_list]) combined_str = '_combined' if combined_complexity else '' only_part_n1n2_str = '_onlyN1N2' if only_part_n1n2 else '' f_name = f"{num_dim}d_{num_neurons}n_box_TSE_{selected_nodes_file_str}{combined_str}{only_part_n1n2_str}" # save file to csv if csv_dir is not None: f_path = os.path.join( csv_dir, f_name + '.csv' ) print('saving csv:', f_path) df = pd.DataFrame(np.transpose(all_NC), columns=x_labels) # 20 x 4 df.to_csv(f_path, index=False) all_NC_not_NaN = [x[~np.isnan(x)] for x in all_NC] plt.boxplot(all_NC_not_NaN, labels=x_labels) selected_nodes_str = ', '.join(selected_nodes_str_list) title = f'Neural Complexity - {num_neurons}n {selected_nodes_str}' if combined_complexity: title += ' - combined' if only_part_n1n2: title += ' - onlyN1N2' plt.title(title) if plot_dir is not None: f_path = os.path.join( plot_dir, f_name + '.pdf' ) plt.savefig(f_path) # remove csv and add pdf print('saving pdf:', f_path) plt.clf() else: plt.show()
def main_line_plot(num_dim = 1, num_neurons = 2, sim_type = 'individuals', analyze_sensors = True, analyze_brain = True, analyze_motors = False, tse_max=2, combined_complexity = False, only_part_n1n2 = True, input_dir = 'data', plot_dir = None, csv_dir = None): """ Given a hard-coded directory with N simulation seeds, plots N line subplots, each containing 2 lines: - performance over generations - hardcoded complexity over generations """ sim_type_dir = { 'individuals': f'{num_dim}d_{num_neurons}n_exc-0.1_zfill', 'generalists': f'{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_switch', 'specialists': f'{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual' } assert sim_type in sim_type_dir, f'sim_type must be in {sim_type_dir.keys()}' exp_dir = sim_type_dir[sim_type] in_dir = os.path.join(input_dir, exp_dir) pop_index = 0 rs = RandomState(1) SEEDS, GEN, BP, NC = get_seeds_generations_complexities( in_dir, analyze_sensors, analyze_brain, analyze_motors, pop_index=pop_index, only_last_generation=False, filter_performance_threshold=20, # exclude not converged seeds combined_complexity=combined_complexity, only_part_n1n2=only_part_n1n2, rs=rs ) fig = plt.figure(figsize=(10, 6)) num_plots = len(GEN) num_plot_cols = 5 num_plot_rows = int(num_plots / num_plot_cols) if num_plots % num_plot_cols > 0: num_plot_rows += 1 if csv_dir is not None or plot_dir is not None: selected_nodes_str_list = [ n for n, b in zip( ['sensors', 'brain', 'motors'], [analyze_sensors, analyze_brain, analyze_motors] ) if b ] selected_nodes_file_str = '_'.join([x[:3] for x in selected_nodes_str_list]) combined_str = '_combined' if combined_complexity else '' only_part_n1n2_str = '_onlyN1N2' if only_part_n1n2 else '' fname = f'{num_dim}d_{num_neurons}n_gen_seeds_TSE_{sim_type}_{selected_nodes_file_str}{combined_str}{only_part_n1n2_str}' if csv_dir is not None: # save file to csv f_path = os.path.join( csv_dir, fname + '.csv' ) print('saving csv:', f_path) num_seeds, gen_list_size = np.array(NC).shape assert gen_list_size == len(GEN[0]) csv_data = np.transpose( # gen_list_size x num_seeds np.concatenate( [ np.array(GEN[0]).reshape((1,gen_list_size)), # need only 1 list of generations NC ] ) ) seeds_str = [f'seed_{s}' for s in SEEDS] assert len(seeds_str) == num_seeds cols_names = ['GEN'] + seeds_str df = pd.DataFrame(csv_data, columns=cols_names) df.to_csv(f_path, index=False) for seed_num, (num_gen_list, best_perfs, nc_seed) in enumerate(zip(GEN, BP, NC), 1): ax1 = fig.add_subplot(num_plot_rows, num_plot_cols, seed_num) ax1.plot(num_gen_list, nc_seed) # , label=str(seed_num) ax1.set_ylim(0, tse_max) ax2 = ax1.twinx() ax2.plot(np.arange(len(best_perfs)), best_perfs, color='orange') ax2.set_ylim(0, 100) # plt.legend() fig.tight_layout() selected_nodes_str = ', '.join(selected_nodes_str_list) title = f'Perf & TSE on generations - {num_neurons}n {sim_type} {selected_nodes_str}' if combined_complexity: title += ' - combined' if only_part_n1n2: title += ' - onlyN1N2' fig.suptitle(title) fig.subplots_adjust(top=0.88) if plot_dir is not None: f_path = os.path.join( plot_dir, fname + '.pdf' ) plt.savefig(f_path) # remove csv and add pdf print('saving pdf:', f_path) else: plt.show()
def min_objective(Xu, Xs, L, last_U, last_V, samples): global skipped_elements learn_rate = 0.05 nusers, nitems = Xu.shape _, nfeatures = Xs.shape nz_indices = np.nonzero(Xu) nz_elements = len(nz_indices[0]) print "non zero entries are: ", nz_elements rs = RandomState(1234567890) U = rs.standard_normal((nusers, dim)) U = normalize(U, norm='l2', axis=1) V = rs.standard_normal((nitems, dim)) V = normalize(V, norm='l2', axis=1) E = np.random.standard_normal((nitems, dim)) prev_V = np.array(V) prev_U = np.array(U) prev_E = np.array(E) skipped = 0 skip_i = [] skip_j = [] # W = np.random.uniform(-0.001, 0.001, (dim, nfeatures))/100 obj = 10 for itera in xrange(nIterations): i = samples[itera][0] j = samples[itera][1] # print "iteration: ", itera # if itera % 2 == 0: # ind = rd.choice(xrange(nz_elements)) # i = nz_indices[0][ind] # j = nz_indices[1][ind] # else: # i = rd.choice(xrange(nusers)) # j = rd.choice(xrange(nitems)) # i = rd.choice(xrange(nusers)) # j = rd.choice(xrange(nitems)) if Xu[i, j] > 0: y = 1 else: y = -1 if skipped_elements[i, j] == 1 or ( last_U != None and y * np.dot(last_U[i, :], last_V[j, :]) < -1): if itera > 1 and itera % 50000 == 0: obj = np.sum((U - prev_U)**2) + np.sum( (V - prev_V)**2) + np.sum((E - prev_E)**2) print "iterations: ", itera, " obj: ", obj prev_V = np.array(V) prev_U = np.array(U) prev_E = np.array(E) learn_rate /= 1.01 if obj < 0.01: break skipped += 1 skip_i.append(i) skip_j.append(j) continue # skipped_elements[i, j] = 1 t = y * np.dot(U[i, :], V[j, :]) if 1 - t > 0: gradu = -1 * y * V[j, :] gradv = -1 * y * U[i, :] else: gradu = 0 gradv = 0 # print "original: ", y # print "before: ", np.dot(U[i, :], V[j, :]) U[i, :] = U[i, :] - learn_rate * ( alpha * gradu) - learn_rate * lambd * 2 * U[i, :] # print "loss1: ", y*np.dot(U[i, :], V[j, :]), "loss2: ", np.linalg.norm(V[j, :]-L[j,:])**2 V[j, :] = V[j, :] - learn_rate * ( alpha * gradv + (1 - alpha) * (2 * V[j, :] - 2 * L[j, :])) - learn_rate * lambd * 2 * V[j, :] # E[j, :] = E[j, :] - learn_rate * ((1-alpha) * ( 2*np.dot(Xs[j, :].toarray(), W.T) - 2*np.dot((V[j,:]-E[j, :]), np.dot(W, W.T)))) - learn_rate * lambd_E * 2 * E[j, :] # V[j, :] = V[j, :] - learn_rate * (alpha * gradv) - learn_rate * lambd * 2 * V[j, :] # print "after: ", np.dot(U[i, :], V[j, :]) if itera > 1 and itera % 50000 == 0: obj = np.sum((U - prev_U)**2) + np.sum((V - prev_V)**2) + np.sum( (E - prev_E)**2) print "iterations: ", itera, " obj: ", obj prev_V = np.array(V) prev_U = np.array(U) prev_E = np.array(E) learn_rate /= 1.01 # learn_rate /= 1.0001 if obj < 0.01: break # W = W - learn_rate * ((1-alpha) *(-2*np.dot(V[j,:].reshape(1, -1).T, Xs[j, :].toarray().reshape(1, -1)) + 2* np.dot(np.dot(V[j,:].reshape(1, -1).T, V[j, :].reshape(1, -1)), W))) - learn_rate*lambd*2*W # if dump > 1: # print "before: ", Xu[i, j], dump # print "after: ", Xu[i, j], np.dot(U[i,:], V[j, :]) # print grad1u, grad1v, grad2u, grad2v # if itera % 1000 ==0: # su =0 # for k in range(10000): # i = rd.choice(xrange(nusers)) # j = rd.choice(xrange(nitems)) # if Xu[i, j] == 0: # y = -1 # else: # y = 1 # if math.isnan(np.dot(U[i,:], V[j, :])): # print U[i,:], V[j, :], np.dot(U[i,:], V[j, :]) # return # su+= np.dot(y*U[i,:], V[j, :]) # print "objective: ", su/10000.0 # # learn_rate /= 1.1 print "skipped: ", skipped skipped_elements[skip_i, skip_j] = 1 return U, V
import time from math import sqrt import numpy as np from numpy.random import RandomState from sklearn.datasets import fetch_olivetti_faces from modl.dict_fact import DictMF n_row, n_col = 3, 6 n_components = n_row * n_col image_shape = (64, 64) rng = RandomState(0) ############################################################################### # Load faces data dataset = fetch_olivetti_faces(shuffle=True, random_state=rng) faces = dataset.data n_samples, n_features = faces.shape # global centering faces_centered = faces - faces.mean(axis=0) # local centering faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1) faces_centered /= np.sqrt(np.sum(faces_centered**2, axis=1))[:, np.newaxis] print("Dataset consists of %d faces" % n_samples)
class Activities(): """ Generates the activity chains. """ ## Activity Activity = collections.namedtuple( 'Activity', ['activity', 'fromEdge', 'toEdge', 'arrivalPos', 'start', 'duration', 'final']) Activity.__new__.__defaults__ = (None,) * len(Activity._fields) def __init__(self, conf, sumo, environment, logger, profiling=False): """ Initialize the synthetic population. :param conf: distionary with the configurations :param sumo: already initialized SUMO simulation (TraCI or LibSUMO) :param profiling=False: enable cProfile """ self._conf = conf self._sumo = sumo self._cache = {} self._env = environment self.logger = logger self._max_retry_number = 1000 if 'maxNumTry' in conf: self._max_retry_number = conf['maxNumTry'] self._profiling = profiling self._random_generator = RandomState(seed=self._conf['seed']) # Activity Locations def _stages_define_locations_position(self, person_stages): """ Define the position of each location in the activity chain. """ home_pos = None primary_pos = None for pos, stage in person_stages.items(): if 'Home' in stage.activity: if not home_pos: home_pos = self._env.get_random_pos_from_edge(stage.toEdge) person_stages[pos] = stage._replace(arrivalPos=home_pos) elif 'P-' in stage.activity: if not primary_pos: primary_pos = self._env.get_random_pos_from_edge(stage.toEdge) person_stages[pos] = stage._replace(arrivalPos=primary_pos) else: ## Secondary activities person_stages[pos] = stage._replace( arrivalPos=self._env.get_random_pos_from_edge(stage.toEdge)) return person_stages def _stages_define_main_locations(self, from_area, to_area, mode): """ Define a generic Home and Primary activity location. The locations must be reachable in some ways. """ ## Mode split: _mode, _ptype, _vtype = sumoutils.get_intermodal_mode_parameters( mode, self._conf['intermodalOptions']['vehicleAllowedParking']) route = None from_edge = None to_edge = None _retry_counter = 0 while not route and _retry_counter < self._max_retry_number: _retry_counter += 1 ## Origin and Destination Selection from_edge, to_edge = self._env.select_pair(from_area, to_area) from_allowed = ( self._env.sumo_network.getEdge(from_edge).allows('pedestrian') and self._env.sumo_network.getEdge(from_edge).allows('passenger') and self._env.sumo_network.getEdge( from_edge).getLength() > self._conf['minEdgeAllowed']) to_allowed = ( self._env.sumo_network.getEdge(to_edge).allows('pedestrian') and self._env.sumo_network.getEdge(to_edge).allows('passenger') and self._env.sumo_network.getEdge(to_edge).getLength() > self._conf['minEdgeAllowed']) if self._env.valid_pair(from_edge, to_edge) and from_allowed and to_allowed: try: route = self._sumo.simulation.findIntermodalRoute( from_edge, to_edge, modes=_mode, pType=_ptype, vType=_vtype) if not sumoutils.is_valid_route( mode, route, self._conf['intermodalOptions']['vehicleAllowedParking']): route = None self.logger.debug( '_stages_define_main_locations: findIntermodalRoute mode unusable.') except TraCIException: self.logger.debug('_stages_define_main_locations: findIntermodalRoute FAILED.') route = None else: self.logger.debug('_stages_define_main_locations: unusable pair of edges.') if route: return from_edge, to_edge raise sagaexceptions.TripGenerationActivityError( 'Locations for the main activities not found between {} and {} using {}.'.format( from_area, to_area, mode)) def _stages_define_secondary_locations(self, person_stages, home, primary): """ Define secondary activity locations. """ for pos, stage in person_stages.items(): if 'S-' in stage.activity: ## look for what is coming before _prec = None _pos = pos - 1 while not _prec and _pos in person_stages: if 'Home' in person_stages[_pos].activity: _prec = 'H' elif 'P-' in person_stages[_pos].activity: _prec = 'P' _pos -= 1 ## look for what is coming next _succ = None _pos = pos + 1 while not _succ and _pos in person_stages: if 'Home' in person_stages[_pos].activity: _succ = 'H' elif 'P-' in person_stages[_pos].activity: _succ = 'P' _pos += 1 destination = None if _prec == 'H' and _succ == 'H': destination = self._random_location_circle(center=home, other=primary) elif _prec == 'P' and _succ == 'P': destination = self._random_location_circle(center=primary, other=home) elif _prec != _succ: destination = self._random_location_ellipse(home, primary) else: raise sagaexceptions.TripGenerationActivityError( 'Invalid sequence in the activity chain: {} --> {}'.format(_prec, _succ), person_stages) person_stages[pos] = stage._replace(toEdge=destination) return person_stages def _random_location_circle(self, center, other): """ Return a random edge in within a radius (*) from the given center. (*) Uses the ellipses defined by the foci center and other, and the major axe of 1.30 * distance between the foci. """ try: length = self._get_cached_dist(center, other) except TraCIException: raise sagaexceptions.TripGenerationActivityError( 'No route between {} and {}'.format(center, other)) major_axe = length * 1.3 minor_axe = numpy.sqrt(numpy.square(major_axe) - numpy.square(length)) radius = minor_axe / 2.0 self.logger.debug('_random_location_circle: %s [%.2f]', center, radius) edges = self._env.get_all_neigh_edges(center, radius) if not edges: raise sagaexceptions.TripGenerationActivityError( 'No edges from {} with range {}.'.format(center, length)) ret = self._random_generator.choice(edges) edges.remove(ret) allowed = ( self._env.sumo_network.getEdge(ret).allows('pedestrian') and self._env.sumo_network.getEdge(ret).allows('passenger') and ret != center and ret != other and self._env.sumo_network.getEdge(ret).getLength() > self._conf['minEdgeAllowed']) while edges and not allowed: ret = self._random_generator.choice(edges) edges.remove(ret) allowed = ( self._env.sumo_network.getEdge(ret).allows('pedestrian') and self._env.sumo_network.getEdge(ret).allows('passenger') and ret != center and ret != other and self._env.sumo_network.getEdge(ret).getLength() > self._conf['minEdgeAllowed']) if not edges: raise sagaexceptions.TripGenerationActivityError( 'No valid edges from {} with range {}.'.format(center, length)) return ret def _random_location_ellipse(self, focus1, focus2): """ Return a random edge in within the ellipse defined by the foci, and the major axe of 1.30 * distance between the foci. """ try: length = self._get_cached_dist(focus1, focus2) self.logger.debug('_random_location_ellipse: %s --> %s [%.2f]', focus1, focus2, length) except TraCIException: raise sagaexceptions.TripGenerationActivityError( 'No route between {} and {}'.format(focus1, focus2)) major_axe = length * 1.3 edges = self._env.get_all_neigh_edges(focus1, length) while edges: edge = self._random_generator.choice(edges) edges.remove(edge) if edge in (focus1, focus2): continue allowed = ( self._env.sumo_network.getEdge(edge).allows('pedestrian') and self._env.sumo_network.getEdge(edge).allows('passenger') and self._env.sumo_network.getEdge(edge).getLength() > self._conf['minEdgeAllowed']) if not allowed: continue try: first = self._get_cached_dist(focus1, edge) second = self._get_cached_dist(edge, focus2) if first + second <= major_axe: self.logger.debug('_random_location_ellipse: %s --> %s [%.2f]', focus1, edge, first) self.logger.debug( '_random_location_ellipse: %s --> %s [%.2f]', edge, focus2, second) return edge except TraCIException: pass raise sagaexceptions.TripGenerationActivityError( "No location available for _random_location_ellipse [{}, {}]".format(focus1, focus2)) # Chain def generate_person_stages(self, from_area, to_area, activity_chain, mode): """ Returns the trip for the given activity chain. """ # Define a generic Home and Primary activity location. from_edge, to_edge = self._stages_define_main_locations(from_area, to_area, mode) ## Generate preliminary stages for a person person_stages = dict() for pos, activity in enumerate(activity_chain): if activity not in self._conf['activities']: raise sagaexceptions.TripGenerationActivityError( 'Activity {} is not define in the config file.'.format(activity)) _start, _duration = self._get_timing_from_activity(activity) if pos == 0: if activity != 'Home': raise sagaexceptions.TripGenerationActivityError( "Every activity chain MUST start with 'Home', '{}' given.".format(activity)) ## Beginning person_stages[pos] = self.Activity( activity=activity, fromEdge=from_edge, start=_start, duration=_duration) elif 'P-' in activity: ## This is a primary activity person_stages[pos] = self.Activity( activity=activity, toEdge=to_edge, start=_start, duration=_duration) elif 'S-' in activity: ## This is a secondary activity person_stages[pos] = self.Activity( activity=activity, start=_start, duration=_duration) elif activity == 'Home': ## End of the activity chain. person_stages[pos] = self.Activity( activity=activity, toEdge=from_edge, start=_start, duration=_duration) if len(person_stages) <= 2: raise sagaexceptions.TripGenerationActivityError( "Invalid activity chain. (Minimal: H -> P-? -> H)", activity_chain) ## Define secondary activity location person_stages = self._stages_define_secondary_locations(person_stages, from_edge, to_edge) ## Remove the initial 'Home' stage and update the from of the second stage. person_stages[1] = person_stages[1]._replace(fromEdge=person_stages[0].fromEdge) if person_stages[0].start: person_stages[1] = person_stages[1]._replace(start=person_stages[0].stage) del person_stages[0] ## Fixing the 'from' field with a forward chain pos = 2 while pos in person_stages: person_stages[pos] = person_stages[pos]._replace(fromEdge=person_stages[pos-1].toEdge) pos += 1 ## Compute the real starting time for the activity chain based on ETT and durations start = self._stages_compute_start_time(person_stages, mode) person_stages[1] = person_stages[1]._replace(start=start) ## Define the position of each location in the activity chain. person_stages = self._stages_define_locations_position(person_stages) ## Final location consistency test last_edge = person_stages[1].toEdge pos = 2 while pos in person_stages: if person_stages[pos].fromEdge != last_edge: raise sagaexceptions.TripGenerationActivityError( 'Inconsistency in the locations for the chain of activities.', person_stages) last_edge = person_stages[pos].toEdge pos += 1 ## Set the final activity to True pos = 1 while pos in person_stages: person_stages[pos] = person_stages[pos]._replace(final=False) pos += 1 person_stages[pos-1] = person_stages[pos-1]._replace(final=True) return person_stages def _stages_compute_start_time(self, person_stages, mode): """ Compute the real starting time for the activity chain. """ ## Mode split: _mode, _ptype, _vtype = sumoutils.get_intermodal_mode_parameters( mode, self._conf['intermodalOptions']['vehicleAllowedParking']) # Find the first 'start' defined. pos = 1 while pos in person_stages: if person_stages[pos].start: break pos += 1 start = person_stages[pos].start while pos in person_stages: ett, route = None, None try: route = self._sumo.simulation.findIntermodalRoute( person_stages[pos].fromEdge, person_stages[pos].toEdge, modes=_mode, pType=_ptype, vType=_vtype) ett = sumoutils.ett_from_route(route) except TraCIException: raise sagaexceptions.TripGenerationRouteError( 'No solution foud for stage {} and modes {}.'.format( pformat(person_stages[pos]), mode)) if pos-1 in person_stages: if person_stages[pos-1].duration: ett += person_stages[pos-1].duration start -= ett pos -= 1 return start def _get_timing_from_activity(self, activity): """ Compute start and duration from the activity defined in the config file. """ start = None if self._conf['activities'][activity]['start']: start = self._random_generator.normal( loc=self._conf['activities'][activity]['start']['m'], scale=self._conf['activities'][activity]['start']['s']) if start < 0: return self._get_timing_from_activity(activity) duration = None if self._conf['activities'][activity]['duration']: duration = self._random_generator.normal( loc=self._conf['activities'][activity]['duration']['m'], scale=self._conf['activities'][activity]['duration']['s']) if duration <= 0: return self._get_timing_from_activity(activity) return start, duration def _get_cached_dist(self, orig, dest): cached = self._cache.get((orig, dest)) if cached is None: cached = self._sumo.simulation.findRoute(orig, dest).length self._cache[(orig, dest)] = cached return cached
def succesiveConditionalSimulatorWithInitialWeightAndData( self, initialParamAndData, K, nStates, nBivariateFeat, bivariateFeatDictionary, seed, bt, nSeq, interLength, HMCPlusBPS, onlyHMC, nLeapFrogSteps, stepSize, nItersPerPathAuxVar, rfOptions, mcmcOptions): thetaStationaryWeights = initialParamAndData['stationaryWeights'] thetaBinaryWeights = initialParamAndData['binaryWeights'] theta0StationarySamples = np.zeros((K, nStates)) theta0BinaryWeightsSamples = np.zeros((K, nBivariateFeat)) ## for debuging reasons, we ave the stationary distribution and exchangeable coef ## actually, we only need the last sample at the last iteration K theta0StationaryDistSamples = np.zeros((K, nStates)) theta0ExchangeableCoefSamples = np.zeros( (K, int(nStates * (nStates - 1) / 2))) theta0StationarySamples[0, :] = thetaStationaryWeights theta0BinaryWeightsSamples[0, :] = thetaBinaryWeights theta0StationaryDistSamples[ 0, :] = np.exp(thetaStationaryWeights) / np.sum( np.exp(thetaStationaryWeights)) initialExchangeCoef = getExchangeCoef(nStates, thetaBinaryWeights, bivariateFeatDictionary) theta0ExchangeableCoefSamples[0, :] = initialExchangeCoef sample = None if onlyHMC: sample = np.zeros((nStates + nBivariateFeat)) sample[0:nStates] = thetaStationaryWeights sample[nStates:(nStates + nBivariateFeat)] = thetaBinaryWeights if HMCPlusBPS: sample = thetaStationaryWeights suffStat = initialParamAndData['suffStat'] nInit = suffStat['nInit'] nTrans = suffStat['nTrans'] holdTime = suffStat['sojourn'] expectedCompleteReversibleObjective = None for i in np.arange(1, K, 1): if HMCPlusBPS: expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective( holdTime, nInit, nTrans, 1.0, initialExchangeCoef) if onlyHMC: expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective( holdTime, nInit, nTrans, 1.0, nBivariateFeatWeightsDictionary=bivariateFeatDictionary) ##################################### hmc = HMC(nLeapFrogSteps, stepSize, expectedCompleteReversibleObjective, expectedCompleteReversibleObjective) lastSample = sample for k in range(nItersPerPathAuxVar): hmcResult = hmc.doIter(nLeapFrogSteps, stepSize, lastSample, expectedCompleteReversibleObjective, expectedCompleteReversibleObjective, True) lastSample = hmcResult.next_q sample = lastSample if onlyHMC: thetaStationaryWeights = sample[0:nStates] thetaBinaryWeights = sample[nStates:(nStates + nBivariateFeat)] theta0StationarySamples[i, :] = thetaStationaryWeights theta0BinaryWeightsSamples[i, :] = thetaBinaryWeights if HMCPlusBPS: thetaStationaryWeights = sample theta0StationarySamples[i, :] = thetaStationaryWeights # update stationary distribution elements to the latest value thetaStationaryDist = np.exp(sample) / np.sum(np.exp(sample)) # sample exchangeable coefficients using local bouncy particle sampler ## define the model model = ExpectedCompleteReversibleModelWithBinaryFactors( expectedCompleteReversibleObjective, nStates, thetaBinaryWeights, thetaStationaryDist, bivariateFeatDictionary) ## define the sampler to use ## local sampler to use if i == 1: self.neighborVariablesForAllFactors = neighborVariableForAllFactors( nStates, model.localFactors, model.bivariateFeatIndexDictionary) self.variableAndFactorInfo = neighbourVariblesAndFactorsAndExtendedNeighborsOfAllFactorsDict( nStates, model.localFactors, model.bivariateFeatIndexDictionary, nBivariateFeat) self.indexOfFactorsForEachBivariateFeat = getIndexOfNeighborFactorsForEachIndexOfBinaryFeature( model.bivariateFeatIndexDictionary, nBivariateFeat, model.localFactors) localSampler = LocalRFSamplerForBinaryWeights( model, rfOptions, mcmcOptions, nStates, self.neighborVariablesForAllFactors, self.variableAndFactorInfo, self.indexOfFactorsForEachBivariateFeat) phyloLocalRFMove = PhyloLocalRFMove( model=model, sampler=localSampler, initialPoints=thetaBinaryWeights, options=rfOptions, prng=RandomState(i)) thetaBinaryWeights = phyloLocalRFMove.execute() theta0BinaryWeightsSamples[i, :] = thetaBinaryWeights initialRateMtx = ReversibleRateMtxPiAndBinaryWeightsWithGraphicalStructure( nStates, thetaStationaryWeights, thetaBinaryWeights, bivariateFeatIndexDictionary=bivariateFeatDictionary) initialStationaryDist = initialRateMtx.getStationaryDist() initialExchangeCoef = initialRateMtx.getExchangeCoef() theta0StationaryDistSamples[i, :] = initialStationaryDist theta0ExchangeableCoefSamples[i, :] = initialExchangeCoef weightGenerationRegime = WeightGenerationRegime( nStates=nStates, nBivariateFeat=nBivariateFeat, stationaryWeights=thetaStationaryWeights, bivariateWeights=thetaBinaryWeights) prng = RandomState(np.random.choice(2**32 - 1, 1)) dataRegime = DataGenerationRegime( nStates=nStates, bivariateFeatIndexDictionary=bivariateFeatDictionary, btLength=bt, nSeq=nSeq, weightGenerationRegime=weightGenerationRegime, prng=prng, interLength=interLength) ## generate the sequences data seqList = dataRegime.generatingSeq() suffStat = dataRegime.getSufficientStatFromSeq(seqList) firstLastStatesArrayAll = dataRegime.generatingSeqGivenRateMtxAndBtInterval( seqList) # replicate K iterations to get new parameters nTrans = suffStat["transitCount"] holdTime = suffStat["sojourn"] nInit = np.zeros(nStates) unique, counts = np.unique(firstLastStatesArrayAll[0][:, 0], return_counts=True) nInitCount = np.asarray((unique, counts)).T nInit[nInitCount[:, 0].astype(int)] = nInitCount[:, 1] result = {} result['stationaryDist'] = theta0StationaryDistSamples[(K - 1), :] result['exchangeCoef'] = theta0ExchangeableCoefSamples[(K - 1), :] result['stationaryWeights'] = thetaStationaryWeights result['binaryWeights'] = thetaBinaryWeights ## after testing the code is right, the following two lines should be removed #result['StationaryDistSamples'] = theta0StationaryDistSamples #result['ExchangeableCoefSamples'] = theta0ExchangeableCoefSamples return result
for directory in [targetDirSpec, targetDirPitch]: if os.path.exists(directory): shutil.rmtree(directory) os.makedirs(directory) #targetDirSpec = './spmel' dirName, subdirList, _ = next(os.walk(rootDir)) print('Found directory: %s' % dirName) for subdir_idx, subdir in enumerate(sorted(subdirList)): print(subdir) os.makedirs(os.path.join(targetDirSpec,subdir)) os.makedirs(os.path.join(targetDirPitch,subdir)) _,_, fileList = next(os.walk(os.path.join(dirName,subdir))) #prng = RandomState(int(subdir[1:])) prng = RandomState(1) for file_idx, fileName in enumerate(sorted(fileList)): # ensure that only mic1 files are processed if fileName.endswith('wav'): print(f'{subdir}, {subdir_idx}/{len(subdirList)}, {fileName}, {file_idx}/{len(fileList)}') # Read audio file audio, sr = sf.read(os.path.join(dirName,subdir,fileName)) # Remove drifting noise y = signal.filtfilt(b, a, audio) # Ddd a little random noise for model roubstness wav = y * 0.96 + (prng.rand(y.shape[0])-0.5)*1e-06 # resample 48kHz to 16kHz resampled_wav = librosa.resample(wav, sr, 16000) # pdb.set_trace() # compute pitch contour
def exponential_lik(X: ndarray, rng: RandomState): return rng.exponential(X)
# Pytorch expects float32 for input and int64 for labels. # X: trials x channels x timesteps X = FeatVect.astype(np.float32) X = X[:, :, :] y = (y_labels[:, 0] - 0).astype(np.int64) for CV in np.arange(0, num_folds): print('Subject No.{} CV {}'.format(i, CV)) # 5th phase: Model evaluation (test) train_set = SignalAndTarget(X, y=y) # 80% training, 20% test train_set, test_set = split_into_train_test( train_set, n_folds=num_folds, i_test_fold=CV, rng=RandomState((2019, 28, 6))) #RandomState((2019, 28, 6)) # 5% training, 95% test #test_set, train_set = split_into_train_test(train_set, n_folds = num_folds, i_test_fold = CV, rng=None) cuda_avail = th.cuda.is_available() set_random_seeds(seed=20190628, cuda=cuda_avail) n_classes = 2 in_chans = train_set.X.shape[1] # number of channels = 128 input_time_length = 150 # length of time of each epoch/trial = 4000 model = ShallowFBCSPNet( split_first_layer=False, in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length,
def plot_faces_decomposition(): # Display progress logs on stdout logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s') n_row, n_col = 2, 3 n_components = n_row * n_col image_shape = (64, 64) rng = RandomState(0) # ############################################################################# # Load faces data faces, _ = fetch_olivetti_faces(return_X_y=True, shuffle=True, random_state=rng) n_samples, n_features = faces.shape # global centering faces_centered = faces - faces.mean(axis=0) # local centering faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1) print("Dataset consists of %d faces" % n_samples) def plot_gallery(title, images, n_col=n_col, n_row=n_row, cmap=plt.cm.gray): plt.figure(figsize=(2. * n_col, 2.26 * n_row)) plt.suptitle(title, size=16) for i, comp in enumerate(images): plt.subplot(n_row, n_col, i + 1) vmax = max(comp.max(), -comp.min()) plt.imshow(comp.reshape(image_shape), cmap=cmap, interpolation='nearest', vmin=-vmax, vmax=vmax) plt.xticks(()) plt.yticks(()) plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.) # ############################################################################# # List of the different estimators, whether to center and transpose the # problem, and whether the transformer uses the clustering API. estimators = [ ('Eigenfaces - PCA using randomized SVD', decomposition.PCA(n_components=n_components, svd_solver='randomized', whiten=True), True), ('Non-negative components - NMF', decomposition.NMF(n_components=n_components, init='nndsvda', tol=5e-3), False), ('Independent components - FastICA', decomposition.FastICA(n_components=n_components, whiten=True), True), ('Sparse comp. - MiniBatchSparsePCA', decomposition.MiniBatchSparsePCA(n_components=n_components, alpha=0.8, n_iter=100, batch_size=3, random_state=rng), True), ('MiniBatchDictionaryLearning', decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, n_iter=50, batch_size=3, random_state=rng), True), ('Cluster centers - MiniBatchKMeans', MiniBatchKMeans(n_clusters=n_components, tol=1e-3, batch_size=20, max_iter=50, random_state=rng), True), ('Factor Analysis components - FA', decomposition.FactorAnalysis(n_components=n_components, max_iter=20), True), ] # ############################################################################# # Plot a sample of the input data plot_gallery("First centered Olivetti faces", faces_centered[:n_components]) # ############################################################################# # Do the estimation and plot it for name, estimator, center in estimators: print("Extracting the top %d %s..." % (n_components, name)) t0 = time() data = faces if center: data = faces_centered estimator.fit(data) train_time = (time() - t0) print("done in %0.3fs" % train_time) if hasattr(estimator, 'cluster_centers_'): components_ = estimator.cluster_centers_ else: components_ = estimator.components_ # Plot an image representing the pixelwise variance provided by the # estimator e.g its noise_variance_ attribute. The Eigenfaces estimator, # via the PCA decomposition, also provides a scalar noise_variance_ # (the mean of pixelwise variance) that cannot be displayed as an image # so we skip it. if (hasattr(estimator, 'noise_variance_') and estimator.noise_variance_.ndim > 0): # Skip the Eigenfaces case plot_gallery("Pixelwise variance", estimator.noise_variance_.reshape(1, -1), n_col=1, n_row=1) plot_gallery('%s - Train time %.1fs' % (name, train_time), components_[:n_components]) plt.show() # ############################################################################# # Various positivity constraints applied to dictionary learning. estimators = [ ('Dictionary learning', decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, n_iter=50, batch_size=3, random_state=rng), True), ('Dictionary learning - positive dictionary', decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, n_iter=50, batch_size=3, random_state=rng, positive_dict=True), True), ('Dictionary learning - positive code', decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, n_iter=50, batch_size=3, fit_algorithm='cd', random_state=rng, positive_code=True), True), ('Dictionary learning - positive dictionary & code', decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1, n_iter=50, batch_size=3, fit_algorithm='cd', random_state=rng, positive_dict=True, positive_code=True), True), ] # ############################################################################# # Plot a sample of the input data plot_gallery("First centered Olivetti faces", faces_centered[:n_components], cmap=plt.cm.RdBu) # ############################################################################# # Do the estimation and plot it for name, estimator, center in estimators: print("Extracting the top %d %s..." % (n_components, name)) t0 = time() data = faces if center: data = faces_centered estimator.fit(data) train_time = (time() - t0) print("done in %0.3fs" % train_time) components_ = estimator.components_ plot_gallery(name, components_[:n_components], cmap=plt.cm.RdBu) plt.show()
obj = csv.reader(file.read().decode('utf-8').splitlines()) title = False raw_data = [] for v in obj: if title == False: title = True continue if len(v) != 0: line_data = [float(v[8]), float(v[9]), float(v[10]), float(v[11])] raw_data.append(line_data) X = PCA(n_components=2).fit_transform(np.array(raw_data)) random_state = RandomState(42) n_samples = len(raw_data) kmeans = KMeans(n_clusters=3, random_state=random_state).fit(X) def plot_2D(data, target, target_names): colors = cycle('rgbcmykw') target_ids = range(len(target_names)) for i, c, label in zip(target_ids, colors, target_names): plt.scatter(data[target == i, 0], data[target == i, 1], c=c, label=label) pl.legend() pl.show()
#在shape的一个维度上使用None,可以方便使用大不的batch大小。在训练是需要把数据分成 #比较小的batch,但是在测试时,可以一次性使用全部的数据。当数据集你叫小的时候这样比较 #方便测试,但数据集比较大时,将大量数据放入一个batch可能会导致内存溢出。 x = tf.placeholder(tf.float32, shape=(None, 2), name='x-input') y_ = tf.placeholder(tf.float32, shape=(None, 1), name='y-input') #定义神经网络前向传播的过程 a = tf.matmul(x, w1) y = tf.matmul(a, w2) #定义损失函数和反向传播的算法 cross_entropy = -tf.reduce_mean(y_ * tf.log(tf.clip_by_value(y, 1e-10, 1.0))) train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy) #通过随机数生成一个模拟数据集 rdm = RandomState(1) dataset_size = 128 X = rdm.rand(dataset_size, 2) #定义规则来给出样本的标签,在这里所有x1+x2<1的样例都被认为是正样本(比如零件合格), #而其他为负样本(比如样本不合格)。和Tensorflow游乐场中的表示不大一样的地方是, #在这里使用0来表示负样本,1表示正样本。大部分解决分类问题的神经网络都会采用 #0和1的表示方法 Y = [[int(x1 + x2 < 1)] for (x1, x2) in X] #创建一个会话来运行tendorflow with tf.Session() as sess: init_op = tf.initialize_all_variables() sess.run(init_op) print("训练之前的") print(sess.run(w1)) print("-----------")
class BPMF(ModelBase): """Bayesian Probabilistic Matrix Factorization """ def __init__( self, n_user, n_item, n_feature, beta=2.0, beta_user=2.0, df_user=None, mu0_user=0., beta_item=2.0, df_item=None, mu0_item=0., converge=1e-5, seed=None, max_rating=None, # seed is useful for reproducibility. min_rating=None): super(BPMF, self).__init__() self.n_user = n_user self.n_item = n_item self.n_feature = n_feature self.rand_state = RandomState(seed) self.max_rating = float(max_rating) if max_rating is not None else None self.min_rating = float(min_rating) if min_rating is not None else None self.converge = converge # Hyper Parameter self.beta = beta # Inv-Whishart (User features) self.WI_user = np.eye(n_feature, dtype='float64') self.beta_user = beta_user self.df_user = int(df_user) if df_user is not None else n_feature self.mu0_user = np.repeat(mu0_user, n_feature).reshape(n_feature, 1) # a vector # Inv-Whishart (item features) self.WI_item = np.eye(n_feature, dtype='float64') self.beta_item = beta_item self.df_item = int(df_item) if df_item is not None else n_feature self.mu0_item = np.repeat(mu0_item, n_feature).reshape(n_feature, 1) # Latent Variables self.mu_user = np.zeros((n_feature, 1), dtype='float64') self.mu_item = np.zeros((n_feature, 1), dtype='float64') self.alpha_user = np.eye( n_feature, dtype='float64') # These are probably the Lambda matrices self.alpha_item = np.eye(n_feature, dtype='float64') self.user_features_ = 0.3 * self.rand_state.rand( n_user, n_feature) # initializes the user features randomly. Why 0.3?? self.item_features_ = 0.3 * self.rand_state.rand(n_item, n_feature) # data state self.mean_rating_ = None self.ratings_csr_ = None self.ratings_csc_ = None def fit(self, ratings, test_ratings=None, n_iters=50): """training models; I modified it such that if you also pass test_ratings, it computes the test RMSE at each step. """ check_ratings( ratings, self.n_user, self.n_item, # checks if ratings matrix is OK self.max_rating, self.min_rating) self.mean_rating_ = np.mean(ratings[:, 2]) # only two different ways of building the matrix. # csr user-item matrix for fast row access (user update) self.ratings_csr_ = build_user_item_matrix(self.n_user, self.n_item, ratings) # keep a csc matrix for fast col access (item update) self.ratings_csc_ = self.ratings_csr_.tocsc() # lists in where I will save the rmse during training train_rmse_list = [] if test_ratings is not None: test_rmse_list = [] # define the lists that will contain the total predictions for training and test set train_preds_total = np.zeros(len(ratings)).tolist() if test_ratings is not None: test_preds_total = np.zeros(len(test_ratings)).tolist() last_rmse = None for iteration in xrange(n_iters): logger.debug("iteration %d...", iteration) # THE FOLLOWING ARE THE GIBBS SAMPLING STEP # update item & user parameter self._update_item_params() self._update_user_params() # update item & user features self._udpate_item_features() self._update_user_features() # compute RMSE train_preds = self.predict(ratings[:, :2]) # in order to make a meaningful MCMC, you need to do this, to sample from the correct distribution; to do this, you have # to average over all previous predictions: train_preds_total = ((np.array(train_preds_total) * iteration + np.array(train_preds)) / (iteration + 1)).tolist() train_rmse = RMSE(train_preds_total, ratings[:, 2]) train_rmse_list.append(train_rmse) if test_ratings is not None: test_preds = self.predict(test_ratings[:, :2]) test_preds_total = ((np.array(test_preds_total) * iteration + np.array(test_preds)) / (iteration + 1)).tolist() test_rmse = RMSE(test_preds_total, test_ratings[:, 2]) test_rmse_list.append(test_rmse) logger.info("iter: %d, train RMSE: %.6f, test RMSE: %.6f", iteration, train_rmse, test_rmse) else: logger.info("iter: %d, train RMSE: %.6f", iteration, train_rmse) # stop when converge if last_rmse and abs( train_rmse - last_rmse ) < self.converge: # if you get convergence -> stop logger.info('converges at iteration %d. stop.', iteration) break else: last_rmse = train_rmse if test_ratings is not None: return train_rmse_list, test_rmse_list else: return train_rmse_list def fit_old(self, ratings, test_ratings=None, n_iters=50): """training models; I modified it such that if you also pass test_ratings, it computes the test RMSE at each step. This is wrong since it does not implemenent correctly the MCMC algorithm; infact, in order to use MCMC correctly, you need to average the predictions of every step to get a final prediction and estimate the RMSE, while in this function the RMSE was computed only on the prediction given by the current value of the feature matrices. Essentially, the predicted ratings are function on the whole chain of values of U and V. The purpose of MCMC chain is infact to estimate an integral by sampling from the correct distribution for U, V, and this is done by averaging over subsequently extracted samples. """ check_ratings( ratings, self.n_user, self.n_item, # checks if ratings matrix is OK self.max_rating, self.min_rating) self.mean_rating_ = np.mean(ratings[:, 2]) # only two different ways of building the matrix. # csr user-item matrix for fast row access (user update) self.ratings_csr_ = build_user_item_matrix(self.n_user, self.n_item, ratings) # keep a csc matrix for fast col access (item update) self.ratings_csc_ = self.ratings_csr_.tocsc() train_rmse_list = [] if test_ratings is not None: test_rmse_list = [] last_rmse = None for iteration in xrange(n_iters): logger.debug("iteration %d...", iteration) # THE FOLLOWING ARE THE GIBBS SAMPLING STEP # update item & user parameter self._update_item_params() self._update_user_params() # update item & user features self._udpate_item_features() self._update_user_features() # compute RMSE; NB: at each step, this code just predicts the R matrix using U, V but does not make the average over more # prediction steps, as it should. This is not how MCMC works!!! train_preds = self.predict(ratings[:, :2]) train_rmse = RMSE(train_preds, ratings[:, 2]) train_rmse_list.append(train_rmse) if test_ratings is not None: test_preds = self.predict(test_ratings[:, :2]) test_rmse = RMSE(test_preds, test_ratings[:, 2]) test_rmse_list.append(test_rmse) logger.info("iter: %d, train RMSE: %.6f, test RMSE: %.6f", iteration, train_rmse, test_rmse) else: logger.info("iter: %d, train RMSE: %.6f", iteration, train_rmse) # stop when converge if last_rmse and abs( train_rmse - last_rmse ) < self.converge: # if you get convergence -> stop logger.info('converges at iteration %d. stop.', iteration) break else: last_rmse = train_rmse if test_ratings is not None: return train_rmse_list, test_rmse_list else: return train_rmse_list def predict(self, data): if not self.mean_rating_: raise NotFittedError() u_features = self.user_features_.take(data.take(0, axis=1), axis=0) i_features = self.item_features_.take(data.take(1, axis=1), axis=0) preds = np.sum(u_features * i_features, 1) + self.mean_rating_ if self.max_rating: # cut the prediction rate. preds[preds > self.max_rating] = self.max_rating if self.min_rating: preds[preds < self.min_rating] = self.min_rating return preds def _update_item_params(self): N = self.n_item X_bar = np.mean(self.item_features_, 0).reshape((self.n_feature, 1)) # print 'X_bar', X_bar.shape S_bar = np.cov(self.item_features_.T) # print 'S_bar', S_bar.shape diff_X_bar = self.mu0_item - X_bar # W_{0}_star WI_post = inv( inv(self.WI_item) + N * S_bar + np.dot(diff_X_bar, diff_X_bar.T) * (N * self.beta_item) / (self.beta_item + N)) # Note: WI_post and WI_post.T should be the same. # Just make sure it is symmertic here WI_post = (WI_post + WI_post.T) / 2.0 # update alpha_item df_post = self.df_item + N self.alpha_item = wishart.rvs(df_post, WI_post, 1, self.rand_state) # update mu_item mu_mean = (self.beta_item * self.mu0_item + N * X_bar) / \ (self.beta_item + N) mu_var = cholesky(inv(np.dot(self.beta_item + N, self.alpha_item))) # print 'lam', lam.shape self.mu_item = mu_mean + np.dot( mu_var, self.rand_state.randn(self.n_feature, 1)) # print 'mu_item', self.mu_item.shape def _update_user_params(self): # same as _update_user_params N = self.n_user X_bar = np.mean(self.user_features_, 0).reshape((self.n_feature, 1)) S_bar = np.cov(self.user_features_.T) # mu_{0} - U_bar diff_X_bar = self.mu0_user - X_bar # W_{0}_star WI_post = inv( inv(self.WI_user) + N * S_bar + np.dot(diff_X_bar, diff_X_bar.T) * (N * self.beta_user) / (self.beta_user + N)) # Note: WI_post and WI_post.T should be the same. # Just make sure it is symmertic here WI_post = (WI_post + WI_post.T) / 2.0 # update alpha_user df_post = self.df_user + N # LAMBDA_{U} ~ W(W{0}_star, df_post) self.alpha_user = wishart.rvs(df_post, WI_post, 1, self.rand_state) # update mu_user # mu_{0}_star = (beta_{0} * mu_{0} + N * U_bar) / (beta_{0} + N) mu_mean = (self.beta_user * self.mu0_user + N * X_bar) / \ (self.beta_user + N) # decomposed inv(beta_{0}_star * LAMBDA_{U}) mu_var = cholesky(inv(np.dot(self.beta_user + N, self.alpha_user))) # sample multivariate gaussian self.mu_user = mu_mean + np.dot( mu_var, self.rand_state.randn(self.n_feature, 1)) def _udpate_item_features(self): # Gibbs sampling for item features for item_id in xrange(self.n_item): indices = self.ratings_csc_[:, item_id].indices features = self.user_features_[indices, :] rating = self.ratings_csc_[:, item_id].data - self.mean_rating_ # IT SUBTRACTS mean_rating_ AND ADD IT AGAIN LATER. rating = np.reshape(rating, (rating.shape[0], 1)) covar = inv(self.alpha_item + self.beta * np.dot(features.T, features)) lam = cholesky(covar) temp = (self.beta * np.dot(features.T, rating) + np.dot(self.alpha_item, self.mu_item)) mean = np.dot(covar, temp) temp_feature = mean + np.dot( lam, self.rand_state.randn(self.n_feature, 1)) self.item_features_[item_id, :] = temp_feature.ravel() def _update_user_features(self): # Gibbs sampling for user features for user_id in xrange(self.n_user): indices = self.ratings_csr_[user_id, :].indices features = self.item_features_[indices, :] rating = self.ratings_csr_[user_id, :].data - self.mean_rating_ rating = np.reshape(rating, (rating.shape[0], 1)) covar = inv(self.alpha_user + self.beta * np.dot(features.T, features)) lam = cholesky(covar) # aplha * sum(V_j * R_ij) + LAMBDA_U * mu_u temp = (self.beta * np.dot(features.T, rating) + np.dot(self.alpha_user, self.mu_user)) # mu_i_star mean = np.dot(covar, temp) temp_feature = mean + np.dot( lam, self.rand_state.randn(self.n_feature, 1)) self.user_features_[user_id, :] = temp_feature.ravel()
def __init__( self, n_user, n_item, n_feature, beta=2.0, beta_user=2.0, df_user=None, mu0_user=0., beta_item=2.0, df_item=None, mu0_item=0., converge=1e-5, seed=None, max_rating=None, # seed is useful for reproducibility. min_rating=None): super(BPMF, self).__init__() self.n_user = n_user self.n_item = n_item self.n_feature = n_feature self.rand_state = RandomState(seed) self.max_rating = float(max_rating) if max_rating is not None else None self.min_rating = float(min_rating) if min_rating is not None else None self.converge = converge # Hyper Parameter self.beta = beta # Inv-Whishart (User features) self.WI_user = np.eye(n_feature, dtype='float64') self.beta_user = beta_user self.df_user = int(df_user) if df_user is not None else n_feature self.mu0_user = np.repeat(mu0_user, n_feature).reshape(n_feature, 1) # a vector # Inv-Whishart (item features) self.WI_item = np.eye(n_feature, dtype='float64') self.beta_item = beta_item self.df_item = int(df_item) if df_item is not None else n_feature self.mu0_item = np.repeat(mu0_item, n_feature).reshape(n_feature, 1) # Latent Variables self.mu_user = np.zeros((n_feature, 1), dtype='float64') self.mu_item = np.zeros((n_feature, 1), dtype='float64') self.alpha_user = np.eye( n_feature, dtype='float64') # These are probably the Lambda matrices self.alpha_item = np.eye(n_feature, dtype='float64') self.user_features_ = 0.3 * self.rand_state.rand( n_user, n_feature) # initializes the user features randomly. Why 0.3?? self.item_features_ = 0.3 * self.rand_state.rand(n_item, n_feature) # data state self.mean_rating_ = None self.ratings_csr_ = None self.ratings_csc_ = None
days_popularity = np.zeros([no_days, 10]) for i in range(0, no_families): choices = family_data[i, 1:-1] no_people = family_data[i, -1] for itr in range(0, len(choices)): days_popularity[choices[itr] - 1, itr] += no_people days_popularity_tot = sum(days_popularity.T) cutoff_orig = 10 max_no_optimization_itrs = 1000 choices_inds = np.zeros([no_families, 1]).astype(int) BackwardMatrix = np.zeros([no_families, no_days]).astype(int) max_cost = 100000000 prng = RandomState(int(time.time())) for itr in range(0, max_no_optimization_itrs): ForwardMatrix = np.zeros([no_families, no_days]).astype(int) previous_choices_inds = choices_inds choices_inds = np.zeros([no_families, 1]).astype(int) cutoff = cutoff_orig / (1. + itr) # Forward step: people send their schedule to the constraints family_inds = np.random.permutation(no_families) day_count = np.zeros([no_days, 1]).astype(int) zero_vec = np.zeros([no_days, 1]) for i in family_inds: choices = family_data[i, 1:-1] no_people = family_data[i, -1] # Check the messages comming from neighbors
def validate(self) -> None: """ Check arguments correctness and consistency. * input files must exist * output files must be in a writeable directory * if no seed specified, set random seed. * length of per-chain lists equals specified # of chains """ if self.model_name is None: raise ValueError('no stan model specified') if self.model_exe is None: raise ValueError('model not compiled') if self.chain_ids is not None: for i in range(len(self.chain_ids)): if self.chain_ids[i] < 1: raise ValueError( 'invalid chain_id {}'.format(self.chain_ids[i]) ) if self.output_dir is not None: self.output_dir = os.path.realpath(os.path.expanduser( self.output_dir)) if not os.path.exists(os.path.dirname(self.output_dir)): raise ValueError( 'invalid path for output files, no such dir: {}'.format( self.output_dir ) ) try: testpath = os.path.join(self.output_dir, str(time())) with open(testpath, 'w+') as fd: pass os.remove(testpath) # cleanup except Exception: raise ValueError( 'invalid path for output files, ' 'cannot write to dir: {}'.format(self.output_dir) ) if self.seed is None: rng = RandomState() self.seed = rng.randint(1, 99999 + 1) else: if not isinstance(self.seed, (int, list)): raise ValueError( 'seed must be an integer between 0 and 2**32-1,' ' found {}'.format(self.seed) ) if isinstance(self.seed, int): if self.seed < 0 or self.seed > 2 ** 32 - 1: raise ValueError( 'seed must be an integer between 0 and 2**32-1,' ' found {}'.format(self.seed) ) else: if self.chain_ids is None: raise ValueError( 'seed must not be a list when no chains used' ) if len(self.seed) != len(self.chain_ids): raise ValueError( 'number of seeds must match number of chains ' ' found {} seed for {} chains '.format( len(self.seed), len(self.chain_ids) ) ) for i in range(len(self.seed)): if self.seed[i] < 0 or self.seed[i] > 2 ** 32 - 1: raise ValueError( 'seed must be an integer value' ' between 0 and 2**32-1,' ' found {}'.format(self.seed[i]) ) if isinstance(self.data, str): if not os.path.exists(self.data): raise ValueError('no such file {}'.format(self.data)) elif self.data is None: if isinstance(self.method_args, OptimizeArgs): raise ValueError('data must be set when optimizing') elif not isinstance(self.data, (str, dict)): raise ValueError('data must be string or dict') if self.inits is not None: if isinstance(self.inits, (Integral, Real)): if self.inits < 0: raise ValueError( 'inits must be > 0, found {}'.format(self.inits) ) elif isinstance(self.inits, str): if not os.path.exists(self.inits): raise ValueError('no such file {}'.format(self.inits)) elif isinstance(self.inits, list): if self.chain_ids is None: raise ValueError( 'inits must not be a list when no chains are used' ) if len(self.inits) != len(self.chain_ids): raise ValueError( 'number of inits files must match number of chains ' ' found {} inits files for {} chains '.format( len(self.inits), len(self.chain_ids) ) ) names_set = set(self.inits) if len(names_set) != len(self.inits): raise ValueError( 'each chain must have its own init file,' ' found duplicates in inits files list.' ) for i in range(len(self.inits)): if not os.path.exists(self.inits[i]): raise ValueError( 'no such file {}'.format(self.inits[i]) )
def plot2(self, X): X_01 = X[:, [0, 1]] kmeans = KMeans(n_clusters=3, random_state=RandomState(42)).fit(X_01) plot_2D(X_01, kmeans.labels_, ["c0", "c1", "c2"], "Scatter Plot: Features-0-1") calculate_IV_EV(X_01, kmeans.labels_)
y_ = tf.placeholder(tf.float32, shape=(None, 1), name='y-input') # 定义一个单层的前向传播的神经网络 w1 = tf.Variable(tf.random_normal([2, 1], stddev=1, seed=1)) y = tf.matmul(x, w1) # 定义预测多了和预测少了的成本 loss_less = 10 loss_more = 1 loss = tf.reduce_sum( tf.where(tf.greater(y, y_), (y - y_) * loss_more, (y_ - y) * loss_less)) train_step = tf.train.AdadeltaOptimizer(0.001).minimize(loss) # 通过随机数生成一个模拟数据集 rdm = RandomState(1) dataset_size = 128 X = rdm.rand(dataset_size, 2) Y = [[x1 + x2 + rdm.rand() / 10.0 - 0.05] for (x1, x2) in X] # 训练神经网络 with tf.Session() as sess: tf.global_variables_initializer().run() STEPS = 5000 for i in range(STEPS): start = (i * batch_size) % dataset_size end = min(start + batch_size, dataset_size) sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]}) if i % 1000 == 0: print("After %d training step(s), w1 is: " % (i))
def plot3(self, X): X_23 = X[:, [2, 3]] kmeans = KMeans(n_clusters=3, random_state=RandomState(42)).fit(X_23) plot_2D(X_23, kmeans.labels_, ["c0", "c1", "c2"], "Scatter Plot: Features-2-3") calculate_IV_EV(X_23, kmeans.labels_)
def __init__(self, input, n_in, n_out, rng=RandomState(4444), W=None, b=None, activation=T.tanh): """ Typical hidden layer of a MLP: units are fully-connected and have sigmoidal activation function. Weight matrix W is of shape (n_in,n_out) and the bias vector b is of shape (n_out,). NOTE : The nonlinearity used here is tanh Hidden unit activation is given by: tanh(dot(input,W) + b) :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.dmatrix :param input: a symbolic tensor of shape (n_examples, n_in) :type n_in: int :param n_in: dimensionality of input :type n_out: int :param n_out: number of hidden units :type activation: theano.Op or function :param activation: Non linearity to be applied in the hidden layer """ self.input = input # end-snippet-1 # `W` is initialized with `W_values` which is uniformely sampled # from sqrt(-6./(n_in+n_hidden)) and sqrt(6./(n_in+n_hidden)) # for tanh activation function # the output of uniform if converted using asarray to dtype # theano.config.floatX so that the code is runable on GPU # Note : optimal initialization of weights is dependent on the # activation function used (among other things). # For example, results presented in [Xavier10] suggest that you # should use 4 times larger initial weights for sigmoid # compared to tanh # We have no info for other function, so we use the same as # tanh. if W is None: W_values = np.asarray(rng.uniform( low=-np.sqrt(6. / (n_in + n_out)), high=np.sqrt(6. / (n_in + n_out)), size=(n_in, n_out)), dtype=theano.config.floatX) if activation == theano.tensor.nnet.sigmoid: W_values *= 4 W = theano.shared(value=W_values, name='W', borrow=True) if b is None: b_values = np.zeros((n_out, ), dtype=theano.config.floatX) b = theano.shared(value=b_values, name='b', borrow=True) self.W = W self.b = b lin_output = T.dot(input, self.W) + self.b self.output = (lin_output if activation is None else activation(lin_output)) # parameters of the model self.params = [self.W, self.b]
def create_app(strD): MODEL_SAVE_PATH = "model/" MODEL_NAME = "model.ckpt" INPUT_NODE_NUM = 7 x = tf.placeholder(tf.float32, shape=(None, INPUT_NODE_NUM), name='x-input') y_ = tf.placeholder(tf.float32, shape=(None, 5), name='y-input') rdm = RandomState(1) dataset_size = 101 w1 = tf.Variable(tf.random_normal([INPUT_NODE_NUM, 20], stddev=1)) w2 = tf.Variable(tf.random_normal([20, 20], stddev=1)) w3 = tf.Variable(tf.random_normal([20, 5], stddev=1)) #w4 = tf.Variable(tf.random_normal([3,1],stddev = 1)) saver = tf.train.Saver() x_w1 = tf.matmul(x, w1) x_w1 = tf.sigmoid(x_w1) w1_w2 = tf.matmul(x_w1, w2) w1_w2 = tf.sigmoid(w1_w2) y = tf.matmul(w1_w2, w3) y = tf.sigmoid(y) InputX = rdm.rand(1, INPUT_NODE_NUM) if len(strD) != 7: return json.dumps('em...try to keep it in 7 numbers') if strD.isdigit(): pass else: return json.dumps('please use int number') with tf.Session() as sess: #读取模型 ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) else: pass InputX[0][0] = strD[0] InputX[0][1] = strD[1] InputX[0][2] = strD[2] InputX[0][3] = strD[3] InputX[0][4] = strD[4] InputX[0][5] = strD[5] InputX[0][6] = strD[6] #print("x :") #print(InputX) predict_output = sess.run(y, {x: InputX}) #print("predict_output :") #print(predict_output) predict_outputInt = tf.round(predict_output) #print("predict_outputInt :") #print(predict_outputInt) output_num = outputByChinese(predict_outputInt, sess) #print("output_num:") #print(output_num) if output_num == 0: #return render_template('homepage.html',file=file) return render_template("veryBad.html") return json.dumps('I don\'t want to go to this place at all') if output_num == 1: return render_template("Bad.html") return json.dumps('well,I just don\'t like there') if output_num == 2: return render_template("justSoSo.html") #return json.dumps( #'Just so so, Would you like to send an e-mail to explain the detail? [email protected]' #) if output_num == 3: return render_template("goodOption.html") #return json.dumps( #'It\'s a good oppotunity,contact me with 15365180821' #) if output_num == 4: return render_template("veryInterested.html") #return json.dumps( #'I am very interested in it,please contact me with 15365180821' #) return render_template("confused.html")
from numpy.random import RandomState from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams from random import Random seed = 42 py_rng = Random(seed) np_rng = RandomState(seed) t_rng = RandomStreams(seed) def set_seed(n): global seed, py_rng, np_rng, t_rng seed = n py_rng = Random(seed) np_rng = RandomState(seed) t_rng = RandomStreams(seed)
import pyfits import math import pylab as pl import matplotlib.pyplot as plt import heapq from operator import xor import scipy.signal from numpy import float64 import astroML.time_series import astroML_addons.periodogram import cython from sklearn.decomposition import PCA from sklearn.cluster import KMeans from sklearn.cluster import DBSCAN,Ward from numpy.random import RandomState rng = RandomState(42) import itertools import sys """cimport numpy as np cimport cython __all__ = ['lombscargle'] cdef extern from "math.h": double cos(double) double sin(double) double atan(double) @cython.boundscheck(False)
from sklearn.datasets import fetch_olivetti_faces from sklearn import decomposition # 非负矩阵分解 Non-negative Matrix Fatorization # 设置2*3图相框,制定图像排列情况,2行3列 n_row, n_col = 2, 3 n_components = n_row * n_col # 设置提取特征的数目为64*64 image_shape = (64, 64) ############################################################################### # Load faces data # 使用随机种子打乱顺序,每次运行程序出现的图像和特征都不一样 dataset = fetch_olivetti_faces(shuffle=True, random_state=RandomState(0)) faces = dataset.data ############################################################################### def plot_gallery(title, images, n_col=n_col, n_row=n_row): # 创建图片,并且制定图片的大小 plt.figure(figsize=(2. * n_col, 2.26 * n_row)) # 设置标题及字号大小 plt.suptitle(title, size=16) for i, comp in enumerate(images): # 选择画制的子图 plt.subplot(n_row, n_col, i + 1) vmax = max(comp.max(), -comp.min())