def generate_rand_obs(nExp, nObs, domainBounds, sigvar=None, rng=None):
    # create random sets of observations for each experiment
    if not sigvar: sigvar = 1.
    if not rng: rng = RandomState()

    domainBounds = npa(domainBounds)
    dimX = domainBounds.shape[0]
    minX = domainBounds[:, 0]
    maxX = domainBounds[:, 1]
    rangeX = maxX - minX
    xObs = rng.uniform(size=(nExp, nObs, dimX))
    xObs *= rangeX
    xObs += minX
    yObs = empty(shape=(nExp, nObs))
    for iexp in xrange(nExp):
        good = False
        while not good:
            yObs0 = rng.normal(size=(nObs))
            if yObs0.max() > 0: good = True
        yObs[iexp, :] = yObs0
    # yObs = rng.normal(size=(nExp, nObs, 1))
    yObs *= sigvar

    return {'x': xObs,
            'y': yObs}
Example #2
0
    def test_param_cov(self):
        """
        Tests that the 'param_cov' fit_info entry gets the right answer for
        *linear* least squares, where the answer is exact
        """
        rs = RandomState(1234567890)

        a = 2
        b = 100

        x = np.linspace(0, 1, 100)
        # y scatter is amplitude ~1 to make sure covarience is non-negligible
        y = x*a + b + rs.randn(len(x))

        #first compute the ordinary least squares covariance matrix
        X = np.matrix(np.vstack([x, np.ones(len(x))]).T)
        beta = np.linalg.inv(X.T * X) * X.T * np.matrix(y).T
        s2 = np.sum((y - (X * beta).A.ravel())**2) / (len(y) - len(beta))
        olscov = np.linalg.inv(X.T * X) * s2

        #now do the non-linear least squares fit
        mod = models.Linear1D(a, b)
        fitter = fitting.NonLinearLSQFitter()

        fmod = fitter(mod, x, y)

        utils.assert_allclose(fmod.parameters, beta.A.ravel())
        utils.assert_allclose(olscov, fitter.fit_info['param_cov'])
Example #3
0
def split(dataset, test_size=0.5, random_state=None):
    if random_state is None:
        random_state = np.random.randint(0, 999999)
    nb = dataset.X.shape[0]
    nb_test = int(nb * test_size)
    nb_train = nb - nb_test
    rng = RandomState(random_state)
    indices = np.arange(0, nb)
    rng.shuffle(indices)
    indices_train = indices[0:nb_train]
    indices_test = indices[nb_train:]

    X = dataset.X[indices_train]
    if hasattr(dataset, 'y') and dataset.y is not None:
        y = dataset.y[indices_train]
    else:
        y = None
    dataset_train = Manual(X, y)
    if hasattr(dataset, "img_dim"):
        dataset_train.img_dim = dataset.img_dim
    if hasattr(dataset, "output_dim"):
        dataset_train.output_dim = dataset.output_dim

    X = dataset.X[indices_test]
    if hasattr(dataset, 'y') and dataset.y is not None:
        y = dataset.y[indices_test]
    else:
        y = None
    dataset_test = Manual(X, y)
    if hasattr(dataset, "img_dim"):
        dataset_test.img_dim = dataset.img_dim
    if hasattr(dataset, "output_dim"):
        dataset_test.output_dim = dataset.output_dim
    return dataset_train, dataset_test
Example #4
0
def test_get_random_state():
    prng1 = RandomState(42)
    prng2 = get_prng(42)
    prng3 = get_prng(prng1)
    prng4 = get_prng(prng2)
    prng5 = get_prng()
    prng6 = get_prng(None)
    prng7 = get_prng(np.random)
    assert(isinstance(prng1, RandomState))
    assert(isinstance(prng3, RandomState))
    assert(isinstance(prng5, RandomState))
    assert(isinstance(prng6, RandomState))
    assert(isinstance(prng7, RandomState))
    x1 = prng1.randint(5, size=10)
    x2 = prng2.randint(5, size=10)
    x3 = prng3.randint(5, size=10)
    x4 = prng4.randint(5, size=10)
    x5 = prng5.randint(5, size=10)
    x6 = prng6.randint(5, size=10)
    x7 = prng7.randint(5, size=10)
    assert_equal(x1, x2)
    assert_equal(x3, x4)
    assert_equal(len(x5), 10)
    assert_equal(len(x6), 10)
    assert_equal(len(x7), 10)
Example #5
0
def test_online_predictor():
    """ Test whether predictions are done at correct timepoints.
    Model actually just returns input """
    
    rng = RandomState(3904890384)
    n_samples_in_buffer = 1000
    dataset = rng.rand(n_samples_in_buffer*2,5).astype(np.float32)
    markers = np.ones((n_samples_in_buffer*2,1)).astype(np.float32)
    set_and_markers = np.concatenate((dataset, markers), axis=1)
    
    factor_new=0.001
    n_stride = 10
    pred_freq = 11
    standardized = exponential_running_standardize(dataset,
        factor_new=factor_new, init_block_size=n_stride)
    model = InputLayer([1,1,1,1])
    
    processor = StandardizeProcessor(factor_new=factor_new,
        n_samples_in_buffer=n_samples_in_buffer)
    
    online_model = OnlineModel(model)
    online_pred = OnlineCoordinator(processor, online_model, pred_freq=pred_freq,
        trainer=NoTrainer())
    
    online_pred.initialize(n_chans=dataset.shape[1])
    all_preds = []
    for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride):
        online_pred.receive_samples(set_and_markers[i_start_sample:i_start_sample+n_stride])
        if online_pred.has_new_prediction():
            pred, _ = online_pred.pop_last_prediction_and_sample_ind()
            all_preds.append(pred)
        
    assert np.array_equal(np.array(all_preds).squeeze(), standardized[10::pred_freq])
Example #6
0
def test_sum_prediction():
    """ Test with a model that predicts sum over four samples """
    rng = RandomState(3904890384)
    n_samples_in_buffer = 1000
    dataset = rng.rand(n_samples_in_buffer*2,5).astype(np.float32)
    markers = np.ones((n_samples_in_buffer*2,1)).astype(np.float32)
    set_and_markers = np.concatenate((dataset, markers), axis=1)
    
    factor_new=0.001
    n_stride = 10
    pred_freq = 11
    standardized = exponential_running_standardize(dataset,
        factor_new=factor_new, init_block_size=n_stride)
    model = InputLayer([1,1,4,1])
    model = GlobalPoolLayer(model,pool_function=T.sum)
    
    expected = [np.sum(standardized[stop-4:stop], axis=0) for stop in xrange(11, dataset.shape[0], 11)]
    expected = np.array(expected)
    
    processor = StandardizeProcessor(factor_new=factor_new,
        n_samples_in_buffer=n_samples_in_buffer)
    
    online_model = OnlineModel(model)
    online_pred = OnlineCoordinator(processor, online_model, pred_freq=pred_freq,
        trainer=NoTrainer())
    
    online_pred.initialize(n_chans=dataset.shape[1])
    all_preds = []
    for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride):
        online_pred.receive_samples(set_and_markers[i_start_sample:i_start_sample+n_stride])
        if online_pred.has_new_prediction():
            pred, _ = online_pred.pop_last_prediction_and_sample_ind()
            all_preds.append(pred)
        
    assert np.allclose(np.array(all_preds).squeeze(), expected, rtol=1e-3)
Example #7
0
    def train(self, X, XY, V, VY, count_dict, word_dict, embed_map):
        """
        Trains the LBL
        """
        self.start = self.seed
        self.init_params(embed_map, count_dict, XY)
        inds = np.arange(len(X))
        numbatches = len(inds) / self.batchsize
        curr = 1e20
        counter = 0
        target=None
        num = 15000

        # Main loop
        stop.display_phase(1)
        for epoch in range(self.maxepoch):
            self.epoch = epoch
            tic = time.time()
            prng = RandomState(self.seed + epoch + 1)
            prng.shuffle(inds)
            for minibatch in range(numbatches):

                batchX = X[inds[minibatch::numbatches]]
                batchY = XY[inds[minibatch::numbatches]]
            
                (words, acts, preds) = self.forward(batchX)
                self.backward(batchY, preds, acts, words, batchX)
                self.update_params(batchX)

            self.update_hyperparams()
            toc = time.time()

            # Results and stopping criteria
            obj = self.compute_obj(X[:num], XY[:num])
            obj_val = self.compute_obj(V[:num], VY[:num])

            if self.verbose > 0:
                stop.display_results(epoch, toc-tic, obj, obj_val)
            (curr, counter) = stop.update_result(curr, obj_val, counter)
            if counter == 0:
                stop.save_model(self, self.loc)
                stopping_target = obj

            if stop.criteria_complete(self, epoch, curr, obj, counter, 
                self.k, obj_val, target):
                if self.criteria == 'maxepoch':
                    break
                elif self.criteria == 'validation_pp':
                    self = stop.load_model(self.loc)
                    counter = 0
                    X = np.r_[X, V]
                    XY = vstack([XY, VY]).tocsr()
                    self.criteria = 'll_train_heldout'
                    target = stopping_target   #obj
                    stop.display_phase(2)
                    inds = range(X.shape[0])
                    prng.shuffle(inds)
                    numbatches = len(inds) / self.batchsize
                elif self.criteria == 'll_train_heldout':
                    break
Example #8
0
def cmc(g, xdists, u_to_x, T, seed, maxitr):
    """
    Crude Monte Carlo simulation.
    """

    # Seed the random number generator if required
    if seed == -1:
        prng = RandomState()
    else:
        prng = RandomState(seed)
    
    # Generate standard normal samples centered at the origin
    u0 = zeros(len(xdists))
    covmat = eye(len(xdists)) 
    u = prng.multivariate_normal(u0, covmat, size=maxitr).T
    g_mc = g(u_to_x(u, xdists, T))

    # Convert g-function output to pass/fail indicator function and estimate pf
    g_mc[g_mc>0] = 0
    g_mc[g_mc<0] = 1
    mu_pf = g_mc.mean()
    beta = -norm.ppf(mu_pf) if mu_pf < 0.5 else norm.ppf(mu_pf)

    # Convergence metrics (standard deviation, standard error, CoV of s.e.)
    std_pf = g_mc.std(ddof=1) # Calculate sample standard deviation
    se_pf = std_pf/sqrt(maxitr)
    cv_pf = se_pf/mu_pf

    return {'vars': xdists, 'beta': beta, 'Pf': mu_pf, 'stderr': se_pf, 
            'stdcv': cv_pf}
Example #9
0
def corr(x, y, reps=10**4, prng=None):
    """
    Simulate permutation p-value for Spearman correlation coefficient

    Parameters
    ----------
    x : array-like
    y : array-like
    reps : int
    prng : RandomState instance or None, optional (default=None)
        If RandomState instance, prng is the pseudorandom number generator;
        If None, the pseudorandom number generator is the RandomState
        instance used by `np.random`.

    Returns
    -------
    tuple
        Returns test statistic, left-sided p-value,
        right-sided p-value, two-sided p-value, simulated distribution
    """
    if prng is None:
        prng = RandomState()
    tst = np.corrcoef(x, y)[0, 1]
    sims = [np.corrcoef(prng.permutation(x), y)[0, 1] for i in range(reps)]
    left_pv = np.sum(sims <= tst)/reps
    right_pv = np.sum(sims >= tst)/reps
    two_sided_pv = np.sum(np.abs(sims) >= np.abs(tst))/reps
    return tst, left_pv, right_pv, two_sided_pv, sims
Example #10
0
def permute_within_groups(x, group, prng=None):
    """
    Permutation of condition within each group.

    Parameters
    ----------
    x : array-like
        A 1-d array indicating treatment.
    group : array-like
        A 1-d array indicating group membership
    prng : RandomState instance or None, optional (default=None)
        If RandomState instance, prng is the pseudorandom number generator;
        If None, the pseudorandom number generator is the RandomState
        instance used by `np.random`.

    Returns
    -------
    permuted : array-like
        The within group permutation of x.
    """
    permuted = x.copy()
    if prng is None:
        prng = RandomState()

    # (avoid additional flops) -- maybe memoize
    for g in np.unique(group):
        gg = group == g
        permuted[gg] = prng.permutation(permuted[gg])
    return permuted
Example #11
0
def assign_random_gt(input_vcf, output, sample_name="HG", default_af=0.01, seed=None):
    vcf_pointer = pysam.VariantFile(filename=input_vcf)
    new_header = vcf_pointer.header.copy()
    if "GT" not in new_header.formats:
        new_header.formats.add("GT", "1", "String", "Consensus Genotype across all datasets with called genotype")
        new_header.samples.add(sample_name)
    output.write(str(new_header))

    default_probs = [1 - default_af * (1 + default_af), default_af/2, default_af/2, default_af * default_af]
    rng = RandomState(seed)
    previous_pos = 0
    for rec in vcf_pointer.fetch():
        rec_copy = rec.copy()
        if "GT" not in rec_copy.format.keys():
            if rec_copy.pos == previous_pos:
                c = "0|0"
            else:
                if "AF" not in rec_copy.info.keys():
                    gt_probs = default_probs
                else:
                    af = rec_copy.info["AF"]
                    gt_probs = [1 - af * (1 + af), af/2, af/2, af * af]
                c = rng.choice(["0|0", "0|1", "1|0", "1|1"], p=gt_probs)
            output.write("\t".join([str(rec_copy)[:-1], "GT", c]) + "\n")
        previous_pos = rec_copy.pos

    vcf_pointer.close()
Example #12
0
def test_experiment_sample_windows():
    data_rng = RandomState(398765905)
    rand_topo = data_rng.rand(200,10,10,3).astype(np.float32)
    rand_y = np.int32(data_rng.rand(200) > 0.5)
    rand_topo[rand_y == 1] += 0.1
    rand_set = DenseDesignMatrixWrapper(topo_view=rand_topo, y=rand_y)
    
    lasagne.random.set_rng(RandomState(9859295))
    in_layer = InputLayer(shape= [None, 10,5,3])
    network = DenseLayer(incoming=in_layer, name='softmax',
        num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
    updates_modifier = MaxNormConstraint({'softmax': 0.5})
    
    dataset = rand_set
    
    dataset_iterator = WindowsIterator(n_samples_per_window=5, 
                                             batch_size=60)
    
    preprocessor = OnlineAxiswiseStandardize(axis=['c', 1])
    dataset_splitter=FixedTrialSplitter(n_train_trials=150, valid_set_fraction=0.1)
    updates_var_func=lasagne.updates.adam
    loss_var_func= lasagne.objectives.categorical_crossentropy
    monitors=[braindecode.veganlasagne.monitors.LossMonitor (),
                    braindecode.veganlasagne.monitors.WindowMisclassMonitor(),
                    braindecode.veganlasagne.monitors.RuntimeMonitor()]
    stop_criterion= braindecode.veganlasagne.stopping.MaxEpochs(num_epochs=5)
    
    
    exp = Experiment(network, dataset, dataset_splitter, preprocessor,
              dataset_iterator, loss_var_func, updates_var_func, 
              updates_modifier, monitors, stop_criterion,
              remember_best_chan='valid_misclass',
              run_after_early_stop=True)
    exp.setup()
    exp.run()
    
    assert np.allclose(
        [0.629630,0.140741,0.029630,0.022222,0.000000,0.000000,0.000000],
        exp.monitor_chans['train_misclass'],
        rtol=1e-4, atol=1e-4)
    assert np.allclose(
        [0.400000,0.133333,0.066667,0.000000,0.000000,0.000000,0.000000],
        exp.monitor_chans['valid_misclass'],
        rtol=1e-4, atol=1e-4)
    assert np.allclose(
        [0.560000,0.060000,0.000000,0.000000,0.000000,0.000000,0.000000],
        exp.monitor_chans['test_misclass'],
        rtol=1e-4, atol=1e-4)
    assert np.allclose(
        [1.180485, 0.574264, 0.420023, 0.330909, 0.278569, 0.245692, 0.242845],
        exp.monitor_chans['train_loss'],
        rtol=1e-4, atol=1e-4)
    assert np.allclose(
        [1.016782, 0.514049, 0.370485, 0.288948, 0.240913, 0.211189, 0.215967],
        exp.monitor_chans['valid_loss'],
        rtol=1e-4, atol=1e-4)
    assert np.allclose(
        [1.031832, 0.504570, 0.352317, 0.269810, 0.223904, 0.196681, 0.197899],
        exp.monitor_chans['test_loss'],
        rtol=1e-4, atol=1e-4)
Example #13
0
def _smepdpsolve_single_trajectory(L, dt, tlist, N_store, N_substeps, rho_t,
                                   c_ops, e_ops, data):
    """ 
    Internal function.
    """
    states_list = []

    rho_t = np.copy(rho_t)

    prng = RandomState() # todo: seed it
    r_jump, r_op = prng.rand(2)

    jump_times = []
    jump_op_idx = []

    for t_idx, t in enumerate(tlist):

        if e_ops:
            for e_idx, e in enumerate(e_ops):
                data.expect[e_idx, t_idx] += expect_rho_vec(e, rho_t)
        else:
            states_list.append(Qobj(vec2mat(rho_t)))

        for j in range(N_substeps):

            if expect_rho_vec(d_op, sigma_t) < r_jump:
                # jump occurs
                p = np.array([rho_expect(c.dag() * c, rho_t) for c in c_ops])
                p = np.cumsum(p / np.sum(p))
                n = np.where(p >= r_op)[0][0]

                # apply jump
                rho_t = c_ops[n] * psi_t * c_ops[n].dag()
                rho_t /= rho_expect(c.dag() * c, rho_t)
                rho_t = np.copy(rho_t)

                # store info about jump
                jump_times.append(tlist[t_idx] + dt * j)
                jump_op_idx.append(n)

                # get new random numbers for next jump
                r_jump, r_op = prng.rand(2)

            # deterministic evolution wihtout correction for norm decay
            dsigma_t = spmv(L.data.data,
                            L.data.indices,
                            L.data.indptr, sigma_t) * dt

            # deterministic evolution with correction for norm decay
            drho_t = spmv(L.data.data,
                          L.data.indices,
                          L.data.indptr, rho_t) * dt

            rho_t += drho_t

            # increment density matrices
            sigma_t += dsigma_t
            rho_t += drho_t

    return states_list, jump_times, jump_op_idx
Example #14
0
def test_one_sample():
    prng = RandomState(42)
    
    x = np.array(range(5))
    y = x-1
    
    # case 1: one sample only
    res = one_sample(x, seed = 42, reps = 100)
    np.testing.assert_almost_equal(res[0], 0.05999999)
    np.testing.assert_equal(res[1], 2)
    
    # case 2: paired sample
    res = one_sample(x, y, seed = 42, reps = 100)
    np.testing.assert_equal(res[0], 0.02)
    np.testing.assert_equal(res[1], 1)
    
    # case 3: break it - supply x and y, but not paired
    y = np.append(y, 10)
    assert_raises(ValueError, one_sample, x, y)
    
    # case 4: say keep_dist=True
    res = one_sample(x, seed = 42, reps = 100, keep_dist=True)
    np.testing.assert_almost_equal(res[0], 0.05999999)
    np.testing.assert_equal(res[1], 2)
    np.testing.assert_equal(min(res[2]), -2)
    np.testing.assert_equal(max(res[2]), 2)
    np.testing.assert_equal(np.median(res[2]), 0)

    # case 5: use t as test statistic
    y = x + prng.normal(size=5)
    res = one_sample(x, y, seed = 42, reps = 100, stat = "t", alternative = "less")
    np.testing.assert_equal(res[0], 0.07)
    np.testing.assert_almost_equal(res[1], 1.4491883)
Example #15
0
def test_minp_one_pvalue():
    prng = RandomState(55)
    pvalues = np.array([1])
    distr = prng.uniform(low=0, high=10, size=20).reshape(20, 1)
    npc(pvalues, distr, "fisher", "greater")

# TODO: more fwer_minp tests
Example #16
0
def transpose_characters(token, index_to_char, n=1, char_pool=None, seed=17):
    if isinstance(seed, RandomState):
        rng = seed
    else:
        rng = RandomState(seed)

    chars = set(token)
    if len(chars) == 1:
        return token

    new_token = token
    for i in six.moves.range(n):
        idx = max(1, rng.randint(len(new_token)))
        neighbor = 0
        if idx == 0:
            neighbor == 1
        elif idx == len(new_token) - 1:
            neighbor = len(new_token) - 2
        else:
            if rng.uniform() > 0.5:
                neighbor = idx + 1
            else:
                neighbor = idx - 1
        left = min(idx, neighbor) 
        right = max(idx, neighbor)
        new_token = unicode(new_token[0:left] + new_token[right] + new_token[left] + new_token[right+1:])
    return new_token
def test_pad_and_unpad_equal_1d():
    'gridder.pad_array and subsequent .unpad_array gives original array: 1D'
    prng = RandomState(12345)
    x = prng.rand(21)
    xpad, nps = gridder.pad_array(x)
    xunpad = gridder.unpad_array(xpad, nps)
    assert_almost(xunpad, x)
Example #18
0
def test_qtl_binomial_scan_covariate_redundance():
    random = RandomState(9)

    N = 200
    G = random.randn(N, N + 100)
    G = stdnorm(G, 0)
    G /= sqrt(G.shape[1])

    p = 2
    X = random.randn(N, p)
    X = stdnorm(X, 0)
    X /= sqrt(X.shape[1])

    ntrials = random.randint(1, 50, N)
    nsuccesses = binomial(
        ntrials,
        -0.1,
        G,
        causal_variants=X,
        causal_variance=0.1,
        random_state=random)

    X[:] = 1
    qtl = scan(BinomialPhenotype(nsuccesses, ntrials), X, G=G, progress=False,
               fast=False)
    assert_allclose(qtl.pvalues(), [1] * p, rtol=1e-4)
Example #19
0
def test_qtl_fast_binomial_scan():
    random = RandomState(9)

    N = 200
    G = random.randn(N, N + 100)
    G = stdnorm(G, 0)
    G /= sqrt(G.shape[1])

    p = 2
    X = random.randn(N, p)
    X = stdnorm(X, 0)
    X /= sqrt(X.shape[1])

    ntrials = random.randint(1, 50, N)
    nsuccesses = binomial(
        ntrials,
        -0.1,
        G,
        causal_variants=X,
        causal_variance=0.1,
        random_state=random)

    qtl = scan(BinomialPhenotype(nsuccesses, ntrials), X, G=G, progress=False,
               fast=True)

    assert_allclose(
        qtl.pvalues(), [
            0.698565827403, 0.443299805368
        ],
        rtol=1e-4)
 def createGraph(self):
     """
         W -> Adjacency Matrix
         For test cases, this method is modified to generate random 
         weight matrix with zero weights for no edges.
         
            Graph is undirected with only positive weights 
            
         @prng: pseudo random number generator
     """
     '''
     _ = 0  # no edge
         # a  b  c  d
     W = [[_, 1, 3, 4],  # a
          [1, _, 2, 1],  # b
          [_, 1, _, 1],  # c
          [5, 1, 2, _]]  # d
     #print W
     '''
     prng = RandomState()  # alternative to random.seed
     w = prng.randint(0, 6, size=16)
     # inflating 1D array to 2D square matrix
     W = w.reshape(4, 4)
     #pprint(W)
     #print W[a][d]
     W = np.array(W) 
     W_symm = (W + W.T)/2  # making the matrix symmetric
     np.fill_diagonal(W_symm, 0)
     return W_symm
Example #21
0
def align_converge(y_LR,size=64):
    """iterate until offsets converge"""
    (h,w) = y_LR.shape
    # split image
    y_L = y_LR[:,:w/2]
    y_R = y_LR[:,w/2:]
    (h,w) = y_L.shape
    s = size / 2
    # now find n offsets
    rand = RandomState(0)
    prev_dx, prev_dy = 0, 0
    series = []
    while True:
        # at a random locations in y_L
        y = rand.randint(h/4,h*3/4)
        x = rand.randint(w/4,w*3/4)
        it = y_L[y:y+s,x:x+s] # take an s x s chunk there
        tm = match_template(y_R,it) # match it against y_R
        ry, rx = maximum_position(tm) # max value is location
        series += [((y-ry), (x-rx))] # accumulatea
        print series
        n = len(series)
        if n % 2 == 0:
            # take the median
            dy, dx = np.median(np.asarray(series),axis=0).astype(int)
            if n > 100 or (abs(dy-prev_dy) == 0 and abs(dx-prev_dx) == 0):
                return dy, dx
            prev_dy, prev_dx = dy, dx
Example #22
0
class RandomGenerator(object):
    def __init__(self, seed=None):
        self._random = RandomState(seed=seed)

    def random(self):
        return self._random.rand()

    def randint(self, a, b=None):
        if b is None:
            b = a
            a = 0
        r = self._random.randint(a, high=b, size=1)
        return r[0]

    def sample(self, population, k):
        if k == 0:
            return []
        return self._random.choice(population, size=k, replace=False)

    def __getattr__(self, attr):
        return getattr(self._random, attr)

    def __getstate__(self):
        return {'_random': self._random}

    def __setstate__(self, d):
        self._random = d['_random']
Example #23
0
File: GA.py Project: lisabang/iqsar
 def mkeindseed(self,desc_in_ind=5):
     if self.mkeindseed.count<=100:
         prng=RandomState(self.seed+self.mkeindseed.count)
     if self.mkeindseed.count>100:
         prng=RandomState(self.seed+(self.mkeindseed.count%100))
     smple=prng.choice(self.basetable.columns,size=desc_in_ind, replace=False)
     return list(smple)
Example #24
0
def setupSeed(hoursBetweenTimestepInROMSFiles,startTime,endTime,startSpawningTime,endSpawningTime,releaseParticles):
    ##################################################
    # Create seed variation as function of day
    ##################################################

    # Make datetime array from start to end at 3 hour interval
    difference=endTime-startTime
    hoursOfSimulation=divmod(difference.total_seconds(), 3600)
    difference=endSpawningTime-startSpawningTime
    hoursOfSpawning=divmod(difference.total_seconds(), 3600)
    timeStepsSimulation=int(int(hoursOfSimulation[0])/hoursBetweenTimestepInROMSFiles)
		
    print "\nKINO TIME EVOLUTION:"
    print "=>SIMULATION: Drift simulation will run for %s simulation hours" %(timeStepsSimulation)
    print "=>SPAWNING: Simulated spawning will run for %s simulation hours\n initiated on %s and ending on %s"%(timeStepsSimulation,startSpawningTime,endSpawningTime)

    interval = timedelta(hours=24)
    hoursPerSpawning=divmod(interval.total_seconds(), 3600) #hours per spawning event
    timeStepsSpawning=int(int(hoursOfSpawning[0])/int(hoursPerSpawning[0])) #number of spawning timesteps
    spawningTimes = [startSpawningTime + interval*n for n in range(timeStepsSpawning)] #times of spawning

    # Normal distribution around 0.5
    mu, sigma = 0.5, 0.1 # mean and standard deviation
    prng = RandomState(1) # random number generator (specify number to ensure the same sequence each time)
    s = prng.normal(mu, sigma, len(spawningTimes)) # random distribution
    num=(s*releaseParticles).astype(int) # number of particles released per spawning event
    print num #timeStepsSpawning * releaseParticles * mu gives approx. number of particles released
    num=np.sort(num) #sort particles in increasing order 
    num=np.concatenate((num[len(num)%2::2],num[::-2]),axis=0) #release the highest number of particles at the midpoint of the spawning period
	
    print "SPAWNING: Simulated spawning will release %s eggs"%(np.sum(num))

    return num, spawningTimes
def randomRotations(D, rng=None):
    from math import pi, sin, cos, sqrt
    from numpy.random import RandomState
    from itertools import product as cartesian, repeat
    from .util import frange

    if not isinstance(rng, RandomState):
        rng = RandomState(rng)

    if D == 2:
        while True:
            yield (rng.uniform(-pi, pi),)
    elif D == 3:
        # Ken Shoemake
        # Graphics Gems III, pp 124-132
        from .quaternion import Quaternion
        while True:
            X = rng.uniform(0, 1)
            theta = rng.uniform(0, 2*pi), rng.uniform(0, 2*pi)
            R = (sqrt(1-X), sqrt(X))
            yield Quaternion(sin(theta[0]) * R[0], cos(theta[0]) * R[0],
                             sin(theta[1]) * R[1], cos(theta[1]) * R[1]).axis_angle
    else:
        raise NotImplementedError("Only defined for D in [2..3], not {}"
                                  .format(D))
class GaussianY(PhaseSpace):
    """Vertical Gaussian particle phase space distribution."""

    def __init__(self, sigma_y, sigma_yp, generator_seed=None):
        """Initiates the vertical beam coordinates
        to the given Gaussian shape.
        """
        self.sigma_y  = sigma_y
        self.sigma_yp = sigma_yp

        self.random_state = RandomState()
        self.random_state.seed(generator_seed)

    @classmethod
    def from_optics(cls, alpha_y, beta_y, epsn_y, betagamma, generator_seed=None):
        """Initialise GaussianY from the given optics functions.
        beta_y is given in meters and epsn_y in micrometers.
        """

        sigma_y  = np.sqrt(beta_y * epsn_y * 1e-6 / betagamma)
        sigma_yp = sigma_y / beta_y

        return cls(sigma_y, sigma_yp, generator_seed)

    def generate(self, beam):
        beam.y = self.sigma_y * self.random_state.randn(beam.n_macroparticles)
        beam.yp = self.sigma_yp * self.random_state.randn(beam.n_macroparticles)
Example #27
0
def stitch(targets,images):
    mask = rois_mask(targets) # True where image data is
    gaps_mask = mask==False # True where infill needs to go
    # compute bounds relative to the camera field
    (x,y,w,h) = stitched_box(targets)
    uroi = img_as_float(stitch_raw(targets,images,(x,y,w,h))) # stitch with black infill

    # step 1: sparsely sample background mostly ignoring blob
    # compute gradient on both axes
    k = [[-3,-1,0,1,3],
         [-3,-1,0,1,3],
         [-3,-1,0,1,3],
         [-3,-1,0,1,3]]
    gy = convolve(uroi,k)
    gx = convolve(uroi,np.rot90(k))
    # ignore all but low-gradient areas
    bg = (abs(gy+gx) < 0.2) & mask

    # step 2: remove less contiguous areas
    filter_size = max(2,int(max(h,w)/200))
    mf = minimum_filter(bg*1,filter_size)

    # step 3: interpolate between samples
    z = inpaint(uroi*mf,mf==False)

    # step 4: subsample and re-interpolate to degrade artifacts in fill region
    random = RandomState(0)
    (h,w)=z.shape
    ng = random.rand(h,w) < 0.01
    z2 = inpaint(z*ng,ng==False)

    # step 5: final composite
    roi = (z2 * gaps_mask) + uroi
    return (roi * 255).astype(np.uint8), mask
Example #28
0
def make_trial():
    try:
        # t0 = time()
        # args:
        #   lenscale
        #   nPassiveObs
        #   rngstate

        params = request.json  # works when ajax request contentType specified as "applications/json"
        # unpack random number generator
        rng = RandomState()
        rngstate = unpack_rngstate(params['rngstate'])
        rng.set_state(rngstate)

        lenscale = float(params['lenscale'])
        nPassiveObs = int(params['nPassiveObs'])

        thisTri = boe.make_trial(nPassiveObs, DOMAIN, lenscale, SIGVAR, NOISEVAR2, XSAM_BOUNDS, rng)

        resp = {'sample': thisTri['sample'].tolist(),
                'xObs': thisTri['xObs'].flatten().tolist(),
                'yObs': thisTri['yObs'].tolist(),
                'iObs': thisTri['iObs'].tolist(),
                'rngstate': pack_rngstate(rng.get_state())}

    except:
        raise ExperimentError('improper_inputs')  # i don't like returning HTML to JSON requests...  maybe should change this

    return jsonify(**resp)
class GaussianX(PhaseSpace):
    """Horizontal Gaussian particle phase space distribution."""

    def __init__(self, sigma_x, sigma_xp, generator_seed=None):
        """Initiates the horizontal beam coordinates
        to the given Gaussian shape.
        """
        self.sigma_x  = sigma_x
        self.sigma_xp = sigma_xp

        self.random_state = RandomState()
        self.random_state.seed(generator_seed)

    @classmethod
    def from_optics(cls, alpha_x, beta_x, epsn_x, betagamma, generator_seed=None):
        """Initialise GaussianX from the given optics functions.
        beta_x is given in meters and epsn_x in micrometers.
        """

        sigma_x  = np.sqrt(beta_x * epsn_x * 1e-6 / betagamma)
        sigma_xp = sigma_x / beta_x

        return cls(sigma_x, sigma_xp, generator_seed)

    def generate(self, beam):
        beam.x = self.sigma_x * self.random_state.randn(beam.n_macroparticles)
        beam.xp = self.sigma_xp * self.random_state.randn(beam.n_macroparticles)
Example #30
0
def shared_normal(shape, scale=1, rng=None, name=None):
    """Initialize a matrix shared variable with normally distributed elements."""
    if rng is None:
        rng = RandomState(seed=np.random.randint(1 << 30))
    return theano.shared(rng.normal(
        scale=scale, size=shape).astype(theano.config.floatX),
        name=name)
Example #31
0
y_predict = tf.matmul(x, w1)

# 自定义损失函数
loss_less = 10
loss_more = 1

loss = tf.reduce_sum(
    tf.where(tf.greater(y, y_predict), (y - y_predict) * loss_more,
             (y_predict - y) * loss_less))

# 优化
trian_step = tf.train.AdamOptimizer(0.0001).minimize(loss)

# 随机生成模拟数据集
rdm = RandomState(1)
data_size = 128
X = rdm.rand(data_size, 2)  # 128个样本

# 加入噪音
Y = [[x1 + x2 + rdm.rand() / 10.0 - 0.05] for (x1, x2) in X]

# 训练
with tf.Session() as sess:
    init_op = tf.global_variables_initializer()
    sess.run(init_op)

    for i in range(5000):
        start = (i * batch_size) % data_size
        end = min(start + batch_size, data_size)
Example #32
0
from utils.loading import get_preproc_data, get_training_data
from utils.postprocess import scale_to_priors
from lasagne.layers import InputLayer, DenseLayer, DropoutLayer
from lasagne.objectives import Objective
from lasagne.updates import nesterov_momentum
from lasagne.init import HeUniform, Constant
from lasagne.nonlinearities import LeakyRectify, softmax
from functools import partial
from validation.crossvalidate import SampleCrossValidator
from validation.optimize_parallel import ParallelGridOptimizer

train_data, true_labels = get_training_data()[:2]
true_labels -= 1

cpus = max(cpu_count() - 1, 1)
random = RandomState()


def train_test(train, labels, test, weight_decay):
    net = NeuralNet(
        layers=[
            ('input', InputLayer),
            ('dropout0', DropoutLayer),
            ('dense1', DenseLayer),
            ('dropout1', DropoutLayer),
            ('dense2', DenseLayer),
            ('dropout2', DropoutLayer),
            ('dense3', DenseLayer),
            ('dropout3', DropoutLayer),
            ('output', DenseLayer),
        ],
Example #33
0
def extract_f0_func(gender):
    floor_sp, ceil_sp = -80, 30
    mel_basis = mel(16000, 1024, fmin=90, fmax=7600, n_mels=80).T
    min_level = np.exp(-100 / 20 * np.log(10))
    b, a = butter_highpass(30, 16000, order=5)

    # Set the directory you want to start from
    ROOT = r'E:\Dataset\VCTK\test_audio'
    rootDir = os.path.join(ROOT, 'audio')
    targetDir_f0 = os.path.join(ROOT, 'f0')
    targetDir = os.path.join(ROOT, 'mel-sp')

    pt = glob.glob1(rootDir, '*')

    cep_all = []
    dirName, subdirList, _ = next(os.walk(rootDir))
    print('Found directory: %s' % dirName)
    for subdir in sorted(pt):
        print(subdir)
        if not os.path.exists(os.path.join(targetDir, subdir)):
            os.makedirs(os.path.join(targetDir, subdir))
        if not os.path.exists(os.path.join(targetDir_f0, subdir)):
            os.makedirs(os.path.join(targetDir_f0, subdir))
        _, _, fileList = next(os.walk(os.path.join(dirName, subdir)))
        if gender == 'M':
            lo, hi = 50, 250
        elif gender == 'F':
            lo, hi = 100, 600
        else:
            raise ValueError
        prng = RandomState(0)
        for fileName in sorted(fileList):
            print(subdir, fileName)
            x, fs = sf.read(os.path.join(dirName, subdir, fileName))
            if (len(x.shape) >= 2):
                x = x[:, 0]
            if x.shape[0] % 256 == 0:
                x = np.concatenate((x, np.array([1e-06])), axis=0)
            y = signal.filtfilt(b, a, x)
            wav = y * 0.95 + (prng.rand(y.shape[0]) - 0.5) * 1e-06
            D = pySTFT(wav).T
            D_mel = np.dot(D, mel_basis)
            D_db = 20 * np.log10(np.maximum(min_level, D_mel)) - 16
            S = (D_db + 100) / 100

            f0_rapt = sptk.rapt(wav.astype(np.float32) * 32768,
                                fs,
                                256,
                                min=lo,
                                max=hi,
                                otype=2)
            index_nonzero = (f0_rapt != -1e10)
            tmp = f0_rapt[index_nonzero]
            mean_f0, std_f0 = np.mean(tmp), np.std(tmp)

            f0_norm = speaker_normalization(f0_rapt, index_nonzero, mean_f0,
                                            std_f0)

            if len(S) != len(f0_norm):
                pdb.set_trace()

            np.save(os.path.join(targetDir, subdir, fileName[:-4]),
                    S.astype(np.float32),
                    allow_pickle=False)

            np.save(os.path.join(targetDir_f0, subdir, fileName[:-4]),
                    f0_norm.astype(np.float32),
                    allow_pickle=False)

            print(S.shape)
            print(f0_norm.shape)
Example #34
0
import numpy as np
from electromorpho.core.gaussian import sample_from_gn
from electromorpho.metrics.score import BGe
from numpy.random import RandomState
from electromorpho.structure.graph_generation import random_dag

from electromorpho.mcmc.graphs.state_space import RestrictionViolation
from electromorpho.mcmc.graphs.checks import check_distribution
from electromorpho.mcmc.graphs.proposal import get_parent_set_distributions

n_variables = 15
seeds = list(range(101, 200))
rng = RandomState(19023)
variables = list(range(n_variables))
n_samples = 200

# Data generation parameters
gen_mean = np.zeros(n_variables)
gen_var = np.zeros(n_variables) + 0.2
gen_weight = 2

# Generate some data form a GN
graph = random_dag(variables, rng=rng)
beta = graph.A.T * gen_weight

sample_seed = rng.randint(0, 2**32 - 1)
data_gn = sample_from_gn(graph, gen_mean, gen_var, beta, n_samples,
                         sample_seed)

# Fit the score and create the parent set distributions
fan_in = 5
Example #35
0
    def __init__(self,
                 bullet_client,
                 offset=(0, 0, 0),
                 sim_time=0.,
                 scale=1.,
                 parameter_distributions=None):

        self.logger = logging.Logger(f"robot:panda:{bullet_client}")

        if parameter_distributions is not None:
            logging.warning("Domain randomization not implemented for IIWA")
            raise NotImplementedError()

        self.time_step = bullet_client.getPhysicsEngineParameters(
        )["fixedTimeStep"]

        if not sim_time:
            sim_time = self.time_step

        self.scale = scale

        self.max_steps = int(sim_time / self.time_step)

        self.offset = offset

        self.bullet_client = bullet_client

        self.random = RandomState(
            int.from_bytes(os.urandom(4), byteorder='little'))

        # load robot in simulation
        urdf_file = os.path.join(str(Path(__file__).absolute().parent),
                                 "iiwa.urdf")

        self.model_id = bullet_client.loadURDF(
            urdf_file,
            np.array([0, 0, 0]) + self.offset,
            useFixedBase=True,
            flags=p.URDF_USE_SELF_COLLISION | p.URDF_MAINTAIN_LINK_ORDER)

        self.joint_name2id = {}

        for jj in range(self.bullet_client.getNumJoints(self.model_id)):
            jointInfo = self.bullet_client.getJointInfo(self.model_id, jj)
            self.joint_name2id[jointInfo[1].decode("utf-8")] = jointInfo[0]

            print(jointInfo[1].decode("utf-8"))

        Joint = namedtuple(
            "Joint",
            ["id", "initial_position", "limits", "max_velocity", "max_torque"])

        # self.joints = {
        #     0: Joint(0, (-2.96705972839, 2.96705972839), 1.71042266695, 320),
        #     1: Joint(0, (-2.09439510239, 2.09439510239), 1.71042266695, 320),
        #     2: Joint(0, (-2.96705972839, 2.96705972839), 1.74532925199, 176),
        #     3: Joint(0, (-2.09439510239, 2.09439510239), 2.26892802759, 176),
        #     4: Joint(0, (-2.96705972839, 2.96705972839), 2.44346095279, 110),
        #     5: Joint(0, (-2.09439510239, 2.09439510239), 3.14159265359, 40),
        #     6: Joint(0, (-3.05432619099, 3.05432619099), 3.14159265359, 40),

        self.joints_arm = {
            "lbr_iiwa_joint_1":
            Joint(self.joint_name2id["lbr_iiwa_joint_1"], 0,
                  (-2.96705972839, 2.96705972839), 1.71042266695, 320),
            "lbr_iiwa_joint_2":
            Joint(self.joint_name2id["lbr_iiwa_joint_2"], 0,
                  (-2.09439510239, 2.09439510239), 1.71042266695, 320),
            "lbr_iiwa_joint_3":
            Joint(self.joint_name2id["lbr_iiwa_joint_3"], 0,
                  (-2.96705972839, 2.96705972839), 1.74532925199, 176),
            "lbr_iiwa_joint_4":
            Joint(self.joint_name2id["lbr_iiwa_joint_4"], 0,
                  (-2.09439510239, 2.09439510239), 2.26892802759, 176),
            "lbr_iiwa_joint_5":
            Joint(self.joint_name2id["lbr_iiwa_joint_5"], 0,
                  (-2.96705972839, 2.96705972839), 2.44346095279, 110),
            "lbr_iiwa_joint_6":
            Joint(self.joint_name2id["lbr_iiwa_joint_6"], 0,
                  (-2.09439510239, 2.09439510239), 3.14159265359, 40),
            "lbr_iiwa_joint_7":
            Joint(self.joint_name2id["lbr_iiwa_joint_7"], 0,
                  (-3.05432619099, 3.05432619099), 3.14159265359, 40),
        }

        self.joints_hand = {
            # hand
            "left_outer_knuckle_joint":
            Joint(self.joint_name2id["left_outer_knuckle_joint"], 0.3,
                  (0.0, 0.725), 2., 15),
            "left_inner_knuckle_joint":
            Joint(self.joint_name2id["left_inner_knuckle_joint"], 0.3,
                  (0.0, 0.8757), 2., 15),
            "right_outer_knuckle_joint":
            Joint(self.joint_name2id["right_outer_knuckle_joint"], 0.3,
                  (0.0, 0.725), 2., 15),
            "right_inner_knuckle_joint":
            Joint(self.joint_name2id["right_inner_knuckle_joint"], 0.3,
                  (0.0, 0.8757), 2., 15),
        }

        self.joints_passive = {
            # hand
            "left_outer_finger_joint":
            Joint(self.joint_name2id["left_outer_finger_joint"], 0.3,
                  (0.0, 0.725), 2., 20),
            "left_inner_finger_joint":
            Joint(self.joint_name2id["left_inner_finger_joint"], 0.3,
                  (0.0, 0.8757), 2., 20),
            "right_outer_finger_joint":
            Joint(self.joint_name2id["right_outer_finger_joint"], 0.3,
                  (0.0, 0.725), 2., 20),
            "right_inner_finger_joint":
            Joint(self.joint_name2id["right_inner_finger_joint"], 0.3,
                  (0.0, 0.8757), 2., 20),
        }

        constraint = self.bullet_client.createConstraint(
            self.model_id, self.joint_name2id["left_inner_finger_loop"],
            self.model_id, self.joint_name2id["left_outer_finger_loop"],
            self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0],
            [0, 0, 0])
        self.bullet_client.changeConstraint(constraint, maxForce=1e4)

        constraint = self.bullet_client.createConstraint(
            self.model_id, self.joint_name2id["right_inner_finger_loop"],
            self.model_id, self.joint_name2id["right_outer_finger_loop"],
            self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0],
            [0, 0, 0])
        self.bullet_client.changeConstraint(constraint, maxForce=1e4)

        from itertools import chain, permutations

        for linkA, linkB in permutations(
                chain(self.joints_hand.keys(), self.joints_passive.keys()), 2):
            self.bullet_client.setCollisionFilterPair(
                self.model_id, self.model_id, self.joint_name2id[linkA],
                self.joint_name2id[linkB], False)

        self.bullet_client.setCollisionFilterPair(
            self.model_id, self.model_id,
            self.joint_name2id["left_inner_finger_joint"],
            self.joint_name2id["right_inner_finger_joint"], False)

        # todo introduce friction
        # self.bullet_client.setJointMotorControlArray(self.model_id,
        #                                              [joint.id for _, joint in self.joints_arm.items()],
        #                                              p.VELOCITY_CONTROL,
        #                                              forces=[0.1 * joint.max_torque
        #                                                      for _, joint in self.joints_arm.items()])

        self.bullet_client.setJointMotorControlArray(
            self.model_id, [joint.id for _, joint in self.joints_hand.items()],
            p.VELOCITY_CONTROL,
            forces=[
                0 * joint.max_torque for _, joint in self.joints_hand.items()
            ])

        self.bullet_client.setJointMotorControlArray(
            self.model_id,
            [joint.id for _, joint in self.joints_passive.items()],
            p.VELOCITY_CONTROL,
            forces=[
                0 * joint.max_torque
                for _, joint in self.joints_passive.items()
            ])

        self.bullet_client.stepSimulation()

        # define spaces
        self.action_space = spaces.Box(-1.,
                                       1.,
                                       shape=(len(self.joints_arm) + 1, ))

        self.observation_space = spaces.Dict({
            "joint_positions":
            spaces.Box(-1., 1., shape=(len(self.joints_arm), )),
            "joint_velocities":
            spaces.Box(-1., 1., shape=(len(self.joints_arm), )),
            "tcp_position":
            spaces.Box(-1., 1., shape=(3, )),
            # "tcp_velocity": spaces.Box(-1., 1., shape=(3,)),
        })

        # reset to initial position
        self.reset()
Example #36
0
class IIWA:
    def __init__(self,
                 bullet_client,
                 offset=(0, 0, 0),
                 sim_time=0.,
                 scale=1.,
                 parameter_distributions=None):

        self.logger = logging.Logger(f"robot:panda:{bullet_client}")

        if parameter_distributions is not None:
            logging.warning("Domain randomization not implemented for IIWA")
            raise NotImplementedError()

        self.time_step = bullet_client.getPhysicsEngineParameters(
        )["fixedTimeStep"]

        if not sim_time:
            sim_time = self.time_step

        self.scale = scale

        self.max_steps = int(sim_time / self.time_step)

        self.offset = offset

        self.bullet_client = bullet_client

        self.random = RandomState(
            int.from_bytes(os.urandom(4), byteorder='little'))

        # load robot in simulation
        urdf_file = os.path.join(str(Path(__file__).absolute().parent),
                                 "iiwa.urdf")

        self.model_id = bullet_client.loadURDF(
            urdf_file,
            np.array([0, 0, 0]) + self.offset,
            useFixedBase=True,
            flags=p.URDF_USE_SELF_COLLISION | p.URDF_MAINTAIN_LINK_ORDER)

        self.joint_name2id = {}

        for jj in range(self.bullet_client.getNumJoints(self.model_id)):
            jointInfo = self.bullet_client.getJointInfo(self.model_id, jj)
            self.joint_name2id[jointInfo[1].decode("utf-8")] = jointInfo[0]

            print(jointInfo[1].decode("utf-8"))

        Joint = namedtuple(
            "Joint",
            ["id", "initial_position", "limits", "max_velocity", "max_torque"])

        # self.joints = {
        #     0: Joint(0, (-2.96705972839, 2.96705972839), 1.71042266695, 320),
        #     1: Joint(0, (-2.09439510239, 2.09439510239), 1.71042266695, 320),
        #     2: Joint(0, (-2.96705972839, 2.96705972839), 1.74532925199, 176),
        #     3: Joint(0, (-2.09439510239, 2.09439510239), 2.26892802759, 176),
        #     4: Joint(0, (-2.96705972839, 2.96705972839), 2.44346095279, 110),
        #     5: Joint(0, (-2.09439510239, 2.09439510239), 3.14159265359, 40),
        #     6: Joint(0, (-3.05432619099, 3.05432619099), 3.14159265359, 40),

        self.joints_arm = {
            "lbr_iiwa_joint_1":
            Joint(self.joint_name2id["lbr_iiwa_joint_1"], 0,
                  (-2.96705972839, 2.96705972839), 1.71042266695, 320),
            "lbr_iiwa_joint_2":
            Joint(self.joint_name2id["lbr_iiwa_joint_2"], 0,
                  (-2.09439510239, 2.09439510239), 1.71042266695, 320),
            "lbr_iiwa_joint_3":
            Joint(self.joint_name2id["lbr_iiwa_joint_3"], 0,
                  (-2.96705972839, 2.96705972839), 1.74532925199, 176),
            "lbr_iiwa_joint_4":
            Joint(self.joint_name2id["lbr_iiwa_joint_4"], 0,
                  (-2.09439510239, 2.09439510239), 2.26892802759, 176),
            "lbr_iiwa_joint_5":
            Joint(self.joint_name2id["lbr_iiwa_joint_5"], 0,
                  (-2.96705972839, 2.96705972839), 2.44346095279, 110),
            "lbr_iiwa_joint_6":
            Joint(self.joint_name2id["lbr_iiwa_joint_6"], 0,
                  (-2.09439510239, 2.09439510239), 3.14159265359, 40),
            "lbr_iiwa_joint_7":
            Joint(self.joint_name2id["lbr_iiwa_joint_7"], 0,
                  (-3.05432619099, 3.05432619099), 3.14159265359, 40),
        }

        self.joints_hand = {
            # hand
            "left_outer_knuckle_joint":
            Joint(self.joint_name2id["left_outer_knuckle_joint"], 0.3,
                  (0.0, 0.725), 2., 15),
            "left_inner_knuckle_joint":
            Joint(self.joint_name2id["left_inner_knuckle_joint"], 0.3,
                  (0.0, 0.8757), 2., 15),
            "right_outer_knuckle_joint":
            Joint(self.joint_name2id["right_outer_knuckle_joint"], 0.3,
                  (0.0, 0.725), 2., 15),
            "right_inner_knuckle_joint":
            Joint(self.joint_name2id["right_inner_knuckle_joint"], 0.3,
                  (0.0, 0.8757), 2., 15),
        }

        self.joints_passive = {
            # hand
            "left_outer_finger_joint":
            Joint(self.joint_name2id["left_outer_finger_joint"], 0.3,
                  (0.0, 0.725), 2., 20),
            "left_inner_finger_joint":
            Joint(self.joint_name2id["left_inner_finger_joint"], 0.3,
                  (0.0, 0.8757), 2., 20),
            "right_outer_finger_joint":
            Joint(self.joint_name2id["right_outer_finger_joint"], 0.3,
                  (0.0, 0.725), 2., 20),
            "right_inner_finger_joint":
            Joint(self.joint_name2id["right_inner_finger_joint"], 0.3,
                  (0.0, 0.8757), 2., 20),
        }

        constraint = self.bullet_client.createConstraint(
            self.model_id, self.joint_name2id["left_inner_finger_loop"],
            self.model_id, self.joint_name2id["left_outer_finger_loop"],
            self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0],
            [0, 0, 0])
        self.bullet_client.changeConstraint(constraint, maxForce=1e4)

        constraint = self.bullet_client.createConstraint(
            self.model_id, self.joint_name2id["right_inner_finger_loop"],
            self.model_id, self.joint_name2id["right_outer_finger_loop"],
            self.bullet_client.JOINT_POINT2POINT, [0, 0, 0], [0, 0, 0],
            [0, 0, 0])
        self.bullet_client.changeConstraint(constraint, maxForce=1e4)

        from itertools import chain, permutations

        for linkA, linkB in permutations(
                chain(self.joints_hand.keys(), self.joints_passive.keys()), 2):
            self.bullet_client.setCollisionFilterPair(
                self.model_id, self.model_id, self.joint_name2id[linkA],
                self.joint_name2id[linkB], False)

        self.bullet_client.setCollisionFilterPair(
            self.model_id, self.model_id,
            self.joint_name2id["left_inner_finger_joint"],
            self.joint_name2id["right_inner_finger_joint"], False)

        # todo introduce friction
        # self.bullet_client.setJointMotorControlArray(self.model_id,
        #                                              [joint.id for _, joint in self.joints_arm.items()],
        #                                              p.VELOCITY_CONTROL,
        #                                              forces=[0.1 * joint.max_torque
        #                                                      for _, joint in self.joints_arm.items()])

        self.bullet_client.setJointMotorControlArray(
            self.model_id, [joint.id for _, joint in self.joints_hand.items()],
            p.VELOCITY_CONTROL,
            forces=[
                0 * joint.max_torque for _, joint in self.joints_hand.items()
            ])

        self.bullet_client.setJointMotorControlArray(
            self.model_id,
            [joint.id for _, joint in self.joints_passive.items()],
            p.VELOCITY_CONTROL,
            forces=[
                0 * joint.max_torque
                for _, joint in self.joints_passive.items()
            ])

        self.bullet_client.stepSimulation()

        # define spaces
        self.action_space = spaces.Box(-1.,
                                       1.,
                                       shape=(len(self.joints_arm) + 1, ))

        self.observation_space = spaces.Dict({
            "joint_positions":
            spaces.Box(-1., 1., shape=(len(self.joints_arm), )),
            "joint_velocities":
            spaces.Box(-1., 1., shape=(len(self.joints_arm), )),
            "tcp_position":
            spaces.Box(-1., 1., shape=(3, )),
            # "tcp_velocity": spaces.Box(-1., 1., shape=(3,)),
        })

        # reset to initial position
        self.reset()

    def reset(self, desired_state=None):
        """Reset robot to initial pose and return new state."""

        contact_points = True

        if desired_state is not None:

            desired_state = list(desired_state)

            for _, joint in self.joints_arm.items():
                joint_position = np.interp(desired_state.pop(0), [-1, 1],
                                           joint.limits)

                self.bullet_client.resetJointState(self.model_id, joint.id,
                                                   joint_position)

            self.bullet_client.stepSimulation()
            contact_points = self.bullet_client.getContactPoints(
                self.model_id, self.model_id)

        # reset until state is valid
        while contact_points:

            for _, joint in self.joints_hand.items():
                joint_position = self.random.uniform(*joint.limits)

                self.bullet_client.resetJointState(self.model_id, joint.id,
                                                   joint_position)

            self.bullet_client.stepSimulation()
            contact_points = self.bullet_client.getContactPoints(
                self.model_id, self.model_id)

        observation = self.get_observation()

        return observation

    def step(self, action: np.ndarray):
        assert self.action_space.contains(action), f"{action}"

        action_arm = action[:6]
        action_hand = action[-1]

        action_arm = list(action_arm * self.scale)  # / self.max_steps)

        joint_ids = []
        target_arm_positions = []
        maxVelocities = []
        torques_arm = []

        for (_, joint), action_joint in zip(self.joints_arm.items(),
                                            action_arm):

            position, _, _, _ = self.bullet_client.getJointState(
                self.model_id, joint.id)

            normalized_joint_position = np.interp(position, joint.limits,
                                                  [-1, 1])
            normalized_target_joint_position = np.clip(
                normalized_joint_position + action_joint, -1, 1)
            target_joint_position = np.interp(normalized_target_joint_position,
                                              [-1, 1], joint.limits)

            joint_ids.append(joint.id)
            target_arm_positions.append(target_joint_position)
            torques_arm.append(joint.max_torque)

            maxVelocities.append(joint.max_velocity)
            # if joint.id == 2:
            #     print(position, target_joint_position)

            # self.bullet_client.setJointMotorControl2(self.model_id,
            #                                              joint.id,
            #                                              p.POSITION_CONTROL,
            #                                              targetPosition=target_joint_position,
            #                                              maxVelocity=joint.max_velocity,
            #                                              force=joint.max_torque
            #                                              )

        print(target_arm_positions)

        self.bullet_client.setJointMotorControlArray(
            self.model_id,
            joint_ids,
            p.POSITION_CONTROL,
            targetPositions=target_arm_positions,
            # maxVelocities=maxVelocities,
            forces=torques_arm)
        # joint_ids = []
        # target_hand_positions = []
        # torques_hand = []
        #
        # for _, joint in self.joints_hand.items():
        #
        #     position, _, _, _ = self.bullet_client.getJointState(self.model_id, joint.id)
        #
        #     normalized_joint_position = np.interp(position, joint.limits, [-1, 1])
        #     normalized_target_joint_position = np.clip(normalized_joint_position + action_hand, -1, 1)
        #     target_joint_position = np.interp(normalized_target_joint_position, [-1, 1], joint.limits)
        #
        #     joint_ids.append(joint.id)
        #     target_hand_positions.append(target_joint_position)
        #     torques_hand.append(joint.max_torque)
        #
        # self.bullet_client.setJointMotorControlArray(self.model_id,
        #                                              joint_ids,
        #                                              p.POSITION_CONTROL,
        #                                              targetPositions=target_hand_positions,
        #                                              forces=torques_hand
        #                                              )

        joint_ids = [joint.id for _, joint in self.joints_hand.items()]
        torques_fingers = [
            action_hand * joint.max_torque
            for _, joint in self.joints_hand.items()
        ]

        for step in range(self.max_steps):
            self.bullet_client.setJointMotorControlArray(
                self.model_id,
                joint_ids,
                p.TORQUE_CONTROL,
                forces=torques_fingers)

            self.bullet_client.stepSimulation()

            if self.bullet_client.getConnectionInfo(
            )["connectionMethod"] == p.GUI:
                time.sleep(self.time_step)

        observation = self.get_observation()

        return observation

    def get_observation(self):
        joint_positions, joint_velocities = [], []

        for _, joint in self.joints_arm.items():
            joint_position, joint_velocity, _, _ = self.bullet_client.getJointState(
                self.model_id, joint.id)

            joint_positions.append(
                np.interp(joint_position, joint.limits, [-1, 1]))
            joint_velocities.append(
                np.interp(joint_velocity,
                          [-joint.max_velocity, joint.max_velocity], [-1, 1]))

        tcp_position, _, _, _, _, _, tcp_velocity, _ = \
            self.bullet_client.getLinkState(self.model_id, self.joint_name2id["tcp"],
                                            computeLinkVelocity=True)

        joint_positions = np.array(joint_positions)
        joint_velocities = np.array(joint_velocities)
        tcp_position = np.array(tcp_position)
        # tcp_velocity = np.array(tcp_velocity)

        observation = {
            "joint_positions": joint_positions,
            "joint_velocities": joint_velocities,
            "tcp_position": tcp_position,
            # "tcp_velocity": tcp_velocity
        }

        for key in observation:
            observation[key] = observation[key].clip(
                self.observation_space[key].low,
                self.observation_space[key].high)

        return observation
def main_box_plot(num_dim = 1, num_neurons = 2, analyze_sensors = True,
    analyze_brain = True, analyze_motors = False,
    combined_complexity = True, only_part_n1n2 = True, 
    input_dir = 'data', csv_dir = None, plot_dir = None):
    """
    Calculates neural complexity for different conditions,
    saves it to CSV and outputs boxplots.
    """
    

    rs = RandomState(1)

    selected_nodes_str_list = [
        n for n, b in zip(
            ['sensors', 'brain', 'motors'],
            [analyze_sensors, analyze_brain, analyze_motors]
        ) if b
    ]

    all_NC = []

    if combined_complexity:
        dir_pop_index = [
            (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_switch', 0),
            (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual', 0),
        ]
        x_labels = ['gen', 'spec']
    else:
        dir_pop_index = [
            (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill/', 0),
            (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_switch', 0),
            (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual', 0),
            (f'{input_dir}/{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual', 1)
        ]
        x_labels = ['iso', 'gen', 'spec-left', 'spec-right']

    for in_dir, pop_index in dir_pop_index:
        SEEDS, GEN, BP, NC = get_seeds_generations_complexities(
            in_dir, analyze_sensors, analyze_brain, analyze_motors,
            pop_index, only_last_generation=True, filter_performance_threshold=20.0,            
            combined_complexity=combined_complexity, 
            only_part_n1n2=only_part_n1n2,
            rs=rs)

        NC = np.squeeze(NC)
        # print(NC)
        # print(NC.shape)
        all_NC.append(NC)

    all_NC = np.array(all_NC)  # 4 x 20
    print(all_NC.shape)
    selected_nodes_file_str = '_'.join([x[:3] for x in selected_nodes_str_list])
    combined_str = '_combined' if combined_complexity else ''    
    only_part_n1n2_str = '_onlyN1N2' if only_part_n1n2 else ''    

    f_name = f"{num_dim}d_{num_neurons}n_box_TSE_{selected_nodes_file_str}{combined_str}{only_part_n1n2_str}" 

    # save file to csv    
    if csv_dir is not None:
        f_path = os.path.join(
            csv_dir,
            f_name + '.csv'
        )
        print('saving csv:', f_path)
        df = pd.DataFrame(np.transpose(all_NC), columns=x_labels)  # 20 x 4
        df.to_csv(f_path, index=False)

    all_NC_not_NaN = [x[~np.isnan(x)] for x in all_NC]
    plt.boxplot(all_NC_not_NaN, labels=x_labels)
    selected_nodes_str = ', '.join(selected_nodes_str_list)
    title = f'Neural Complexity - {num_neurons}n {selected_nodes_str}'
    if combined_complexity:
        title += ' - combined'
    if only_part_n1n2:
        title += ' - onlyN1N2'
    plt.title(title)

    if plot_dir is not None:
        f_path = os.path.join(
            plot_dir,
            f_name + '.pdf'
        )
        plt.savefig(f_path)  # remove csv and add pdf
        print('saving pdf:', f_path)
        plt.clf()
    else:
        plt.show()
def main_line_plot(num_dim = 1, num_neurons = 2, sim_type = 'individuals',
    analyze_sensors = True, analyze_brain = True, analyze_motors = False,
    tse_max=2, combined_complexity = False, only_part_n1n2 = True, 
    input_dir = 'data', plot_dir = None, csv_dir = None):
    """
    Given a hard-coded directory with N simulation seeds,
    plots N line subplots, each containing 2 lines:
    - performance over generations
    - hardcoded complexity over generations
    """    

    sim_type_dir = {
        'individuals': f'{num_dim}d_{num_neurons}n_exc-0.1_zfill',
        'generalists': f'{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_switch',
        'specialists': f'{num_dim}d_{num_neurons}n_exc-0.1_zfill_rp-3_dual'
    }
    
    assert sim_type in sim_type_dir, f'sim_type must be in {sim_type_dir.keys()}'
    
    exp_dir = sim_type_dir[sim_type]
    in_dir = os.path.join(input_dir, exp_dir)

    pop_index = 0

    rs = RandomState(1)

    SEEDS, GEN, BP, NC = get_seeds_generations_complexities(
        in_dir, analyze_sensors, analyze_brain, analyze_motors,
        pop_index=pop_index, only_last_generation=False,        
        filter_performance_threshold=20, # exclude not converged seeds
        combined_complexity=combined_complexity, 
        only_part_n1n2=only_part_n1n2,
        rs=rs
    )

    fig = plt.figure(figsize=(10, 6))
    num_plots = len(GEN)
    num_plot_cols = 5
    num_plot_rows = int(num_plots / num_plot_cols)
    if num_plots % num_plot_cols > 0:
        num_plot_rows += 1

    if csv_dir is not None or plot_dir is not None:
        selected_nodes_str_list = [
                n for n, b in zip(
                    ['sensors', 'brain', 'motors'],
                    [analyze_sensors, analyze_brain, analyze_motors]
                ) if b
            ]
        selected_nodes_file_str = '_'.join([x[:3] for x in selected_nodes_str_list])
        combined_str = '_combined' if combined_complexity else ''    
        only_part_n1n2_str = '_onlyN1N2' if only_part_n1n2 else ''    
        fname = f'{num_dim}d_{num_neurons}n_gen_seeds_TSE_{sim_type}_{selected_nodes_file_str}{combined_str}{only_part_n1n2_str}'

    if csv_dir is not None:
        # save file to csv    
    
        f_path = os.path.join(
            csv_dir, 
            fname + '.csv'
        )
        print('saving csv:', f_path)    

        num_seeds, gen_list_size = np.array(NC).shape
        assert gen_list_size == len(GEN[0])
        csv_data = np.transpose( # gen_list_size x num_seeds
            np.concatenate(
                [
                    np.array(GEN[0]).reshape((1,gen_list_size)), # need only 1 list of generations
                    NC
                ]
            )
        )
        seeds_str = [f'seed_{s}' for s in SEEDS]
        assert len(seeds_str) == num_seeds
        cols_names = ['GEN'] + seeds_str
        df = pd.DataFrame(csv_data, columns=cols_names) 
        df.to_csv(f_path, index=False)

    for seed_num, (num_gen_list, best_perfs, nc_seed) in enumerate(zip(GEN, BP, NC), 1):
        ax1 = fig.add_subplot(num_plot_rows, num_plot_cols, seed_num)
        ax1.plot(num_gen_list, nc_seed)  # , label=str(seed_num)
        ax1.set_ylim(0, tse_max)
        ax2 = ax1.twinx()
        ax2.plot(np.arange(len(best_perfs)), best_perfs, color='orange')
        ax2.set_ylim(0, 100)
        # plt.legend()
    fig.tight_layout()
    
    selected_nodes_str = ', '.join(selected_nodes_str_list)    
    title = f'Perf & TSE on generations - {num_neurons}n {sim_type} {selected_nodes_str}'
    if combined_complexity:
        title += ' - combined'
    if only_part_n1n2:
        title += ' - onlyN1N2'
    
    fig.suptitle(title)    
    fig.subplots_adjust(top=0.88)

    if plot_dir is not None:
        f_path = os.path.join(
            plot_dir, 
            fname + '.pdf'
        )
        plt.savefig(f_path)  # remove csv and add pdf
        print('saving pdf:', f_path)        
    else:
        plt.show()
Example #39
0
def min_objective(Xu, Xs, L, last_U, last_V, samples):
    global skipped_elements
    learn_rate = 0.05
    nusers, nitems = Xu.shape
    _, nfeatures = Xs.shape
    nz_indices = np.nonzero(Xu)
    nz_elements = len(nz_indices[0])
    print "non zero entries are: ", nz_elements
    rs = RandomState(1234567890)
    U = rs.standard_normal((nusers, dim))
    U = normalize(U, norm='l2', axis=1)
    V = rs.standard_normal((nitems, dim))
    V = normalize(V, norm='l2', axis=1)
    E = np.random.standard_normal((nitems, dim))
    prev_V = np.array(V)
    prev_U = np.array(U)
    prev_E = np.array(E)
    skipped = 0
    skip_i = []
    skip_j = []
    # W = np.random.uniform(-0.001, 0.001, (dim, nfeatures))/100
    obj = 10
    for itera in xrange(nIterations):
        i = samples[itera][0]
        j = samples[itera][1]
        # print "iteration: ", itera
        # if itera % 2 == 0:
        # 	ind = rd.choice(xrange(nz_elements))
        # 	i = nz_indices[0][ind]
        # 	j = nz_indices[1][ind]
        # else:
        # 	i = rd.choice(xrange(nusers))
        # 	j = rd.choice(xrange(nitems))
        # i = rd.choice(xrange(nusers))
        # j = rd.choice(xrange(nitems))
        if Xu[i, j] > 0:
            y = 1
        else:
            y = -1

        if skipped_elements[i, j] == 1 or (
                last_U != None
                and y * np.dot(last_U[i, :], last_V[j, :]) < -1):
            if itera > 1 and itera % 50000 == 0:
                obj = np.sum((U - prev_U)**2) + np.sum(
                    (V - prev_V)**2) + np.sum((E - prev_E)**2)
                print "iterations: ", itera, " obj: ", obj
                prev_V = np.array(V)
                prev_U = np.array(U)
                prev_E = np.array(E)
                learn_rate /= 1.01
                if obj < 0.01:
                    break
            skipped += 1
            skip_i.append(i)
            skip_j.append(j)
            continue
            # skipped_elements[i, j] = 1

        t = y * np.dot(U[i, :], V[j, :])

        if 1 - t > 0:
            gradu = -1 * y * V[j, :]
            gradv = -1 * y * U[i, :]
        else:
            gradu = 0
            gradv = 0

        # print "original: ", y
        # print "before: ",  np.dot(U[i, :], V[j, :])

        U[i, :] = U[i, :] - learn_rate * (
            alpha * gradu) - learn_rate * lambd * 2 * U[i, :]
        # print "loss1: ", y*np.dot(U[i, :], V[j, :]), "loss2: ", np.linalg.norm(V[j, :]-L[j,:])**2
        V[j, :] = V[j, :] - learn_rate * (
            alpha * gradv + (1 - alpha) *
            (2 * V[j, :] - 2 * L[j, :])) - learn_rate * lambd * 2 * V[j, :]
        # E[j, :] = E[j, :] - learn_rate * ((1-alpha) * ( 2*np.dot(Xs[j, :].toarray(), W.T) - 2*np.dot((V[j,:]-E[j, :]), np.dot(W, W.T)))) - learn_rate * lambd_E * 2 * E[j, :]
        # V[j, :] = V[j, :] - learn_rate * (alpha * gradv) - learn_rate * lambd * 2 * V[j, :]

        # print "after: ", np.dot(U[i, :], V[j, :])
        if itera > 1 and itera % 50000 == 0:
            obj = np.sum((U - prev_U)**2) + np.sum((V - prev_V)**2) + np.sum(
                (E - prev_E)**2)
            print "iterations: ", itera, " obj: ", obj
            prev_V = np.array(V)
            prev_U = np.array(U)
            prev_E = np.array(E)
            learn_rate /= 1.01
            # learn_rate /= 1.0001
            if obj < 0.01:
                break

        # W = W - learn_rate * ((1-alpha) *(-2*np.dot(V[j,:].reshape(1, -1).T, Xs[j, :].toarray().reshape(1, -1)) + 2* np.dot(np.dot(V[j,:].reshape(1, -1).T, V[j, :].reshape(1, -1)), W))) - learn_rate*lambd*2*W

        # if dump > 1:
        # 	print "before: ", Xu[i, j], dump
        # 	print "after: ", Xu[i, j], np.dot(U[i,:], V[j, :])
        # 	print grad1u, grad1v, grad2u, grad2v

        # if itera % 1000 ==0:
        # 	su =0
        # 	for k in range(10000):
        # 		i = rd.choice(xrange(nusers))
        # 		j = rd.choice(xrange(nitems))
        # 		if Xu[i, j] == 0:
        # 			y = -1
        # 		else:
        # 			y = 1
        # 		if math.isnan(np.dot(U[i,:], V[j, :])):
        # 			print U[i,:], V[j, :], np.dot(U[i,:], V[j, :])
        # 			return
        # 		su+= np.dot(y*U[i,:], V[j, :])
        # 	print "objective: ", su/10000.0
        # 	# learn_rate /= 1.1
    print "skipped: ", skipped
    skipped_elements[skip_i, skip_j] = 1
    return U, V
Example #40
0
import time
from math import sqrt

import numpy as np
from numpy.random import RandomState
from sklearn.datasets import fetch_olivetti_faces

from modl.dict_fact import DictMF

n_row, n_col = 3, 6
n_components = n_row * n_col
image_shape = (64, 64)
rng = RandomState(0)

###############################################################################
# Load faces data
dataset = fetch_olivetti_faces(shuffle=True, random_state=rng)
faces = dataset.data

n_samples, n_features = faces.shape

# global centering
faces_centered = faces - faces.mean(axis=0)

# local centering
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)
faces_centered /= np.sqrt(np.sum(faces_centered**2, axis=1))[:, np.newaxis]

print("Dataset consists of %d faces" % n_samples)

Example #41
0
class Activities():
    """ Generates the activity chains. """

    ## Activity
    Activity = collections.namedtuple(
        'Activity',
        ['activity', 'fromEdge', 'toEdge', 'arrivalPos', 'start', 'duration', 'final'])
    Activity.__new__.__defaults__ = (None,) * len(Activity._fields)

    def __init__(self, conf, sumo, environment, logger, profiling=False):
        """
        Initialize the synthetic population.
            :param conf: distionary with the configurations
            :param sumo: already initialized SUMO simulation (TraCI or LibSUMO)
            :param profiling=False: enable cProfile
        """
        self._conf = conf
        self._sumo = sumo
        self._cache = {}
        self._env = environment
        self.logger = logger

        self._max_retry_number = 1000
        if 'maxNumTry' in conf:
            self._max_retry_number = conf['maxNumTry']

        self._profiling = profiling

        self._random_generator = RandomState(seed=self._conf['seed'])

    # Activity Locations

    def _stages_define_locations_position(self, person_stages):
        """ Define the position of each location in the activity chain. """
        home_pos = None
        primary_pos = None

        for pos, stage in person_stages.items():
            if 'Home' in stage.activity:
                if not home_pos:
                    home_pos = self._env.get_random_pos_from_edge(stage.toEdge)
                person_stages[pos] = stage._replace(arrivalPos=home_pos)
            elif 'P-' in stage.activity:
                if not primary_pos:
                    primary_pos = self._env.get_random_pos_from_edge(stage.toEdge)
                person_stages[pos] = stage._replace(arrivalPos=primary_pos)
            else:
                ## Secondary activities
                person_stages[pos] = stage._replace(
                    arrivalPos=self._env.get_random_pos_from_edge(stage.toEdge))

        return person_stages

    def _stages_define_main_locations(self, from_area, to_area, mode):
        """ Define a generic Home and Primary activity location.
            The locations must be reachable in some ways.
        """
        ## Mode split:
        _mode, _ptype, _vtype = sumoutils.get_intermodal_mode_parameters(
            mode, self._conf['intermodalOptions']['vehicleAllowedParking'])

        route = None
        from_edge = None
        to_edge = None
        _retry_counter = 0
        while not route and _retry_counter < self._max_retry_number:
            _retry_counter += 1
            ## Origin and Destination Selection
            from_edge, to_edge = self._env.select_pair(from_area, to_area)
            from_allowed = (
                self._env.sumo_network.getEdge(from_edge).allows('pedestrian') and
                self._env.sumo_network.getEdge(from_edge).allows('passenger') and
                self._env.sumo_network.getEdge(
                    from_edge).getLength() > self._conf['minEdgeAllowed'])
            to_allowed = (
                self._env.sumo_network.getEdge(to_edge).allows('pedestrian') and
                self._env.sumo_network.getEdge(to_edge).allows('passenger') and
                self._env.sumo_network.getEdge(to_edge).getLength() > self._conf['minEdgeAllowed'])
            if self._env.valid_pair(from_edge, to_edge) and from_allowed and to_allowed:
                try:
                    route = self._sumo.simulation.findIntermodalRoute(
                        from_edge, to_edge, modes=_mode, pType=_ptype, vType=_vtype)
                    if not sumoutils.is_valid_route(
                            mode, route, self._conf['intermodalOptions']['vehicleAllowedParking']):
                        route = None
                        self.logger.debug(
                            '_stages_define_main_locations: findIntermodalRoute mode unusable.')
                except TraCIException:
                    self.logger.debug('_stages_define_main_locations: findIntermodalRoute FAILED.')
                    route = None
            else:
                self.logger.debug('_stages_define_main_locations: unusable pair of edges.')
        if route:
            return from_edge, to_edge
        raise sagaexceptions.TripGenerationActivityError(
            'Locations for the main activities not found between {} and {} using {}.'.format(
                from_area, to_area, mode))

    def _stages_define_secondary_locations(self, person_stages, home, primary):
        """ Define secondary activity locations. """
        for pos, stage in person_stages.items():
            if  'S-' in stage.activity:
                ## look for what is coming before
                _prec = None
                _pos = pos - 1
                while not _prec and _pos in person_stages:
                    if 'Home' in person_stages[_pos].activity:
                        _prec = 'H'
                    elif 'P-' in person_stages[_pos].activity:
                        _prec = 'P'
                    _pos -= 1

                ## look for what is coming next
                _succ = None
                _pos = pos + 1
                while not _succ and _pos in person_stages:
                    if 'Home' in person_stages[_pos].activity:
                        _succ = 'H'
                    elif 'P-' in person_stages[_pos].activity:
                        _succ = 'P'
                    _pos += 1

                destination = None
                if _prec == 'H' and _succ == 'H':
                    destination = self._random_location_circle(center=home, other=primary)
                elif _prec == 'P' and _succ == 'P':
                    destination = self._random_location_circle(center=primary, other=home)
                elif _prec != _succ:
                    destination = self._random_location_ellipse(home, primary)
                else:
                    raise sagaexceptions.TripGenerationActivityError(
                        'Invalid sequence in the activity chain: {} --> {}'.format(_prec, _succ),
                        person_stages)

                person_stages[pos] = stage._replace(toEdge=destination)
        return person_stages

    def _random_location_circle(self, center, other):
        """ Return a random edge in within a radius (*) from the given center.

            (*) Uses the ellipses defined by the foci center and other,
                and the major axe of 1.30 * distance between the foci.
        """
        try:
            length = self._get_cached_dist(center, other)
        except TraCIException:
            raise sagaexceptions.TripGenerationActivityError(
                'No route between {} and {}'.format(center, other))
        major_axe = length * 1.3
        minor_axe = numpy.sqrt(numpy.square(major_axe) - numpy.square(length))
        radius = minor_axe / 2.0

        self.logger.debug('_random_location_circle: %s [%.2f]', center, radius)
        edges = self._env.get_all_neigh_edges(center, radius)
        if not edges:
            raise sagaexceptions.TripGenerationActivityError(
                'No edges from {} with range {}.'.format(center, length))

        ret = self._random_generator.choice(edges)
        edges.remove(ret)
        allowed = (
            self._env.sumo_network.getEdge(ret).allows('pedestrian') and
            self._env.sumo_network.getEdge(ret).allows('passenger') and
            ret != center and ret != other and
            self._env.sumo_network.getEdge(ret).getLength() > self._conf['minEdgeAllowed'])
        while edges and not allowed:
            ret = self._random_generator.choice(edges)
            edges.remove(ret)
            allowed = (
                self._env.sumo_network.getEdge(ret).allows('pedestrian') and
                self._env.sumo_network.getEdge(ret).allows('passenger') and
                ret != center and ret != other and
                self._env.sumo_network.getEdge(ret).getLength() > self._conf['minEdgeAllowed'])

        if not edges:
            raise sagaexceptions.TripGenerationActivityError(
                'No valid edges from {} with range {}.'.format(center, length))
        return ret

    def _random_location_ellipse(self, focus1, focus2):
        """ Return a random edge in within the ellipse defined by the foci,
            and the major axe of 1.30 * distance between the foci.
        """
        try:
            length = self._get_cached_dist(focus1, focus2)
            self.logger.debug('_random_location_ellipse: %s --> %s [%.2f]', focus1, focus2, length)
        except TraCIException:
            raise sagaexceptions.TripGenerationActivityError(
                'No route between {} and {}'.format(focus1, focus2))

        major_axe = length * 1.3

        edges = self._env.get_all_neigh_edges(focus1, length)
        while edges:
            edge = self._random_generator.choice(edges)
            edges.remove(edge)
            if edge in (focus1, focus2):
                continue
            allowed = (
                self._env.sumo_network.getEdge(edge).allows('pedestrian') and
                self._env.sumo_network.getEdge(edge).allows('passenger') and
                self._env.sumo_network.getEdge(edge).getLength() > self._conf['minEdgeAllowed'])
            if not allowed:
                continue
            try:
                first = self._get_cached_dist(focus1, edge)
                second = self._get_cached_dist(edge, focus2)
                if first + second <= major_axe:
                    self.logger.debug('_random_location_ellipse: %s --> %s [%.2f]', focus1, edge, first)
                    self.logger.debug(
                        '_random_location_ellipse: %s --> %s [%.2f]', edge, focus2, second)
                    return edge
            except TraCIException:
                pass

        raise sagaexceptions.TripGenerationActivityError(
            "No location available for _random_location_ellipse [{}, {}]".format(focus1, focus2))

    # Chain

    def generate_person_stages(self, from_area, to_area, activity_chain, mode):
        """ Returns the trip for the given activity chain. """

        # Define a generic Home and Primary activity location.
        from_edge, to_edge = self._stages_define_main_locations(from_area, to_area, mode)

        ## Generate preliminary stages for a person
        person_stages = dict()
        for pos, activity in enumerate(activity_chain):
            if activity not in self._conf['activities']:
                raise sagaexceptions.TripGenerationActivityError(
                    'Activity {} is not define in the config file.'.format(activity))
            _start, _duration = self._get_timing_from_activity(activity)
            if pos == 0:
                if activity != 'Home':
                    raise sagaexceptions.TripGenerationActivityError(
                        "Every activity chain MUST start with 'Home', '{}' given.".format(activity))
                ## Beginning
                person_stages[pos] = self.Activity(
                    activity=activity, fromEdge=from_edge, start=_start, duration=_duration)
            elif 'P-' in activity:
                ## This is a primary activity
                person_stages[pos] = self.Activity(
                    activity=activity, toEdge=to_edge, start=_start, duration=_duration)
            elif 'S-' in activity:
                ## This is a secondary activity
                person_stages[pos] = self.Activity(
                    activity=activity, start=_start, duration=_duration)
            elif activity == 'Home':
                ## End of the activity chain.
                person_stages[pos] = self.Activity(
                    activity=activity, toEdge=from_edge, start=_start, duration=_duration)

        if len(person_stages) <= 2:
            raise sagaexceptions.TripGenerationActivityError(
                "Invalid activity chain. (Minimal: H -> P-? -> H)", activity_chain)

        ## Define secondary activity location
        person_stages = self._stages_define_secondary_locations(person_stages, from_edge, to_edge)

        ## Remove the initial 'Home' stage and update the from of the second stage.
        person_stages[1] = person_stages[1]._replace(fromEdge=person_stages[0].fromEdge)
        if person_stages[0].start:
            person_stages[1] = person_stages[1]._replace(start=person_stages[0].stage)
        del person_stages[0]

        ## Fixing the 'from' field with a forward chain
        pos = 2
        while pos in person_stages:
            person_stages[pos] = person_stages[pos]._replace(fromEdge=person_stages[pos-1].toEdge)
            pos += 1

        ## Compute the real starting time for the activity chain based on ETT and durations
        start = self._stages_compute_start_time(person_stages, mode)
        person_stages[1] = person_stages[1]._replace(start=start)

        ## Define the position of each location in the activity chain.
        person_stages = self._stages_define_locations_position(person_stages)

        ## Final location consistency test
        last_edge = person_stages[1].toEdge
        pos = 2
        while pos in person_stages:
            if person_stages[pos].fromEdge != last_edge:
                raise sagaexceptions.TripGenerationActivityError(
                    'Inconsistency in the locations for the chain of activities.',
                    person_stages)
            last_edge = person_stages[pos].toEdge
            pos += 1

        ## Set the final activity to True
        pos = 1
        while pos in person_stages:
            person_stages[pos] = person_stages[pos]._replace(final=False)
            pos += 1
        person_stages[pos-1] = person_stages[pos-1]._replace(final=True)

        return person_stages

    def _stages_compute_start_time(self, person_stages, mode):
        """ Compute the real starting time for the activity chain. """

        ## Mode split:
        _mode, _ptype, _vtype = sumoutils.get_intermodal_mode_parameters(
            mode, self._conf['intermodalOptions']['vehicleAllowedParking'])

        # Find the first 'start' defined.
        pos = 1
        while pos in person_stages:
            if person_stages[pos].start:
                break
            pos += 1

        start = person_stages[pos].start
        while pos in person_stages:
            ett, route = None, None
            try:
                route = self._sumo.simulation.findIntermodalRoute(
                    person_stages[pos].fromEdge, person_stages[pos].toEdge,
                    modes=_mode, pType=_ptype, vType=_vtype)
                ett = sumoutils.ett_from_route(route)
            except TraCIException:
                raise sagaexceptions.TripGenerationRouteError(
                    'No solution foud for stage {} and modes {}.'.format(
                        pformat(person_stages[pos]), mode))
            if pos-1 in person_stages:
                if person_stages[pos-1].duration:
                    ett += person_stages[pos-1].duration
            start -= ett
            pos -= 1
        return start

    def _get_timing_from_activity(self, activity):
        """ Compute start and duration from the activity defined in the config file. """
        start = None
        if self._conf['activities'][activity]['start']:
            start = self._random_generator.normal(
                loc=self._conf['activities'][activity]['start']['m'],
                scale=self._conf['activities'][activity]['start']['s'])
            if start < 0:
                return self._get_timing_from_activity(activity)
        duration = None
        if self._conf['activities'][activity]['duration']:
            duration = self._random_generator.normal(
                loc=self._conf['activities'][activity]['duration']['m'],
                scale=self._conf['activities'][activity]['duration']['s'])
            if duration <= 0:
                return self._get_timing_from_activity(activity)
        return start, duration

    def _get_cached_dist(self, orig, dest):
        cached = self._cache.get((orig, dest))
        if cached is None:
            cached = self._sumo.simulation.findRoute(orig, dest).length
            self._cache[(orig, dest)] = cached
        return cached
Example #42
0
    def succesiveConditionalSimulatorWithInitialWeightAndData(
            self, initialParamAndData, K, nStates, nBivariateFeat,
            bivariateFeatDictionary, seed, bt, nSeq, interLength, HMCPlusBPS,
            onlyHMC, nLeapFrogSteps, stepSize, nItersPerPathAuxVar, rfOptions,
            mcmcOptions):

        thetaStationaryWeights = initialParamAndData['stationaryWeights']
        thetaBinaryWeights = initialParamAndData['binaryWeights']

        theta0StationarySamples = np.zeros((K, nStates))
        theta0BinaryWeightsSamples = np.zeros((K, nBivariateFeat))

        ## for debuging reasons, we ave the stationary distribution and exchangeable coef
        ## actually, we only need the last sample at the last iteration K
        theta0StationaryDistSamples = np.zeros((K, nStates))
        theta0ExchangeableCoefSamples = np.zeros(
            (K, int(nStates * (nStates - 1) / 2)))

        theta0StationarySamples[0, :] = thetaStationaryWeights
        theta0BinaryWeightsSamples[0, :] = thetaBinaryWeights
        theta0StationaryDistSamples[
            0, :] = np.exp(thetaStationaryWeights) / np.sum(
                np.exp(thetaStationaryWeights))
        initialExchangeCoef = getExchangeCoef(nStates, thetaBinaryWeights,
                                              bivariateFeatDictionary)
        theta0ExchangeableCoefSamples[0, :] = initialExchangeCoef

        sample = None

        if onlyHMC:
            sample = np.zeros((nStates + nBivariateFeat))
            sample[0:nStates] = thetaStationaryWeights
            sample[nStates:(nStates + nBivariateFeat)] = thetaBinaryWeights

        if HMCPlusBPS:
            sample = thetaStationaryWeights

        suffStat = initialParamAndData['suffStat']
        nInit = suffStat['nInit']
        nTrans = suffStat['nTrans']
        holdTime = suffStat['sojourn']

        expectedCompleteReversibleObjective = None
        for i in np.arange(1, K, 1):

            if HMCPlusBPS:
                expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective(
                    holdTime, nInit, nTrans, 1.0, initialExchangeCoef)
            if onlyHMC:
                expectedCompleteReversibleObjective = ExpectedCompleteReversibleObjective(
                    holdTime,
                    nInit,
                    nTrans,
                    1.0,
                    nBivariateFeatWeightsDictionary=bivariateFeatDictionary)

            #####################################
            hmc = HMC(nLeapFrogSteps, stepSize,
                      expectedCompleteReversibleObjective,
                      expectedCompleteReversibleObjective)
            lastSample = sample

            for k in range(nItersPerPathAuxVar):
                hmcResult = hmc.doIter(nLeapFrogSteps, stepSize, lastSample,
                                       expectedCompleteReversibleObjective,
                                       expectedCompleteReversibleObjective,
                                       True)
                lastSample = hmcResult.next_q
            sample = lastSample

            if onlyHMC:
                thetaStationaryWeights = sample[0:nStates]
                thetaBinaryWeights = sample[nStates:(nStates + nBivariateFeat)]
                theta0StationarySamples[i, :] = thetaStationaryWeights
                theta0BinaryWeightsSamples[i, :] = thetaBinaryWeights

            if HMCPlusBPS:
                thetaStationaryWeights = sample
                theta0StationarySamples[i, :] = thetaStationaryWeights
                # update stationary distribution elements to the latest value
                thetaStationaryDist = np.exp(sample) / np.sum(np.exp(sample))
                # sample exchangeable coefficients using local bouncy particle sampler
                ## define the model
                model = ExpectedCompleteReversibleModelWithBinaryFactors(
                    expectedCompleteReversibleObjective, nStates,
                    thetaBinaryWeights, thetaStationaryDist,
                    bivariateFeatDictionary)
                ## define the sampler to use
                ## local sampler to use

                if i == 1:
                    self.neighborVariablesForAllFactors = neighborVariableForAllFactors(
                        nStates, model.localFactors,
                        model.bivariateFeatIndexDictionary)
                    self.variableAndFactorInfo = neighbourVariblesAndFactorsAndExtendedNeighborsOfAllFactorsDict(
                        nStates, model.localFactors,
                        model.bivariateFeatIndexDictionary, nBivariateFeat)
                    self.indexOfFactorsForEachBivariateFeat = getIndexOfNeighborFactorsForEachIndexOfBinaryFeature(
                        model.bivariateFeatIndexDictionary, nBivariateFeat,
                        model.localFactors)

                localSampler = LocalRFSamplerForBinaryWeights(
                    model, rfOptions, mcmcOptions, nStates,
                    self.neighborVariablesForAllFactors,
                    self.variableAndFactorInfo,
                    self.indexOfFactorsForEachBivariateFeat)

                phyloLocalRFMove = PhyloLocalRFMove(
                    model=model,
                    sampler=localSampler,
                    initialPoints=thetaBinaryWeights,
                    options=rfOptions,
                    prng=RandomState(i))
                thetaBinaryWeights = phyloLocalRFMove.execute()
                theta0BinaryWeightsSamples[i, :] = thetaBinaryWeights

            initialRateMtx = ReversibleRateMtxPiAndBinaryWeightsWithGraphicalStructure(
                nStates,
                thetaStationaryWeights,
                thetaBinaryWeights,
                bivariateFeatIndexDictionary=bivariateFeatDictionary)

            initialStationaryDist = initialRateMtx.getStationaryDist()
            initialExchangeCoef = initialRateMtx.getExchangeCoef()
            theta0StationaryDistSamples[i, :] = initialStationaryDist
            theta0ExchangeableCoefSamples[i, :] = initialExchangeCoef

            weightGenerationRegime = WeightGenerationRegime(
                nStates=nStates,
                nBivariateFeat=nBivariateFeat,
                stationaryWeights=thetaStationaryWeights,
                bivariateWeights=thetaBinaryWeights)

            prng = RandomState(np.random.choice(2**32 - 1, 1))

            dataRegime = DataGenerationRegime(
                nStates=nStates,
                bivariateFeatIndexDictionary=bivariateFeatDictionary,
                btLength=bt,
                nSeq=nSeq,
                weightGenerationRegime=weightGenerationRegime,
                prng=prng,
                interLength=interLength)
            ## generate the sequences data
            seqList = dataRegime.generatingSeq()
            suffStat = dataRegime.getSufficientStatFromSeq(seqList)
            firstLastStatesArrayAll = dataRegime.generatingSeqGivenRateMtxAndBtInterval(
                seqList)

            # replicate K iterations to get new parameters
            nTrans = suffStat["transitCount"]
            holdTime = suffStat["sojourn"]

            nInit = np.zeros(nStates)
            unique, counts = np.unique(firstLastStatesArrayAll[0][:, 0],
                                       return_counts=True)
            nInitCount = np.asarray((unique, counts)).T
            nInit[nInitCount[:, 0].astype(int)] = nInitCount[:, 1]

        result = {}
        result['stationaryDist'] = theta0StationaryDistSamples[(K - 1), :]
        result['exchangeCoef'] = theta0ExchangeableCoefSamples[(K - 1), :]
        result['stationaryWeights'] = thetaStationaryWeights
        result['binaryWeights'] = thetaBinaryWeights
        ## after testing the code is right, the following two lines should be removed
        #result['StationaryDistSamples'] = theta0StationaryDistSamples
        #result['ExchangeableCoefSamples'] = theta0ExchangeableCoefSamples
        return result
Example #43
0
for directory in [targetDirSpec, targetDirPitch]:
    if os.path.exists(directory):
        shutil.rmtree(directory)
    os.makedirs(directory)

#targetDirSpec = './spmel'
dirName, subdirList, _ = next(os.walk(rootDir))
print('Found directory: %s' % dirName)

for subdir_idx, subdir in enumerate(sorted(subdirList)):
    print(subdir)
    os.makedirs(os.path.join(targetDirSpec,subdir))
    os.makedirs(os.path.join(targetDirPitch,subdir))
    _,_, fileList = next(os.walk(os.path.join(dirName,subdir)))
    #prng = RandomState(int(subdir[1:])) 
    prng = RandomState(1) 

    for file_idx, fileName in enumerate(sorted(fileList)):
        # ensure that only mic1 files are processed
        if fileName.endswith('wav'):
            print(f'{subdir}, {subdir_idx}/{len(subdirList)}, {fileName}, {file_idx}/{len(fileList)}')
            # Read audio file
            audio, sr = sf.read(os.path.join(dirName,subdir,fileName))
            # Remove drifting noise
            y = signal.filtfilt(b, a, audio)
            # Ddd a little random noise for model roubstness
            wav = y * 0.96 + (prng.rand(y.shape[0])-0.5)*1e-06
            # resample 48kHz to 16kHz
            resampled_wav = librosa.resample(wav, sr, 16000)
            # pdb.set_trace()
            # compute pitch contour
Example #44
0
def exponential_lik(X: ndarray, rng: RandomState):
    return rng.exponential(X)
Example #45
0
        # Pytorch expects float32 for input and int64 for labels.
        # X: trials x channels x timesteps
        X = FeatVect.astype(np.float32)
        X = X[:, :, :]
        y = (y_labels[:, 0] - 0).astype(np.int64)

        for CV in np.arange(0, num_folds):
            print('Subject No.{} CV {}'.format(i, CV))
            # 5th phase: Model evaluation (test)
            train_set = SignalAndTarget(X, y=y)
            # 80% training, 20% test
            train_set, test_set = split_into_train_test(
                train_set,
                n_folds=num_folds,
                i_test_fold=CV,
                rng=RandomState((2019, 28, 6)))  #RandomState((2019, 28, 6))
            # 5% training, 95% test
            #test_set, train_set = split_into_train_test(train_set, n_folds = num_folds, i_test_fold = CV, rng=None)

            cuda_avail = th.cuda.is_available()
            set_random_seeds(seed=20190628, cuda=cuda_avail)

            n_classes = 2
            in_chans = train_set.X.shape[1]  # number of channels = 128
            input_time_length = 150  # length of time of each epoch/trial = 4000

            model = ShallowFBCSPNet(
                split_first_layer=False,
                in_chans=in_chans,
                n_classes=n_classes,
                input_time_length=input_time_length,
def plot_faces_decomposition():
    # Display progress logs on stdout
    logging.basicConfig(level=logging.INFO,
                        format='%(asctime)s %(levelname)s %(message)s')
    n_row, n_col = 2, 3
    n_components = n_row * n_col
    image_shape = (64, 64)
    rng = RandomState(0)

    # #############################################################################
    # Load faces data
    faces, _ = fetch_olivetti_faces(return_X_y=True, shuffle=True,
                                    random_state=rng)
    n_samples, n_features = faces.shape

    # global centering
    faces_centered = faces - faces.mean(axis=0)

    # local centering
    faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)

    print("Dataset consists of %d faces" % n_samples)

    def plot_gallery(title, images, n_col=n_col, n_row=n_row, cmap=plt.cm.gray):
        plt.figure(figsize=(2. * n_col, 2.26 * n_row))
        plt.suptitle(title, size=16)
        for i, comp in enumerate(images):
            plt.subplot(n_row, n_col, i + 1)
            vmax = max(comp.max(), -comp.min())
            plt.imshow(comp.reshape(image_shape), cmap=cmap,
                       interpolation='nearest',
                       vmin=-vmax, vmax=vmax)
            plt.xticks(())
            plt.yticks(())
        plt.subplots_adjust(0.01, 0.05, 0.99, 0.93, 0.04, 0.)

    # #############################################################################
    # List of the different estimators, whether to center and transpose the
    # problem, and whether the transformer uses the clustering API.
    estimators = [
        ('Eigenfaces - PCA using randomized SVD',
         decomposition.PCA(n_components=n_components, svd_solver='randomized',
                           whiten=True),
         True),

        ('Non-negative components - NMF',
         decomposition.NMF(n_components=n_components, init='nndsvda', tol=5e-3),
         False),

        ('Independent components - FastICA',
         decomposition.FastICA(n_components=n_components, whiten=True),
         True),

        ('Sparse comp. - MiniBatchSparsePCA',
         decomposition.MiniBatchSparsePCA(n_components=n_components, alpha=0.8,
                                          n_iter=100, batch_size=3,
                                          random_state=rng),
         True),

        ('MiniBatchDictionaryLearning',
         decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                   n_iter=50, batch_size=3,
                                                   random_state=rng),
         True),

        ('Cluster centers - MiniBatchKMeans',
         MiniBatchKMeans(n_clusters=n_components, tol=1e-3, batch_size=20,
                         max_iter=50, random_state=rng),
         True),

        ('Factor Analysis components - FA',
         decomposition.FactorAnalysis(n_components=n_components, max_iter=20),
         True),
    ]

    # #############################################################################
    # Plot a sample of the input data

    plot_gallery("First centered Olivetti faces", faces_centered[:n_components])

    # #############################################################################
    # Do the estimation and plot it

    for name, estimator, center in estimators:
        print("Extracting the top %d %s..." % (n_components, name))
        t0 = time()
        data = faces
        if center:
            data = faces_centered
        estimator.fit(data)
        train_time = (time() - t0)
        print("done in %0.3fs" % train_time)
        if hasattr(estimator, 'cluster_centers_'):
            components_ = estimator.cluster_centers_
        else:
            components_ = estimator.components_

        # Plot an image representing the pixelwise variance provided by the
        # estimator e.g its noise_variance_ attribute. The Eigenfaces estimator,
        # via the PCA decomposition, also provides a scalar noise_variance_
        # (the mean of pixelwise variance) that cannot be displayed as an image
        # so we skip it.
        if (hasattr(estimator, 'noise_variance_') and
                estimator.noise_variance_.ndim > 0):  # Skip the Eigenfaces case
            plot_gallery("Pixelwise variance",
                         estimator.noise_variance_.reshape(1, -1), n_col=1,
                         n_row=1)
        plot_gallery('%s - Train time %.1fs' % (name, train_time),
                     components_[:n_components])

    plt.show()

    # #############################################################################
    # Various positivity constraints applied to dictionary learning.
    estimators = [
        ('Dictionary learning',
         decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                   n_iter=50, batch_size=3,
                                                   random_state=rng),
         True),
        ('Dictionary learning - positive dictionary',
         decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                   n_iter=50, batch_size=3,
                                                   random_state=rng,
                                                   positive_dict=True),
         True),
        ('Dictionary learning - positive code',
         decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                   n_iter=50, batch_size=3,
                                                   fit_algorithm='cd',
                                                   random_state=rng,
                                                   positive_code=True),
         True),
        ('Dictionary learning - positive dictionary & code',
         decomposition.MiniBatchDictionaryLearning(n_components=15, alpha=0.1,
                                                   n_iter=50, batch_size=3,
                                                   fit_algorithm='cd',
                                                   random_state=rng,
                                                   positive_dict=True,
                                                   positive_code=True),
         True),
    ]

    # #############################################################################
    # Plot a sample of the input data

    plot_gallery("First centered Olivetti faces", faces_centered[:n_components],
                 cmap=plt.cm.RdBu)

    # #############################################################################
    # Do the estimation and plot it

    for name, estimator, center in estimators:
        print("Extracting the top %d %s..." % (n_components, name))
        t0 = time()
        data = faces
        if center:
            data = faces_centered
        estimator.fit(data)
        train_time = (time() - t0)
        print("done in %0.3fs" % train_time)
        components_ = estimator.components_
        plot_gallery(name, components_[:n_components], cmap=plt.cm.RdBu)

    plt.show()
Example #47
0
obj = csv.reader(file.read().decode('utf-8').splitlines())

title = False
raw_data = []

for v in obj:
    if title == False:
        title = True
        continue
    if len(v) != 0:
        line_data = [float(v[8]), float(v[9]), float(v[10]), float(v[11])]
        raw_data.append(line_data)

X = PCA(n_components=2).fit_transform(np.array(raw_data))

random_state = RandomState(42)
n_samples = len(raw_data)

kmeans = KMeans(n_clusters=3, random_state=random_state).fit(X)


def plot_2D(data, target, target_names):
    colors = cycle('rgbcmykw')
    target_ids = range(len(target_names))
    for i, c, label in zip(target_ids, colors, target_names):
        plt.scatter(data[target == i, 0],
                    data[target == i, 1],
                    c=c,
                    label=label)
    pl.legend()
    pl.show()
Example #48
0
#在shape的一个维度上使用None,可以方便使用大不的batch大小。在训练是需要把数据分成
#比较小的batch,但是在测试时,可以一次性使用全部的数据。当数据集你叫小的时候这样比较
#方便测试,但数据集比较大时,将大量数据放入一个batch可能会导致内存溢出。
x = tf.placeholder(tf.float32, shape=(None, 2), name='x-input')
y_ = tf.placeholder(tf.float32, shape=(None, 1), name='y-input')

#定义神经网络前向传播的过程
a = tf.matmul(x, w1)
y = tf.matmul(a, w2)

#定义损失函数和反向传播的算法
cross_entropy = -tf.reduce_mean(y_ * tf.log(tf.clip_by_value(y, 1e-10, 1.0)))
train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)

#通过随机数生成一个模拟数据集
rdm = RandomState(1)
dataset_size = 128
X = rdm.rand(dataset_size, 2)

#定义规则来给出样本的标签,在这里所有x1+x2<1的样例都被认为是正样本(比如零件合格),
#而其他为负样本(比如样本不合格)。和Tensorflow游乐场中的表示不大一样的地方是,
#在这里使用0来表示负样本,1表示正样本。大部分解决分类问题的神经网络都会采用
#0和1的表示方法
Y = [[int(x1 + x2 < 1)] for (x1, x2) in X]
#创建一个会话来运行tendorflow
with tf.Session() as sess:
    init_op = tf.initialize_all_variables()
    sess.run(init_op)
    print("训练之前的")
    print(sess.run(w1))
    print("-----------")
Example #49
0
class BPMF(ModelBase):
    """Bayesian Probabilistic Matrix Factorization
    """
    def __init__(
            self,
            n_user,
            n_item,
            n_feature,
            beta=2.0,
            beta_user=2.0,
            df_user=None,
            mu0_user=0.,
            beta_item=2.0,
            df_item=None,
            mu0_item=0.,
            converge=1e-5,
            seed=None,
            max_rating=None,  # seed is useful for reproducibility.
            min_rating=None):

        super(BPMF, self).__init__()

        self.n_user = n_user
        self.n_item = n_item
        self.n_feature = n_feature
        self.rand_state = RandomState(seed)
        self.max_rating = float(max_rating) if max_rating is not None else None
        self.min_rating = float(min_rating) if min_rating is not None else None
        self.converge = converge

        # Hyper Parameter
        self.beta = beta

        # Inv-Whishart (User features)
        self.WI_user = np.eye(n_feature, dtype='float64')
        self.beta_user = beta_user
        self.df_user = int(df_user) if df_user is not None else n_feature
        self.mu0_user = np.repeat(mu0_user, n_feature).reshape(n_feature,
                                                               1)  # a vector

        # Inv-Whishart (item features)
        self.WI_item = np.eye(n_feature, dtype='float64')
        self.beta_item = beta_item
        self.df_item = int(df_item) if df_item is not None else n_feature
        self.mu0_item = np.repeat(mu0_item, n_feature).reshape(n_feature, 1)

        # Latent Variables
        self.mu_user = np.zeros((n_feature, 1), dtype='float64')
        self.mu_item = np.zeros((n_feature, 1), dtype='float64')

        self.alpha_user = np.eye(
            n_feature,
            dtype='float64')  # These are probably the Lambda matrices
        self.alpha_item = np.eye(n_feature, dtype='float64')

        self.user_features_ = 0.3 * self.rand_state.rand(
            n_user,
            n_feature)  # initializes the user features randomly. Why 0.3??
        self.item_features_ = 0.3 * self.rand_state.rand(n_item, n_feature)

        # data state
        self.mean_rating_ = None
        self.ratings_csr_ = None
        self.ratings_csc_ = None

    def fit(self, ratings, test_ratings=None, n_iters=50):
        """training models; I modified it such that if you also pass test_ratings, it computes the test RMSE at each step. """

        check_ratings(
            ratings,
            self.n_user,
            self.n_item,  # checks if ratings matrix is OK
            self.max_rating,
            self.min_rating)

        self.mean_rating_ = np.mean(ratings[:, 2])

        # only two different ways of building the matrix.
        # csr user-item matrix for fast row access (user update)
        self.ratings_csr_ = build_user_item_matrix(self.n_user, self.n_item,
                                                   ratings)
        # keep a csc matrix for fast col access (item update)
        self.ratings_csc_ = self.ratings_csr_.tocsc()

        # lists in where I will save the rmse during training
        train_rmse_list = []
        if test_ratings is not None:
            test_rmse_list = []

        # define the lists that will contain the total predictions for training and test set
        train_preds_total = np.zeros(len(ratings)).tolist()
        if test_ratings is not None:
            test_preds_total = np.zeros(len(test_ratings)).tolist()

        last_rmse = None
        for iteration in xrange(n_iters):
            logger.debug("iteration %d...", iteration)

            # THE FOLLOWING ARE THE GIBBS SAMPLING STEP

            # update item & user parameter
            self._update_item_params()
            self._update_user_params()

            # update item & user features
            self._udpate_item_features()
            self._update_user_features()

            # compute RMSE
            train_preds = self.predict(ratings[:, :2])

            # in order to make a meaningful MCMC, you need to do this, to sample from the correct distribution; to do this, you have
            # to average over all previous predictions:

            train_preds_total = ((np.array(train_preds_total) * iteration +
                                  np.array(train_preds)) /
                                 (iteration + 1)).tolist()
            train_rmse = RMSE(train_preds_total, ratings[:, 2])

            train_rmse_list.append(train_rmse)
            if test_ratings is not None:
                test_preds = self.predict(test_ratings[:, :2])
                test_preds_total = ((np.array(test_preds_total) * iteration +
                                     np.array(test_preds)) /
                                    (iteration + 1)).tolist()
                test_rmse = RMSE(test_preds_total, test_ratings[:, 2])
                test_rmse_list.append(test_rmse)
                logger.info("iter: %d, train RMSE: %.6f, test RMSE: %.6f",
                            iteration, train_rmse, test_rmse)
            else:
                logger.info("iter: %d, train RMSE: %.6f", iteration,
                            train_rmse)

            # stop when converge
            if last_rmse and abs(
                    train_rmse - last_rmse
            ) < self.converge:  # if you get convergence -> stop
                logger.info('converges at iteration %d. stop.', iteration)
                break
            else:
                last_rmse = train_rmse
        if test_ratings is not None:
            return train_rmse_list, test_rmse_list
        else:
            return train_rmse_list

    def fit_old(self, ratings, test_ratings=None, n_iters=50):
        """training models; I modified it such that if you also pass test_ratings, it computes the test RMSE at each step. 
        This is wrong since it does not implemenent correctly the MCMC algorithm; infact, in order to use MCMC correctly, 
        you need to average the predictions of every step to get a final prediction and estimate the RMSE, while in this function
        the RMSE was computed only on the prediction given by the current value of the feature matrices. Essentially, the predicted
        ratings are function on the whole chain of values of U and V. The purpose of MCMC chain is infact to estimate an integral by 
        sampling from the correct distribution for U, V, and this is done by averaging over subsequently extracted samples. """

        check_ratings(
            ratings,
            self.n_user,
            self.n_item,  # checks if ratings matrix is OK
            self.max_rating,
            self.min_rating)

        self.mean_rating_ = np.mean(ratings[:, 2])

        # only two different ways of building the matrix.
        # csr user-item matrix for fast row access (user update)
        self.ratings_csr_ = build_user_item_matrix(self.n_user, self.n_item,
                                                   ratings)
        # keep a csc matrix for fast col access (item update)
        self.ratings_csc_ = self.ratings_csr_.tocsc()

        train_rmse_list = []
        if test_ratings is not None:
            test_rmse_list = []

        last_rmse = None
        for iteration in xrange(n_iters):
            logger.debug("iteration %d...", iteration)

            # THE FOLLOWING ARE THE GIBBS SAMPLING STEP

            # update item & user parameter
            self._update_item_params()
            self._update_user_params()

            # update item & user features
            self._udpate_item_features()
            self._update_user_features()

            # compute RMSE; NB: at each step, this code just predicts the R matrix using U, V but does not make the average over more
            # prediction steps, as it should. This is not how MCMC works!!!
            train_preds = self.predict(ratings[:, :2])
            train_rmse = RMSE(train_preds, ratings[:, 2])
            train_rmse_list.append(train_rmse)
            if test_ratings is not None:
                test_preds = self.predict(test_ratings[:, :2])
                test_rmse = RMSE(test_preds, test_ratings[:, 2])
                test_rmse_list.append(test_rmse)
                logger.info("iter: %d, train RMSE: %.6f, test RMSE: %.6f",
                            iteration, train_rmse, test_rmse)
            else:
                logger.info("iter: %d, train RMSE: %.6f", iteration,
                            train_rmse)

            # stop when converge
            if last_rmse and abs(
                    train_rmse - last_rmse
            ) < self.converge:  # if you get convergence -> stop
                logger.info('converges at iteration %d. stop.', iteration)
                break
            else:
                last_rmse = train_rmse
        if test_ratings is not None:
            return train_rmse_list, test_rmse_list
        else:
            return train_rmse_list

    def predict(self, data):

        if not self.mean_rating_:
            raise NotFittedError()

        u_features = self.user_features_.take(data.take(0, axis=1), axis=0)
        i_features = self.item_features_.take(data.take(1, axis=1), axis=0)
        preds = np.sum(u_features * i_features, 1) + self.mean_rating_

        if self.max_rating:  # cut the prediction rate.
            preds[preds > self.max_rating] = self.max_rating

        if self.min_rating:
            preds[preds < self.min_rating] = self.min_rating

        return preds

    def _update_item_params(self):
        N = self.n_item
        X_bar = np.mean(self.item_features_, 0).reshape((self.n_feature, 1))
        # print 'X_bar', X_bar.shape
        S_bar = np.cov(self.item_features_.T)
        # print 'S_bar', S_bar.shape

        diff_X_bar = self.mu0_item - X_bar

        # W_{0}_star
        WI_post = inv(
            inv(self.WI_item) + N * S_bar + np.dot(diff_X_bar, diff_X_bar.T) *
            (N * self.beta_item) / (self.beta_item + N))

        # Note: WI_post and WI_post.T should be the same.
        #       Just make sure it is symmertic here
        WI_post = (WI_post + WI_post.T) / 2.0

        # update alpha_item
        df_post = self.df_item + N
        self.alpha_item = wishart.rvs(df_post, WI_post, 1, self.rand_state)

        # update mu_item
        mu_mean = (self.beta_item * self.mu0_item + N * X_bar) / \
            (self.beta_item + N)
        mu_var = cholesky(inv(np.dot(self.beta_item + N, self.alpha_item)))
        # print 'lam', lam.shape
        self.mu_item = mu_mean + np.dot(
            mu_var, self.rand_state.randn(self.n_feature, 1))
        # print 'mu_item', self.mu_item.shape

    def _update_user_params(self):
        # same as _update_user_params
        N = self.n_user
        X_bar = np.mean(self.user_features_, 0).reshape((self.n_feature, 1))
        S_bar = np.cov(self.user_features_.T)

        # mu_{0} - U_bar
        diff_X_bar = self.mu0_user - X_bar

        # W_{0}_star
        WI_post = inv(
            inv(self.WI_user) + N * S_bar + np.dot(diff_X_bar, diff_X_bar.T) *
            (N * self.beta_user) / (self.beta_user + N))
        # Note: WI_post and WI_post.T should be the same.
        #       Just make sure it is symmertic here
        WI_post = (WI_post + WI_post.T) / 2.0

        # update alpha_user
        df_post = self.df_user + N
        # LAMBDA_{U} ~ W(W{0}_star, df_post)
        self.alpha_user = wishart.rvs(df_post, WI_post, 1, self.rand_state)

        # update mu_user
        # mu_{0}_star = (beta_{0} * mu_{0} + N * U_bar) / (beta_{0} + N)
        mu_mean = (self.beta_user * self.mu0_user + N * X_bar) / \
                  (self.beta_user + N)

        # decomposed inv(beta_{0}_star * LAMBDA_{U})
        mu_var = cholesky(inv(np.dot(self.beta_user + N, self.alpha_user)))
        # sample multivariate gaussian
        self.mu_user = mu_mean + np.dot(
            mu_var, self.rand_state.randn(self.n_feature, 1))

    def _udpate_item_features(self):
        # Gibbs sampling for item features
        for item_id in xrange(self.n_item):
            indices = self.ratings_csc_[:, item_id].indices
            features = self.user_features_[indices, :]
            rating = self.ratings_csc_[:,
                                       item_id].data - self.mean_rating_  # IT SUBTRACTS mean_rating_ AND ADD IT AGAIN LATER.
            rating = np.reshape(rating, (rating.shape[0], 1))

            covar = inv(self.alpha_item +
                        self.beta * np.dot(features.T, features))
            lam = cholesky(covar)

            temp = (self.beta * np.dot(features.T, rating) +
                    np.dot(self.alpha_item, self.mu_item))

            mean = np.dot(covar, temp)
            temp_feature = mean + np.dot(
                lam, self.rand_state.randn(self.n_feature, 1))
            self.item_features_[item_id, :] = temp_feature.ravel()

    def _update_user_features(self):
        # Gibbs sampling for user features
        for user_id in xrange(self.n_user):
            indices = self.ratings_csr_[user_id, :].indices
            features = self.item_features_[indices, :]
            rating = self.ratings_csr_[user_id, :].data - self.mean_rating_
            rating = np.reshape(rating, (rating.shape[0], 1))

            covar = inv(self.alpha_user +
                        self.beta * np.dot(features.T, features))
            lam = cholesky(covar)
            # aplha * sum(V_j * R_ij) + LAMBDA_U * mu_u
            temp = (self.beta * np.dot(features.T, rating) +
                    np.dot(self.alpha_user, self.mu_user))
            # mu_i_star
            mean = np.dot(covar, temp)
            temp_feature = mean + np.dot(
                lam, self.rand_state.randn(self.n_feature, 1))
            self.user_features_[user_id, :] = temp_feature.ravel()
Example #50
0
    def __init__(
            self,
            n_user,
            n_item,
            n_feature,
            beta=2.0,
            beta_user=2.0,
            df_user=None,
            mu0_user=0.,
            beta_item=2.0,
            df_item=None,
            mu0_item=0.,
            converge=1e-5,
            seed=None,
            max_rating=None,  # seed is useful for reproducibility.
            min_rating=None):

        super(BPMF, self).__init__()

        self.n_user = n_user
        self.n_item = n_item
        self.n_feature = n_feature
        self.rand_state = RandomState(seed)
        self.max_rating = float(max_rating) if max_rating is not None else None
        self.min_rating = float(min_rating) if min_rating is not None else None
        self.converge = converge

        # Hyper Parameter
        self.beta = beta

        # Inv-Whishart (User features)
        self.WI_user = np.eye(n_feature, dtype='float64')
        self.beta_user = beta_user
        self.df_user = int(df_user) if df_user is not None else n_feature
        self.mu0_user = np.repeat(mu0_user, n_feature).reshape(n_feature,
                                                               1)  # a vector

        # Inv-Whishart (item features)
        self.WI_item = np.eye(n_feature, dtype='float64')
        self.beta_item = beta_item
        self.df_item = int(df_item) if df_item is not None else n_feature
        self.mu0_item = np.repeat(mu0_item, n_feature).reshape(n_feature, 1)

        # Latent Variables
        self.mu_user = np.zeros((n_feature, 1), dtype='float64')
        self.mu_item = np.zeros((n_feature, 1), dtype='float64')

        self.alpha_user = np.eye(
            n_feature,
            dtype='float64')  # These are probably the Lambda matrices
        self.alpha_item = np.eye(n_feature, dtype='float64')

        self.user_features_ = 0.3 * self.rand_state.rand(
            n_user,
            n_feature)  # initializes the user features randomly. Why 0.3??
        self.item_features_ = 0.3 * self.rand_state.rand(n_item, n_feature)

        # data state
        self.mean_rating_ = None
        self.ratings_csr_ = None
        self.ratings_csc_ = None
Example #51
0
days_popularity = np.zeros([no_days, 10])
for i in range(0, no_families):
    choices = family_data[i, 1:-1]
    no_people = family_data[i, -1]
    for itr in range(0, len(choices)):
        days_popularity[choices[itr] - 1, itr] += no_people

days_popularity_tot = sum(days_popularity.T)

cutoff_orig = 10
max_no_optimization_itrs = 1000
choices_inds = np.zeros([no_families, 1]).astype(int)
BackwardMatrix = np.zeros([no_families, no_days]).astype(int)
max_cost = 100000000
prng = RandomState(int(time.time()))
for itr in range(0, max_no_optimization_itrs):
    ForwardMatrix = np.zeros([no_families, no_days]).astype(int)
    previous_choices_inds = choices_inds
    choices_inds = np.zeros([no_families, 1]).astype(int)

    cutoff = cutoff_orig / (1. + itr)
    # Forward step: people send their schedule to the constraints
    family_inds = np.random.permutation(no_families)
    day_count = np.zeros([no_days, 1]).astype(int)
    zero_vec = np.zeros([no_days, 1])
    for i in family_inds:
        choices = family_data[i, 1:-1]
        no_people = family_data[i, -1]

        # Check the messages comming from neighbors
Example #52
0
    def validate(self) -> None:
        """
        Check arguments correctness and consistency.

        * input files must exist
        * output files must be in a writeable directory
        * if no seed specified, set random seed.
        * length of per-chain lists equals specified # of chains
        """
        if self.model_name is None:
            raise ValueError('no stan model specified')
        if self.model_exe is None:
            raise ValueError('model not compiled')

        if self.chain_ids is not None:
            for i in range(len(self.chain_ids)):
                if self.chain_ids[i] < 1:
                    raise ValueError(
                        'invalid chain_id {}'.format(self.chain_ids[i])
                    )

        if self.output_dir is not None:
            self.output_dir = os.path.realpath(os.path.expanduser(
                self.output_dir))
            if not os.path.exists(os.path.dirname(self.output_dir)):
                raise ValueError(
                    'invalid path for output files, no such dir: {}'.format(
                        self.output_dir
                    )
                )
            try:
                testpath = os.path.join(self.output_dir, str(time()))
                with open(testpath, 'w+') as fd:
                    pass
                os.remove(testpath)  # cleanup
            except Exception:
                raise ValueError(
                    'invalid path for output files, '
                    'cannot write to dir: {}'.format(self.output_dir)
                )

        if self.seed is None:
            rng = RandomState()
            self.seed = rng.randint(1, 99999 + 1)
        else:
            if not isinstance(self.seed, (int, list)):
                raise ValueError(
                    'seed must be an integer between 0 and 2**32-1,'
                    ' found {}'.format(self.seed)
                )
            if isinstance(self.seed, int):
                if self.seed < 0 or self.seed > 2 ** 32 - 1:
                    raise ValueError(
                        'seed must be an integer between 0 and 2**32-1,'
                        ' found {}'.format(self.seed)
                    )
            else:
                if self.chain_ids is None:
                    raise ValueError(
                        'seed must not be a list when no chains used'
                    )

                if len(self.seed) != len(self.chain_ids):
                    raise ValueError(
                        'number of seeds must match number of chains '
                        ' found {} seed for {} chains '.format(
                            len(self.seed), len(self.chain_ids)
                        )
                    )
                for i in range(len(self.seed)):
                    if self.seed[i] < 0 or self.seed[i] > 2 ** 32 - 1:
                        raise ValueError(
                            'seed must be an integer value'
                            ' between 0 and 2**32-1,'
                            ' found {}'.format(self.seed[i])
                        )

        if isinstance(self.data, str):
            if not os.path.exists(self.data):
                raise ValueError('no such file {}'.format(self.data))
        elif self.data is None:
            if isinstance(self.method_args, OptimizeArgs):
                raise ValueError('data must be set when optimizing')
        elif not isinstance(self.data, (str, dict)):
            raise ValueError('data must be string or dict')

        if self.inits is not None:
            if isinstance(self.inits, (Integral, Real)):
                if self.inits < 0:
                    raise ValueError(
                        'inits must be > 0, found {}'.format(self.inits)
                    )
            elif isinstance(self.inits, str):
                if not os.path.exists(self.inits):
                    raise ValueError('no such file {}'.format(self.inits))
            elif isinstance(self.inits, list):
                if self.chain_ids is None:
                    raise ValueError(
                        'inits must not be a list when no chains are used'
                    )

                if len(self.inits) != len(self.chain_ids):
                    raise ValueError(
                        'number of inits files must match number of chains '
                        ' found {} inits files for {} chains '.format(
                            len(self.inits), len(self.chain_ids)
                        )
                    )
                names_set = set(self.inits)
                if len(names_set) != len(self.inits):
                    raise ValueError(
                        'each chain must have its own init file,'
                        ' found duplicates in inits files list.'
                    )
                for i in range(len(self.inits)):
                    if not os.path.exists(self.inits[i]):
                        raise ValueError(
                            'no such file {}'.format(self.inits[i])
                        )
Example #53
0
 def plot2(self, X):
     X_01 = X[:, [0, 1]]
     kmeans = KMeans(n_clusters=3, random_state=RandomState(42)).fit(X_01)
     plot_2D(X_01, kmeans.labels_, ["c0", "c1", "c2"],
             "Scatter Plot: Features-0-1")
     calculate_IV_EV(X_01, kmeans.labels_)
Example #54
0
y_ = tf.placeholder(tf.float32, shape=(None, 1), name='y-input')

# 定义一个单层的前向传播的神经网络
w1 = tf.Variable(tf.random_normal([2, 1], stddev=1, seed=1))
y = tf.matmul(x, w1)

# 定义预测多了和预测少了的成本
loss_less = 10
loss_more = 1
loss = tf.reduce_sum(
    tf.where(tf.greater(y, y_), (y - y_) * loss_more, (y_ - y) * loss_less))

train_step = tf.train.AdadeltaOptimizer(0.001).minimize(loss)

# 通过随机数生成一个模拟数据集
rdm = RandomState(1)
dataset_size = 128
X = rdm.rand(dataset_size, 2)

Y = [[x1 + x2 + rdm.rand() / 10.0 - 0.05] for (x1, x2) in X]

# 训练神经网络
with tf.Session() as sess:
    tf.global_variables_initializer().run()
    STEPS = 5000
    for i in range(STEPS):
        start = (i * batch_size) % dataset_size
        end = min(start + batch_size, dataset_size)
        sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})
        if i % 1000 == 0:
            print("After %d training step(s), w1 is: " % (i))
Example #55
0
 def plot3(self, X):
     X_23 = X[:, [2, 3]]
     kmeans = KMeans(n_clusters=3, random_state=RandomState(42)).fit(X_23)
     plot_2D(X_23, kmeans.labels_, ["c0", "c1", "c2"],
             "Scatter Plot: Features-2-3")
     calculate_IV_EV(X_23, kmeans.labels_)
Example #56
0
    def __init__(self,
                 input,
                 n_in,
                 n_out,
                 rng=RandomState(4444),
                 W=None,
                 b=None,
                 activation=T.tanh):
        """
        Typical hidden layer of a MLP: units are fully-connected and have
        sigmoidal activation function. Weight matrix W is of shape (n_in,n_out)
        and the bias vector b is of shape (n_out,).

        NOTE : The nonlinearity used here is tanh

        Hidden unit activation is given by: tanh(dot(input,W) + b)

        :type rng: numpy.random.RandomState
        :param rng: a random number generator used to initialize weights

        :type input: theano.tensor.dmatrix
        :param input: a symbolic tensor of shape (n_examples, n_in)

        :type n_in: int
        :param n_in: dimensionality of input

        :type n_out: int
        :param n_out: number of hidden units

        :type activation: theano.Op or function
        :param activation: Non linearity to be applied in the hidden
                           layer
        """
        self.input = input
        # end-snippet-1

        # `W` is initialized with `W_values` which is uniformely sampled
        # from sqrt(-6./(n_in+n_hidden)) and sqrt(6./(n_in+n_hidden))
        # for tanh activation function
        # the output of uniform if converted using asarray to dtype
        # theano.config.floatX so that the code is runable on GPU
        # Note : optimal initialization of weights is dependent on the
        #        activation function used (among other things).
        #        For example, results presented in [Xavier10] suggest that you
        #        should use 4 times larger initial weights for sigmoid
        #        compared to tanh
        #        We have no info for other function, so we use the same as
        #        tanh.
        if W is None:
            W_values = np.asarray(rng.uniform(
                low=-np.sqrt(6. / (n_in + n_out)),
                high=np.sqrt(6. / (n_in + n_out)),
                size=(n_in, n_out)),
                                  dtype=theano.config.floatX)
            if activation == theano.tensor.nnet.sigmoid:
                W_values *= 4

            W = theano.shared(value=W_values, name='W', borrow=True)

        if b is None:
            b_values = np.zeros((n_out, ), dtype=theano.config.floatX)
            b = theano.shared(value=b_values, name='b', borrow=True)

        self.W = W
        self.b = b

        lin_output = T.dot(input, self.W) + self.b
        self.output = (lin_output
                       if activation is None else activation(lin_output))
        # parameters of the model
        self.params = [self.W, self.b]
Example #57
0
def create_app(strD):
    MODEL_SAVE_PATH = "model/"
    MODEL_NAME = "model.ckpt"
    INPUT_NODE_NUM = 7

    x = tf.placeholder(tf.float32,
                       shape=(None, INPUT_NODE_NUM),
                       name='x-input')
    y_ = tf.placeholder(tf.float32, shape=(None, 5), name='y-input')

    rdm = RandomState(1)
    dataset_size = 101

    w1 = tf.Variable(tf.random_normal([INPUT_NODE_NUM, 20], stddev=1))
    w2 = tf.Variable(tf.random_normal([20, 20], stddev=1))
    w3 = tf.Variable(tf.random_normal([20, 5], stddev=1))
    #w4 = tf.Variable(tf.random_normal([3,1],stddev = 1))
    saver = tf.train.Saver()
    x_w1 = tf.matmul(x, w1)
    x_w1 = tf.sigmoid(x_w1)
    w1_w2 = tf.matmul(x_w1, w2)
    w1_w2 = tf.sigmoid(w1_w2)
    y = tf.matmul(w1_w2, w3)
    y = tf.sigmoid(y)

    InputX = rdm.rand(1, INPUT_NODE_NUM)
    if len(strD) != 7:
        return json.dumps('em...try to keep it in 7 numbers')
    if strD.isdigit():
        pass
    else:
        return json.dumps('please use int number')

    with tf.Session() as sess:
        #读取模型
        ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)
        else:
            pass
        InputX[0][0] = strD[0]
        InputX[0][1] = strD[1]
        InputX[0][2] = strD[2]
        InputX[0][3] = strD[3]
        InputX[0][4] = strD[4]
        InputX[0][5] = strD[5]
        InputX[0][6] = strD[6]
        #print("x :")
        #print(InputX)

        predict_output = sess.run(y, {x: InputX})
        #print("predict_output :")
        #print(predict_output)
        predict_outputInt = tf.round(predict_output)
        #print("predict_outputInt :")
        #print(predict_outputInt)
        output_num = outputByChinese(predict_outputInt, sess)
        #print("output_num:")
        #print(output_num)

    if output_num == 0:
        #return render_template('homepage.html',file=file)
        return render_template("veryBad.html")
        return json.dumps('I don\'t want to go to this place at all')
    if output_num == 1:
        return render_template("Bad.html")
        return json.dumps('well,I just don\'t like there')

    if output_num == 2:
        return render_template("justSoSo.html")
        #return json.dumps(
        #'Just so so, Would you like to send an e-mail to explain the detail? [email protected]'
        #)

    if output_num == 3:
        return render_template("goodOption.html")
        #return json.dumps(
        #'It\'s a good oppotunity,contact me with 15365180821'
        #)

    if output_num == 4:
        return render_template("veryInterested.html")
        #return json.dumps(
        #'I am very interested in it,please contact me with 15365180821'
        #)

    return render_template("confused.html")
Example #58
0
from numpy.random import RandomState
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
from random import Random

seed = 42

py_rng = Random(seed)
np_rng = RandomState(seed)
t_rng = RandomStreams(seed)

def set_seed(n):
	global seed, py_rng, np_rng, t_rng

	seed = n
	py_rng = Random(seed)
	np_rng = RandomState(seed)
	t_rng = RandomStreams(seed)
Example #59
0
import pyfits
import math
import pylab as pl
import matplotlib.pyplot as plt
import heapq
from operator import xor
import scipy.signal
from numpy import float64
import astroML.time_series
import astroML_addons.periodogram
import cython
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from sklearn.cluster import DBSCAN,Ward
from numpy.random import RandomState
rng = RandomState(42)
import itertools
import sys

"""cimport numpy as np
cimport cython

__all__ = ['lombscargle']


cdef extern from "math.h":
    double cos(double)
    double sin(double)
    double atan(double)

@cython.boundscheck(False)
Example #60
0
from sklearn.datasets import fetch_olivetti_faces
from sklearn import decomposition

# 非负矩阵分解 Non-negative Matrix Fatorization

# 设置2*3图相框,制定图像排列情况,2行3列
n_row, n_col = 2, 3
n_components = n_row * n_col

# 设置提取特征的数目为64*64
image_shape = (64, 64)

###############################################################################
# Load faces data
# 使用随机种子打乱顺序,每次运行程序出现的图像和特征都不一样
dataset = fetch_olivetti_faces(shuffle=True, random_state=RandomState(0))
faces = dataset.data

###############################################################################
def plot_gallery(title, images, n_col=n_col, n_row=n_row):
    # 创建图片,并且制定图片的大小
    plt.figure(figsize=(2. * n_col, 2.26 * n_row))
    # 设置标题及字号大小
    plt.suptitle(title, size=16)

    for i, comp in enumerate(images):
        # 选择画制的子图
        plt.subplot(n_row, n_col, i + 1)

        vmax = max(comp.max(), -comp.min())