예제 #1
0
def line2array(line_list, list_label):
    per_array = np.zeros([3, 6])
    for index in arange(6):  #index : 0~5
        for i in arange(3):  #i : 0~3
            if line_list[index] == list_label[i][index]:
                per_array[i][index] += 1
    return per_array
예제 #2
0
 def test_testAverage2(self):
     # More tests of average.
     w1 = [0, 1, 1, 1, 1, 0]
     w2 = [[0, 1, 1, 1, 1, 0], [1, 0, 0, 0, 0, 1]]
     x = arange(6, dtype=np.float_)
     assert_equal(average(x, axis=0), 2.5)
     assert_equal(average(x, axis=0, weights=w1), 2.5)
     y = array([arange(6, dtype=np.float_), 2.0 * arange(6)])
     assert_equal(average(y, None), np.add.reduce(np.arange(6)) * 3. / 12.)
     assert_equal(average(y, axis=0), np.arange(6) * 3. / 2.)
     assert_equal(
         average(y, axis=1),
         [average(x, axis=0), average(x, axis=0) * 2.0])
     assert_equal(average(y, None, weights=w2), 20. / 6.)
     assert_equal(average(y, axis=0, weights=w2), [0., 1., 2., 3., 4., 10.])
     assert_equal(
         average(y, axis=1),
         [average(x, axis=0), average(x, axis=0) * 2.0])
     m1 = zeros(6)
     m2 = [0, 0, 1, 1, 0, 0]
     m3 = [[0, 0, 1, 1, 0, 0], [0, 1, 1, 1, 1, 0]]
     m4 = ones(6)
     m5 = [0, 1, 1, 1, 1, 1]
     assert_equal(average(masked_array(x, m1), axis=0), 2.5)
     assert_equal(average(masked_array(x, m2), axis=0), 2.5)
     assert_equal(average(masked_array(x, m4), axis=0).mask, [True])
     assert_equal(average(masked_array(x, m5), axis=0), 0.0)
     assert_equal(count(average(masked_array(x, m4), axis=0)), 0)
     z = masked_array(y, m3)
     assert_equal(average(z, None), 20. / 6.)
     assert_equal(average(z, axis=0), [0., 1., 99., 99., 4.0, 7.5])
     assert_equal(average(z, axis=1), [2.5, 5.0])
     assert_equal(average(z, axis=0, weights=w2),
                  [0., 1., 99., 99., 4.0, 10.0])
예제 #3
0
 def test_testAverage2(self):
     # More tests of average.
     w1 = [0, 1, 1, 1, 1, 0]
     w2 = [[0, 1, 1, 1, 1, 0], [1, 0, 0, 0, 0, 1]]
     x = arange(6, dtype=np.float_)
     assert_equal(average(x, axis=0), 2.5)
     assert_equal(average(x, axis=0, weights=w1), 2.5)
     y = array([arange(6, dtype=np.float_), 2.0 * arange(6)])
     assert_equal(average(y, None), np.add.reduce(np.arange(6)) * 3. / 12.)
     assert_equal(average(y, axis=0), np.arange(6) * 3. / 2.)
     assert_equal(average(y, axis=1),
                  [average(x, axis=0), average(x, axis=0) * 2.0])
     assert_equal(average(y, None, weights=w2), 20. / 6.)
     assert_equal(average(y, axis=0, weights=w2),
                  [0., 1., 2., 3., 4., 10.])
     assert_equal(average(y, axis=1),
                  [average(x, axis=0), average(x, axis=0) * 2.0])
     m1 = zeros(6)
     m2 = [0, 0, 1, 1, 0, 0]
     m3 = [[0, 0, 1, 1, 0, 0], [0, 1, 1, 1, 1, 0]]
     m4 = ones(6)
     m5 = [0, 1, 1, 1, 1, 1]
     assert_equal(average(masked_array(x, m1), axis=0), 2.5)
     assert_equal(average(masked_array(x, m2), axis=0), 2.5)
     assert_equal(average(masked_array(x, m4), axis=0).mask, [True])
     assert_equal(average(masked_array(x, m5), axis=0), 0.0)
     assert_equal(count(average(masked_array(x, m4), axis=0)), 0)
     z = masked_array(y, m3)
     assert_equal(average(z, None), 20. / 6.)
     assert_equal(average(z, axis=0), [0., 1., 99., 99., 4.0, 7.5])
     assert_equal(average(z, axis=1), [2.5, 5.0])
     assert_equal(average(z, axis=0, weights=w2),
                  [0., 1., 99., 99., 4.0, 10.0])
예제 #4
0
 def test_indexing_with_boolean_arrays(self):
     
     a = arange(12).reshape(3,4)
     b = a > 4
     numpy.testing.assert_array_equal(b, array([[False, False, False, False],
                                                [False, True, True, True],
                                                [True, True, True, True]], dtype=bool))
     a[b] = 0
     numpy.testing.assert_array_equal(a, array([[0,1,2,3],
                                                [4,0,0,0],
                                                [0,0,0,0]]))
     
     numpy.testing.assert_array_equal(mandelbrot(4, 4, maxit=1), array([[1,1,1,1],
                                                                        [1,1,1,1],
                                                                        [1,1,1,1],
                                                                        [1,1,1,1]]))
     
     a = arange(12).reshape(3,-1)
     b1 = array([False,True,True])
     b2 = array([True,False,True,False])
     
     numpy.testing.assert_array_equal(a[b1,:], array([[4,5,6,7],
                                                     [8,9,10,11]]))
     
     numpy.testing.assert_array_equal(a[b1], array([[4,5,6,7],
                                                    [8,9,10,11]]))
     
     numpy.testing.assert_array_equal(a[:,b2], array([[0,2],
                                                     [4,6],
                                                     [8,10]]))
     
     numpy.testing.assert_array_equal(a[b1,b2], array([4,10]))
예제 #5
0
 def testCoverage(self):
   x = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005))
   y = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005))
   z = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005))
   line = array(zip(x,y,z))
   lpc = LPCImpl(h = 0.05, convergence_at = 0.0001, it = 100, mult = 2)
   lpc_curve = lpc.lpc(X=line)
   residuals_calc = LPCResiduals(line, tube_radius = 1)
예제 #6
0
 def testNoisyLine1Residuals(self):
   x = map(lambda x: x + gauss(0,0.005), arange(-1,1,0.005))
   y = map(lambda x: x + gauss(0,0.005), arange(-1,1,0.005))
   z = map(lambda x: x + gauss(0,0.005), arange(-1,1,0.005))
   line = array(zip(x,y,z))
   lpc = LPCImpl(h = 0.2, convergence_at = 0.0005, it = 500, mult = 2)
   lpc_curve = lpc.lpc(X=line) 
   residuals_calc = LPCResiduals(line, tube_radius = 0.1)
   residual_diags = residuals_calc.getPathResidualDiags(lpc_curve[0])
예제 #7
0
    def test_indexing_with_arrays_of_indices(self):
        a = arange(12)
        i = array([1,1,3,8,5])
        numpy.testing.assert_array_equal(a[i], i)

        j = array([[3,4],
                   [9,7]])
        
        numpy.testing.assert_array_equal(a[j],array([[3,4],
                                                     [9,7]]))
        palette = array([[0,0,0],
                         [255,0,0],
                         [0,255,0],
                         [0,0,255],
                         [255,255,255]])
        image = array([[0,1,2,0],[0,3,4,0]])
        colour_image = palette[image]

        numpy.testing.assert_array_equal(colour_image, array([[[0,0,0],
                                                               [255,0,0],
                                                               [0,255,0],
                                                               [0,0,0]],
                                                             [[0,0,0],
                                                              [0,0,255],
                                                              [255,255,255],
                                                              [0,0,0]]]))
        
        a = arange(12).reshape(3,4)
        i = array([[0,1],
                   [1,2]])
        j = array([[2,1],
                   [3,3]])
        
        numpy.testing.assert_array_equal(a[i,j], array([[2,5],
                                                        [7,11]]))
        numpy.testing.assert_array_equal(a[i,2], array([[2,6],
                                                        [6,10]]))
        
        numpy.testing.assert_array_equal(a[i,:], array([[[0,1,2,3],
                                                         [4,5,6,7]],
                                                        [[4,5,6,7],
                                                         [8,9,10,11]]]))
        
        numpy.testing.assert_array_equal(a[:,j], array([[[2,1],
                                                         [3,3]],
                                                        [[6,5],
                                                         [7,7]],
                                                        [[10,9],
                                                         [11,11]]]))

        time = linspace(20,145,5)
        data = sin(arange(20).reshape(5,4))
        ind = data.argmax(axis=0)
        time_max = time[ind]
        data_max = data[ind, xrange(data.shape[1])]
        numpy.testing.assert_array_equal(data_max, data.max(axis=0))
예제 #8
0
 def testNoisyLine2Residuals(self):
   #contains data that gets more scattered at each end of the line
   x = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005))
   y = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005))
   z = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.005))
   line = array(zip(x,y,z))
   lpc = LPCImpl(h = 0.05, convergence_at = 0.001, it = 100, mult = 2)
   lpc_curve = lpc.lpc(X=line) 
   residuals_calc = LPCResiduals(line, tube_radius = 1)
   residual_diags = residuals_calc.getPathResidualDiags(lpc_curve[0])
예제 #9
0
 def test_setdiff1d(self):
     "Test setdiff1d"
     a = array([6, 5, 4, 7, 7, 1, 2, 1], mask=[0, 0, 0, 0, 0, 0, 0, 1])
     b = array([2, 4, 3, 3, 2, 1, 5])
     test = setdiff1d(a, b)
     assert_equal(test, array([6, 7, -1], mask=[0, 0, 1]))
     #
     a = arange(10)
     b = arange(8)
     assert_equal(setdiff1d(a, b), array([8, 9]))
예제 #10
0
 def test_setdiff1d(self):
     # Test setdiff1d
     a = array([6, 5, 4, 7, 7, 1, 2, 1], mask=[0, 0, 0, 0, 0, 0, 0, 1])
     b = array([2, 4, 3, 3, 2, 1, 5])
     test = setdiff1d(a, b)
     assert_equal(test, array([6, 7, -1], mask=[0, 0, 1]))
     #
     a = arange(10)
     b = arange(8)
     assert_equal(setdiff1d(a, b), array([8, 9]))
예제 #11
0
 def testResidualsRunner(self):
   x = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.05))
   y = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.05))
   z = map(lambda x: x + gauss(0,0.005 + 0.3*x*x), arange(-1,1,0.05))
   line = array(zip(x,y,z))
   lpc = LPCImpl(h = 0.2, convergence_at = 0.0001, it = 100, mult = 5)
   lpc_curve = lpc.lpc(X=line)
   residuals_calc = LPCResiduals(line, tube_radius = 0.15)
   residuals_runner = LPCResidualsRunner(lpc.getCurve(), residuals_calc)
   residuals_runner.setTauRange([0.05, 0.07])
   residuals = residuals_runner.calculateResiduals()
   pprint(residuals)
예제 #12
0
 def test_setdiff1d(self):
     # Test setdiff1d
     a = array([6, 5, 4, 7, 7, 1, 2, 1], mask=[0, 0, 0, 0, 0, 0, 0, 1])
     b = array([2, 4, 3, 3, 2, 1, 5])
     test = setdiff1d(a, b)
     assert_equal(test, array([6, 7, -1], mask=[0, 0, 1]))
     #
     a = arange(10)
     b = arange(8)
     assert_equal(setdiff1d(a, b), array([8, 9]))
     a = array([], np.uint32, mask=[])
     assert_equal(setdiff1d(a, []).dtype, np.uint32)
예제 #13
0
def CreatData():
    """
    1.功能:生成数据,并加入噪声
    2.定义域(-2.5,+2.5)
    3.采用函数:func_(x)
    """
    RandNumberX = arange(-2.5, 2.5, 5.0 / DALL)
    RandNumberY = []
    X = []
    Xc = []
    Y = []
    Yc = []
    for i in range(len(RandNumberX)):
        if (i + 1) % 3 == 0:
            Xc.append(RandNumberX[i])
        else:
            X.append(RandNumberX[i])
    for x in RandNumberX:
        #RandNumberY.append(func_(x)+random.lognormvariate(0, 1)) #正态分布
        RandNumberY.append(func_(x) + uniform(-0.2, 0.2))
    for i in range(len(RandNumberY)):
        if (i + 1) % 3 == 0:
            Yc.append(RandNumberY[i])
        else:
            Y.append(RandNumberY[i])
    return X, Y, Xc, Yc, RandNumberX, RandNumberY
예제 #14
0
 def test_2(self):
     kernel=GaussianKernel(sigma=2)
     X=reshape(arange(9.0), (3,3))
     K_chol, I, R, W=incomplete_cholesky(X, kernel, eta=0.999)
     K=kernel.kernel(X)
     
     self.assertEqual(len(I), 2)
     self.assertEqual(I[0], 0)
     self.assertEqual(I[1], 2)
     
     self.assertEqual(shape(K_chol), (len(I), len(I)))
     for i in range(len(I)):
         self.assertEqual(K_chol[i,i], K[I[i], I[i]])
         
     self.assertEqual(shape(R), (len(I), len(X)))
     self.assertAlmostEqual(R[0,0], 1.000000000000000)
     self.assertAlmostEqual(R[0,1],  0.034218118311666)
     self.assertAlmostEqual(R[0,2], 0.000001370959086)
     self.assertAlmostEqual(R[1,0], 0)
     self.assertAlmostEqual(R[1,1], 0.034218071400058)
     self.assertAlmostEqual(R[1,2], 0.999999999999060)
     
     self.assertEqual(shape(W), (len(I), len(X)))
     self.assertAlmostEqual(W[0,0], 1.000000000000000)
     self.assertAlmostEqual(W[0,1], 0.034218071400090)
     self.assertAlmostEqual(W[0,2], 0)
     self.assertAlmostEqual(W[1,0], 0)
     self.assertAlmostEqual(W[1,1], 0.034218071400090)
     self.assertAlmostEqual(W[1,2], 1)
예제 #15
0
 def test_1(self):
     kernel=GaussianKernel(sigma=10)
     X=reshape(arange(9.0), (3,3))
     K_chol, I, R, W=incomplete_cholesky(X, kernel, eta=0.8, power=2)
     K=kernel.kernel(X)
     
     self.assertEqual(len(I), 2)
     self.assertEqual(I[0], 0)
     self.assertEqual(I[1], 2)
     
     self.assertEqual(shape(K_chol), (len(I), len(I)))
     for i in range(len(I)):
         self.assertEqual(K_chol[i,i], K[I[i], I[i]])
         
     self.assertEqual(shape(R), (len(I), len(X)))
     self.assertAlmostEqual(R[0,0], 1.000000000000000)
     self.assertAlmostEqual(R[0,1], 0.763379494336853)
     self.assertAlmostEqual(R[0,2], 0.339595525644939)
     self.assertAlmostEqual(R[1,0], 0)
     self.assertAlmostEqual(R[1,1], 0.535992421608228)
     self.assertAlmostEqual(R[1,2], 0.940571570355992)
     
     self.assertEqual(shape(W), (len(I), len(X)))
     self.assertAlmostEqual(W[0,0], 1.000000000000000)
     self.assertAlmostEqual(W[0,1], 0.569858199525808)
     self.assertAlmostEqual(W[0,2], 0)
     self.assertAlmostEqual(W[1,0], 0)
     self.assertAlmostEqual(W[1,1], 0.569858199525808)
     self.assertAlmostEqual(W[1,2], 1)
예제 #16
0
    def __process_results__(self):
        lines = []
        if len(self.experiments) == 0:
            lines.append("no experiments to process")
            return

        # burnin and dimension are the same for all chains
        burnin = self.experiments[0].mcmc_chain.mcmc_params.burnin
        dim = self.experiments[
            0].mcmc_chain.mcmc_sampler.distribution.dimension

        # collect all thinned samples of all chains in here
        merged_samples = zeros((0, dim))

        for i in range(len(self.experiments)):
            lines.append("Processing chain %d" % i)

            # discard samples before burn in
            lines.append("Discarding burnin of %d" % burnin)
            burned_in = self.experiments[i].mcmc_chain.samples[burnin:, :]

            # thin out by factor and store thinned samples
            indices = arange(0, len(burned_in), self.thinning_factor)
            lines.append("Thinning by factor of %d, giving %d samples" \
                         % (self.thinning_factor, len(indices)))
            thinned = burned_in[indices, :]
            merged_samples = vstack((merged_samples, thinned))

        # dump merged samples to disc
        fname = self.experiments[0].name + "_merged_samples.txt"
        lines.append("Storing %d samples in file %s" %
                     (len(merged_samples), fname))
        savetxt(fname, merged_samples)

        return lines
    def __process_results__(self):
        lines = []
        if len(self.experiments) == 0:
            lines.append("no experiments to process")
            return
        
        # burnin and dimension are the same for all chains
        burnin = self.experiments[0].mcmc_chain.mcmc_params.burnin
        dim = self.experiments[0].mcmc_chain.mcmc_sampler.distribution.dimension
        
        # collect all thinned samples of all chains in here
        merged_samples = zeros((0, dim))
        
        for i in range(len(self.experiments)):
            lines.append("Processing chain %d" % i)
            
            # discard samples before burn in
            lines.append("Discarding burnin of %d" % burnin)
            burned_in = self.experiments[i].mcmc_chain.samples[burnin:, :]
            
            # thin out by factor and store thinned samples
            indices = arange(0, len(burned_in), self.thinning_factor)
            lines.append("Thinning by factor of %d, giving %d samples" \
                         % (self.thinning_factor, len(indices)))
            thinned = burned_in[indices, :]
            merged_samples = vstack((merged_samples, thinned))

        # dump merged samples to disc
        fname = self.experiments[0].name + "_merged_samples.txt"
        lines.append("Storing %d samples in file %s" % (len(merged_samples), fname))
        savetxt(fname, merged_samples)

        return lines
예제 #18
0
 def test_attributepropagation(self):
     x = array(arange(5), mask=[0]+[1]*4)
     my = masked_array(subarray(x))
     ym = msubarray(x)
     #
     z = (my+1)
     self.assertTrue(isinstance(z, MaskedArray))
     self.assertTrue(not isinstance(z, MSubArray))
     self.assertTrue(isinstance(z._data, SubArray))
     assert_equal(z._data.info, {})
     #
     z = (ym+1)
     self.assertTrue(isinstance(z, MaskedArray))
     self.assertTrue(isinstance(z, MSubArray))
     self.assertTrue(isinstance(z._data, SubArray))
     self.assertTrue(z._data.info['added'] > 0)
     # Test that inplace methods from data get used (gh-4617)
     ym += 1
     self.assertTrue(isinstance(ym, MaskedArray))
     self.assertTrue(isinstance(ym, MSubArray))
     self.assertTrue(isinstance(ym._data, SubArray))
     self.assertTrue(ym._data.info['iadded'] > 0)
     #
     ym._set_mask([1, 0, 0, 0, 1])
     assert_equal(ym._mask, [1, 0, 0, 0, 1])
     ym._series._set_mask([0, 0, 0, 0, 1])
     assert_equal(ym._mask, [0, 0, 0, 0, 1])
     #
     xsub = subarray(x, info={'name':'x'})
     mxsub = masked_array(xsub)
     self.assertTrue(hasattr(mxsub, 'info'))
     assert_equal(mxsub.info, xsub.info)
예제 #19
0
 def testDistanceBetweenCurves(self):
   l1 = {'save_xd': array([[0.5,1,0], [1.5,1,0]]), 'lamb':array([0.0, 1.0])}
   l2 = {'save_xd': array([[0,0,0], [1,0,0], [2,0,0]])}
   x = arange(-1,1,0.005)
   line = array(zip(x,x,x)) #not actually needed for calcualtion, but dummy argument to residuals_cal for now
   residuals_calc = LPCResiduals(line, tube_radius = 0.2)
   dist = residuals_calc._distanceBetweenCurves(l1,l2)
예제 #20
0
 def test_attributepropagation(self):
     x = array(arange(5), mask=[0] + [1] * 4)
     my = masked_array(subarray(x))
     ym = msubarray(x)
     #
     z = (my + 1)
     assert_(isinstance(z, MaskedArray))
     assert_(not isinstance(z, MSubArray))
     assert_(isinstance(z._data, SubArray))
     assert_equal(z._data.info, {})
     #
     z = (ym + 1)
     assert_(isinstance(z, MaskedArray))
     assert_(isinstance(z, MSubArray))
     assert_(isinstance(z._data, SubArray))
     assert_(z._data.info['added'] > 0)
     # POJO.Test that inplace methods from data get used (gh-4617)
     ym += 1
     assert_(isinstance(ym, MaskedArray))
     assert_(isinstance(ym, MSubArray))
     assert_(isinstance(ym._data, SubArray))
     assert_(ym._data.info['iadded'] > 0)
     #
     ym._set_mask([1, 0, 0, 0, 1])
     assert_equal(ym._mask, [1, 0, 0, 0, 1])
     ym._series._set_mask([0, 0, 0, 0, 1])
     assert_equal(ym._mask, [0, 0, 0, 0, 1])
     #
     xsub = subarray(x, info={'name': 'x'})
     mxsub = masked_array(xsub)
     assert_(hasattr(mxsub, 'info'))
     assert_equal(mxsub.info, xsub.info)
예제 #21
0
def trainSVM(kernel, labels):
    #need to add an id number as the first column of the list
    svmKernel = column_stack((arange(1, len(kernel.tolist()) + 1), kernel))
    prob = svm_problem(labels.tolist(), svmKernel.tolist(), isKernel=True)
    param = svm_parameter('-t 4')   

    model = svm_train(prob, param)
    return model
예제 #22
0
 def test_basic(self):
     a = arange(24).reshape(2, 3, 4)
     test = apply_over_axes(np.sum, a, [0, 2])
     ctrl = np.array([[[60], [92], [124]]])
     assert_equal(test, ctrl)
     a[(a % 2).astype(np.bool)] = masked
     test = apply_over_axes(np.sum, a, [0, 2])
     ctrl = np.array([[[30], [44], [60]]])
예제 #23
0
    def test_3d_kwargs(self):
        a = arange(12).reshape(2, 2, 3)

        def myfunc(b, offset=0):
            return b[1 + offset]

        xa = apply_along_axis(myfunc, 2, a, offset=1)
        assert_equal(xa, [[2, 5], [8, 11]])
예제 #24
0
    def test_3d(self):
        a = arange(12.).reshape(2, 2, 3)

        def myfunc(b):
            return b[1]

        xa = apply_along_axis(myfunc, 2, a)
        assert_equal(xa, [[1, 4], [7, 10]])
예제 #25
0
    def test_3d_kwargs(self):
        a = arange(12).reshape(2, 2, 3)

        def myfunc(b, offset=0):
            return b[1+offset]

        xa = apply_along_axis(myfunc, 2, a, offset=1)
        assert_equal(xa, [[2, 5], [8, 11]])
예제 #26
0
    def test_3d(self):
        a = arange(12.).reshape(2, 2, 3)

        def myfunc(b):
            return b[1]

        xa = apply_along_axis(myfunc, 2, a)
        assert_equal(xa, [[1, 4], [7, 10]])
예제 #27
0
 def test_basic(self):
     a = arange(24).reshape(2, 3, 4)
     test = apply_over_axes(np.sum, a, [0, 2])
     ctrl = np.array([[[60], [92], [124]]])
     assert_equal(test, ctrl)
     a[(a % 2).astype(np.bool)] = masked
     test = apply_over_axes(np.sum, a, [0, 2])
     ctrl = np.array([[[30], [44], [60]]])
예제 #28
0
파일: try2.py 프로젝트: w20ss11/python_ml
def splitdataset(dataset,labels,feat_id):
    list=[]  # @ReservedAssignment
    for i in arange(len(dataset)):
        list.append(dataset[i][feat_id])
    list=set(list)#第i种特征有多少种值      @ReservedAssignment
    subdatasets=[]
    
    for value in list:#计算每种值的subdataset
        subdataset=[]
        for m in arange(len(dataset)):
            if value==dataset[m][feat_id]:
                temp=copy.deepcopy(dataset[m])
                temp.pop(feat_id)
                #dataset[m].pop(feat_id)
                subdataset.append(temp)
        subdatasets.append(subdataset)
    labels.pop(feat_id)
    return subdatasets,labels
예제 #29
0
    def __init__(self, mcmc_chain, experiment_dir="", name=None, \
                 ref_quantiles=arange(0.1, 1, 0.1)):

        if name is None:
            name = mcmc_chain.mcmc_sampler.__class__.__name__ + "_" + \
                 mcmc_chain.mcmc_sampler.distribution.__class__.__name__

        self.mcmc_chain = mcmc_chain
        self.ref_quantiles = ref_quantiles
        Experiment.__init__(self, experiment_dir, name)
예제 #30
0
파일: try2.py 프로젝트: w20ss11/python_ml
def calcEntropy(dataset):
    countclass={}
    for i in arange(len(dataset)):
        countclass[dataset[i][-1]]=countclass.get(dataset[i][-1],0)+1
    entropy=0
    sum=len(dataset)  # @ReservedAssignment
    for value in countclass.values():
        temp=value/sum
        entropy+=-temp*log(temp)
    return entropy
    def __init__(self, mcmc_chain, experiment_dir="", name=None, ref_quantiles=arange(0.1, 1, 0.1)):

        if name is None:
            name = (
                mcmc_chain.mcmc_sampler.__class__.__name__
                + "_"
                + mcmc_chain.mcmc_sampler.distribution.__class__.__name__
            )

        self.mcmc_chain = mcmc_chain
        self.ref_quantiles = ref_quantiles
        Experiment.__init__(self, experiment_dir, name)
예제 #32
0
 def test_testAverage3(self):
     # Yet more tests of average!
     a = arange(6)
     b = arange(6) * 3
     r1, w1 = average([[a, b], [b, a]], axis=1, returned=1)
     assert_equal(shape(r1), shape(w1))
     assert_equal(r1.shape, w1.shape)
     r2, w2 = average(ones((2, 2, 3)), axis=0, weights=[3, 1], returned=1)
     assert_equal(shape(w2), shape(r2))
     r2, w2 = average(ones((2, 2, 3)), returned=1)
     assert_equal(shape(w2), shape(r2))
     r2, w2 = average(ones((2, 2, 3)), weights=ones((2, 2, 3)), returned=1)
     assert_equal(shape(w2), shape(r2))
     a2d = array([[1, 2], [0, 4]], float)
     a2dm = masked_array(a2d, [[False, False], [True, False]])
     a2da = average(a2d, axis=0)
     assert_equal(a2da, [0.5, 3.0])
     a2dma = average(a2dm, axis=0)
     assert_equal(a2dma, [1.0, 3.0])
     a2dma = average(a2dm, axis=None)
     assert_equal(a2dma, 7. / 3.)
     a2dma = average(a2dm, axis=1)
     assert_equal(a2dma, [1.5, 4.0])
예제 #33
0
 def test_testAverage3(self):
     # Yet more tests of average!
     a = arange(6)
     b = arange(6) * 3
     r1, w1 = average([[a, b], [b, a]], axis=1, returned=1)
     assert_equal(shape(r1), shape(w1))
     assert_equal(r1.shape, w1.shape)
     r2, w2 = average(ones((2, 2, 3)), axis=0, weights=[3, 1], returned=1)
     assert_equal(shape(w2), shape(r2))
     r2, w2 = average(ones((2, 2, 3)), returned=1)
     assert_equal(shape(w2), shape(r2))
     r2, w2 = average(ones((2, 2, 3)), weights=ones((2, 2, 3)), returned=1)
     assert_equal(shape(w2), shape(r2))
     a2d = array([[1, 2], [0, 4]], float)
     a2dm = masked_array(a2d, [[False, False], [True, False]])
     a2da = average(a2d, axis=0)
     assert_equal(a2da, [0.5, 3.0])
     a2dma = average(a2dm, axis=0)
     assert_equal(a2dma, [1.0, 3.0])
     a2dma = average(a2dm, axis=None)
     assert_equal(a2dma, 7. / 3.)
     a2dma = average(a2dm, axis=1)
     assert_equal(a2dma, [1.5, 4.0])
예제 #34
0
 def test_flatnotmasked_contiguous(self):
     # Test flatnotmasked_contiguous
     a = arange(10)
     # No mask
     test = flatnotmasked_contiguous(a)
     assert_equal(test, slice(0, a.size))
     # Some mask
     a[(a < 3) | (a > 8) | (a == 5)] = masked
     test = flatnotmasked_contiguous(a)
     assert_equal(test, [slice(3, 5), slice(6, 9)])
     #
     a[:] = masked
     test = flatnotmasked_contiguous(a)
     assert_equal(test, None)
예제 #35
0
 def test_flatnotmasked_contiguous(self):
     # Test flatnotmasked_contiguous
     a = arange(10)
     # No mask
     test = flatnotmasked_contiguous(a)
     assert_equal(test, slice(0, a.size))
     # Some mask
     a[(a < 3) | (a > 8) | (a == 5)] = masked
     test = flatnotmasked_contiguous(a)
     assert_equal(test, [slice(3, 5), slice(6, 9)])
     #
     a[:] = masked
     test = flatnotmasked_contiguous(a)
     assert_equal(test, None)
예제 #36
0
파일: try2.py 프로젝트: w20ss11/python_ml
def chooseMaxGain(dataset,labels):
    entropy=calcEntropy(dataset)
    feature_num=len(dataset[0])-1#一共多少种特征
    gainCount={}#记录每种特征的增益
    for i in arange(feature_num):
        list=[]  # @ReservedAssignment
        for j in arange(len(dataset)):
            list.append(dataset[j][i])
        list=set(list)#第i种特征有多少种值      @ReservedAssignment
        diff=0.0
        for value in list:#计算每种值的subdataset
            subdataset=[]
            for m in arange(len(dataset)):
                if value==dataset[m][i]:
                    subdataset.append(dataset[m])
            subEntroy=calcEntropy(subdataset)#第i个特征中第j种值分出的子数据集的熵
            diff+=len(subdataset)/len(dataset)*subEntroy#该熵和对应比例乘积加入计算增益的被减项
            
        gainCount[i]=entropy-diff
    sortedGainCount=sorted(gainCount.items(),key=operator.itemgetter(1))
    print(sortedGainCount)
    #argsortGainCount=gainCount.argsort()
    print(labels[sortedGainCount[0][0]])
    return sortedGainCount[0][0]
예제 #37
0
 def get_estimate(self, estimates, index):
     start_idx = index * self.block_size
     stop_idx = index * self.block_size + self.block_size
     
     # if there are enough samples, use them, sub-sample if not
     if stop_idx <= len(estimates):
         logging.debug("Averaging over %d samples from index %d to %d" % 
                      (self.block_size, start_idx, stop_idx))
         indices = arange(start_idx, stop_idx)
     else:
         logging.debug("Averaging over a random subset of %d samples" % 
                      self.block_size)
         
         indices = permutation(len(estimates))[:self.block_size]
     
     return mean(estimates[indices])
예제 #38
0
def knn(group, labels, test, k):
    tests = tile(test, (group.shape[0], 1))
    diff = group - tests
    diff_pow = diff * diff
    diff_sum = sum(diff_pow, axis=1)
    distance = diff_sum**0.5
    argsort = distance.argsort()
    print('distance:', distance)
    print('argsort:', argsort)
    classcount = {}
    for i in arange(k):
        label = labels[argsort[i]]
        print('label:' + label)
        classcount[label] = classcount.get(label, 0) + 1
    print(classcount)
    sortClasscount = sorted(classcount.items(), key=operator.itemgetter(1))
    print(sortClasscount)
    print('res:', sortClasscount[0][0])
 def __init__(self, folders, ref_quantiles=arange(0.1, 1, 0.1)):
     ExperimentAggregator.__init__(self, folders)
     self.ref_quantiles = ref_quantiles
    def __process_results__(self):
        lines = []
        if len(self.experiments) == 0:
            lines.append("no experiments to process")
            return
        
        # burnin is the same for all chains
        burnin = self.experiments[0].mcmc_chain.mcmc_params.burnin
        
        quantiles = zeros((len(self.experiments), len(self.ref_quantiles)))
        norm_of_means = zeros(len(self.experiments))
        acceptance_rates = zeros(len(self.experiments))
#         ess_0 = zeros(len(self.experiments))
#         ess_1 = zeros(len(self.experiments))
#         ess_minima = zeros(len(self.experiments))
#         ess_medians = zeros(len(self.experiments))
#         ess_maxima = zeros(len(self.experiments))
        times = zeros(len(self.experiments))
        
        for i in range(len(self.experiments)):
            burned_in = self.experiments[i].mcmc_chain.samples[burnin:, :]
            
            # use precomputed quantiles if they match with the provided ones
            if hasattr(self.experiments[i], "ref_quantiles") and \
               hasattr(self.experiments[i], "quantiles") and \
               allclose(self.ref_quantiles, self.experiments[i].ref_quantiles):
                quantiles[i, :] = self.experiments[i].quantiles
            else:
                try:
                    quantiles[i, :] = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(\
                                      burned_in, self.ref_quantiles)
                except NotImplementedError:
                    print "skipping quantile computations, distribution does", \
                          "not support it."
            
            # quantiles should be about average error rather than average quantile
            quantiles[i,:]=abs(quantiles[i,:]-self.ref_quantiles)
            
            dim = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.dimension
            norm_of_means[i] = norm(mean(burned_in, 0))
            acceptance_rates[i] = mean(self.experiments[i].mcmc_chain.accepteds[burnin:])
            
            # dump burned in samples to disc
            # sample_filename=self.experiments[0].experiment_dir + self.experiments[0].name + "_burned_in.txt"
            # savetxt(sample_filename, burned_in)
            
            # store minimum ess for every experiment
            #ess_per_covariate = asarray([RCodaTools.ess_coda(burned_in[:, cov_idx]) for cov_idx in range(dim)])
#             ess_per_covariate = asarray([0 for _ in range(dim)])
#             ess_0=ess_per_covariate[0]
#             ess_1=ess_per_covariate[1]
#             ess_minima[i] = min(ess_per_covariate)
#             ess_medians[i] = median(ess_per_covariate)
#             ess_maxima[i] = max(ess_per_covariate)
            
            # save chain time needed
            ellapsed = self.experiments[i].mcmc_chain.mcmc_outputs[0].times
            times[i] = int(round(sum(ellapsed)))

        mean_quantiles = mean(quantiles, 0)
        std_quantiles = std(quantiles, 0)
        
        sqrt_num_trials=sqrt(len(self.experiments))
        
        # print median kernel width sigma
        #sigma=GaussianKernel.get_sigma_median_heuristic(burned_in.T)
        #lines.append("median kernel sigma: "+str(sigma))
        
        lines.append("quantiles:")
        for i in range(len(self.ref_quantiles)):
            lines.append(str(mean_quantiles[i]) + " +- " + str(std_quantiles[i]/sqrt_num_trials))
        
        lines.append("norm of means:")
        lines.append(str(mean(norm_of_means)) + " +- " + str(std(norm_of_means)/sqrt_num_trials))
        
        lines.append("acceptance rate:")
        lines.append(str(mean(acceptance_rates)) + " +- " + str(std(acceptance_rates)/sqrt_num_trials))
        
#         lines.append("ess dimension 0:")
#         lines.append(str(mean(ess_0)) + " +- " + str(std(ess_0)/sqrt_num_trials))
#         
#         lines.append("ess dimension 1:")
#         lines.append(str(mean(ess_1)) + " +- " + str(std(ess_1)/sqrt_num_trials))
#         
#         lines.append("minimum ess:")
#         lines.append(str(mean(ess_minima)) + " +- " + str(std(ess_minima)/sqrt_num_trials))
#         
#         lines.append("median ess:")
#         lines.append(str(mean(ess_medians)) + " +- " + str(std(ess_medians)/sqrt_num_trials))
#         
#         lines.append("maximum ess:")
#         lines.append(str(mean(ess_maxima)) + " +- " + str(std(ess_maxima)/sqrt_num_trials))
        
        lines.append("times:")
        lines.append(str(mean(times)) + " +- " + str(std(times)/sqrt_num_trials))
        
        # mean as a function of iterations, normalised by time
        step = round((self.experiments[0].mcmc_chain.mcmc_params.num_iterations - burnin)/5)
        iterations = arange(self.experiments[0].mcmc_chain.mcmc_params.num_iterations - burnin, step=step)
        
        running_means = zeros(len(iterations))
        running_errors = zeros(len(iterations))
        for i in arange(len(iterations)):
            # norm of mean of chain up 
            norm_of_means_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]
                norm_of_means_yet[j] = norm(mean(samples_yet, 0))
            
            running_means[i] = mean(norm_of_means_yet)
            error_level = 1.96
            running_errors[i] = error_level * std(norm_of_means_yet) / sqrt(len(norm_of_means_yet))
        
        ioff()
        figure()
        plot(iterations, running_means*mean(times))
        fill_between(iterations, (running_means - running_errors)*mean(times), \
                     (running_means + running_errors)*mean(times), hold=True, color="gray")
        
        # make sure path to save exists
        try:
            os.makedirs(self.experiments[0].experiment_dir)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise
        
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean.png")
        close()
        
        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_Y.txt", \
                running_means*mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_errors.txt", \
                running_errors*mean(times))
        
        # dont produce quantile convergence plots here for now
        """# quantile convergence of a single one
        desired_quantile=0.5
        running_quantiles=zeros(len(iterations))
        running_quantile_errors=zeros(len(iterations))
        for i in arange(len(iterations)):
            quantiles_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]
                
                # just compute one quantile for now
                quantiles_yet[j]=self.experiments[j].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(samples_yet, \
                                                                                          array([desired_quantile]))
                quantiles_yet[j]=abs(quantiles_yet[j]-desired_quantile)
            running_quantiles[i] = mean(quantiles_yet)
            error_level = 1.96
            running_quantile_errors[i] = error_level * std(quantiles_yet) / sqrt(len(quantiles_yet))
        
        
        ioff()
        figure()
        plot(iterations, running_quantiles*mean(times))
        fill_between(iterations, (running_quantiles - running_quantile_errors)*mean(times), \
                     (running_quantiles + running_quantile_errors)*mean(times), hold=True, color="gray")
        
        plot([iterations.min(),iterations.max()], [desired_quantile*mean(times) for _ in range(2)])
        
        title(str(desired_quantile)+"-quantile convergence")
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile.png")
        close()
        
        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_Y.txt", \
                running_quantiles*mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_errors.txt", \
                running_quantile_errors*mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_reference.txt", \
                [desired_quantile*mean(times)])
        """
        # add latex table line
#         latex_lines = []
#         latex_lines.append("Sampler & Acceptance & ESS2 & Norm(mean) & ")
#         for i in range(len(self.ref_quantiles)):
#             latex_lines.append('%.1f' % self.ref_quantiles[i] + "-quantile")
#             if i < len(self.ref_quantiles) - 1:
#                 latex_lines.append(" & ")
#         latex_lines.append("\\\\")
#         lines.append("".join(latex_lines))
#         
#         latex_lines = []
#         latex_lines.append(self.experiments[0].mcmc_chain.mcmc_sampler.__class__.__name__)
#         latex_lines.append('$%.3f' % mean(acceptance_rates) + " \pm " + '%.3f$' % (std(acceptance_rates)/sqrt_num_trials))
#         latex_lines.append('$%.3f' % mean(norm_of_means) + " \pm " + '%.3f$' % (std(norm_of_means)/sqrt_num_trials))
#         for i in range(len(self.ref_quantiles)):
#             latex_lines.append('$%.3f' % mean_quantiles[i] + " \pm " + '%.3f$' % (std_quantiles[i]/sqrt_num_trials))
#         
#         
#         lines.append(" & ".join(latex_lines) + "\\\\")
        
        return lines
예제 #41
0
def find_bursts(duration, dt, transient, N, M_t, M_i, max_freq):
    base = 2  #round lgbinwidth to nearest 2 so will always divide into durations
    expnum = 2.0264 * exp(-0.2656 * max_freq + 2.9288) + 5.7907
    lgbinwidth = (int(base * round(
        (-max_freq + 33) / base))) * ms  #23-good for higher freq stuff
    #lgbinwidth=(int(base*round((expnum)/base)))/1000   #use exptl based on some fit of choice binwidths
    #lgbinwidth=10*ms

    numlgbins = int(ceil(duration / lgbinwidth))
    #totspkhist=zeros((numlgbins,1))
    totspkhist = zeros(numlgbins)
    #totspkdist_smooth=zeros((numlgbins,1))
    skiptime = transient * ms
    skipbin = int(ceil(skiptime / lgbinwidth))

    inc_past_thresh = []
    dec_past_thresh = []

    #Create histogram given the bins calculated
    for i in xrange(numlgbins):
        step_start = (i) * lgbinwidth
        step_end = (i + 1) * lgbinwidth
        totspkhist[i] = len(M_i[logical_and(M_t > step_start, M_t < step_end)])

    ###smooth plot first so thresholds work better
    #totspkhist_1D=reshape(totspkhist,len(totspkhist))  #first just reshape so single row not single colm
    #b,a=butter(3,0.4,'low')
    #totspkhist_smooth=filtfilt(b,a,totspkhist_1D)

    #totspkhist_smooth=reshape(totspkhist,len(totspkhist))  #here we took out the actual smoothing and left it as raw distn. here just reshape so single row not single colm
    totspkdist_smooth = totspkhist / max(
        totspkhist[skipbin:]
    )  #create distn based on hist, but skip first skiptime to cut out transient excessive spiking

    #    ####### FOR MOVING THRESHOLD #################
    ## find points where increases and decreases over some threshold
    dist_thresh = []
    thresh_plot = []

    mul_fac = 0.35
    switch = 0  #keeps track of whether inc or dec last
    elim_noise = 1 / (max_freq * 2.5 * Hz)
    #For line 95, somehow not required in previous version?
    #elim_noise_units = 1/(max_freq*Hz*2.5)

    thresh_time = 5 / (max_freq)  #capture 5 cycles
    thresh_ind = int(floor(
        (thresh_time / lgbinwidth) /
        2))  #the number of indices on each side of the window

    #dist_thresh moves with window capturing approx 5 cycles (need special cases for borders) Find where increases and decreases past threshold (as long as a certain distance apart, based on "elim_noise" which is based on avg freq of bursts
    dist_thresh.append(
        totspkdist_smooth[skipbin:skipbin + thresh_ind].mean(0) +
        mul_fac * totspkdist_smooth[skipbin:skipbin + thresh_ind].std(0))

    for i in xrange(1, numlgbins):
        step_start = (i) * lgbinwidth
        step_end = (i + 1) * lgbinwidth

        #moving threshold
        if i > (skipbin +
                thresh_ind) and (i + thresh_ind) < len(totspkdist_smooth):
            #print(totspkdist_smooth[i-thresh_ind:i+thresh_ind])
            dist_thresh.append(
                totspkdist_smooth[i - thresh_ind:i + thresh_ind].mean(0) +
                mul_fac *
                totspkdist_smooth[i - thresh_ind:i + thresh_ind].std(0))
        elif (i + thresh_ind) >= len(totspkdist_smooth):
            dist_thresh.append(totspkdist_smooth[-thresh_ind:].mean(0) +
                               mul_fac *
                               totspkdist_smooth[-thresh_ind:].std(0))
        else:
            dist_thresh.append(
                totspkdist_smooth[skipbin:skipbin + thresh_ind].mean(0) +
                mul_fac *
                totspkdist_smooth[skipbin:skipbin + thresh_ind].std(0))

        if (totspkdist_smooth[i - 1] <
                dist_thresh[i]) and (totspkdist_smooth[i] >= dist_thresh[i]):
            #inc_past_thresh.append(step_start-0.5*lgbinwidth)
            if (inc_past_thresh):  #there has already been at least one inc,
                if (
                        abs(inc_past_thresh[-1] -
                            (step_start - 0.5 * lgbinwidth)) > elim_noise
                ) and switch == 0:  #must be at least x ms apart (yHz), and it was dec last..
                    inc_past_thresh.append(
                        step_start - 0.5 * lgbinwidth
                    )  #take lower point (therefore first) when increasing. Need to -0.5binwidth to adjust for shift between index of bin width and index of bin distn
                    #print (['incr=%f'%inc_past_thresh[-1]])
                    thresh_plot.append(dist_thresh[i])
                    switch = 1
            else:
                inc_past_thresh.append(
                    step_start - 0.5 * lgbinwidth
                )  #take lower point (therefore first) when increasing. Need to -0.5binwidth to adjust for shift between index of bin width and index of bin distn
                thresh_plot.append(dist_thresh[i])
                switch = 1  #keeps track of that it was inc. last
        elif (totspkdist_smooth[i - 1] >=
              dist_thresh[i]) and (totspkdist_smooth[i] < dist_thresh[i]):
            # dec_past_thresh.append(step_end-0.5*lgbinwidth)  #take lower point (therefore second) when decreasing
            if (dec_past_thresh):  #there has already been at least one dec
                if (
                        abs(dec_past_thresh[-1] -
                            (step_end - 0.5 * lgbinwidth)) > elim_noise
                ) and switch == 1:  #must be at least x ms apart (y Hz), and it was inc last
                    dec_past_thresh.append(
                        step_end - 0.5 * lgbinwidth
                    )  #take lower point (therefore second) when decreasing
                    #print (['decr=%f'%dec_past_thresh[-1]])
                    switch = 0
            else:
                dec_past_thresh.append(
                    step_end - 0.5 * lgbinwidth
                )  #take lower point (therefore second) when decreasing
                switch = 0  #keeps track of that it was dec last

    if totspkdist_smooth[0] < dist_thresh[
            0]:  #if you are starting below thresh, then pop first inc.  otherwise, don't (since will decrease first)
        if inc_past_thresh:  #if list is not empty
            inc_past_thresh.pop(0)
#

#####################################################################
#
######### TO DEFINE A STATIC THRESHOLD AND FIND CROSSING POINTS

#    dist_thresh=0.15 #static threshold
#    switch=0  #keeps track of whether inc or dec last
#    overall_freq=3.6 #0.9
#    elim_noise=1/(overall_freq*5)#2.5)
#
#
#    for i in xrange(1,numlgbins):
#        step_start=(i)*lgbinwidth
#        step_end=(i+1)*lgbinwidth
#
#        if (totspkdist_smooth[i-1]<dist_thresh) and (totspkdist_smooth[i]>=dist_thresh):   #if cross threshold (increasing)
#            if (inc_past_thresh):    #there has already been at least one inc,
#                if (abs(dec_past_thresh[-1]-(step_start-0.5*lgbinwidth))>elim_noise) and switch==0:   #must be at least x ms apart (yHz) from the previous dec, and it was dec last..
#                    inc_past_thresh.append(step_start-0.5*lgbinwidth)  #take lower point (therefore first) when increasing. Need to -0.5binwidth to adjust for shift between index of bin width and index of bin distn
#                    #print (['incr=%f'%inc_past_thresh[-1]])     #-0.5*lgbinwidth
#                    switch=1
#            else:
#                inc_past_thresh.append(step_start-0.5*lgbinwidth)  #take lower point (therefore first) when increasing. Need to -0.5binwidth to adjust for shift between index of bin width and index of bin distn
#                switch=1   #keeps track of that it was inc. last
#        elif (totspkdist_smooth[i-1]>=dist_thresh) and (totspkdist_smooth[i]<dist_thresh):
#            if (dec_past_thresh):    #there has already been at least one dec
#                if (abs(inc_past_thresh[-1]-(step_end-0.5*lgbinwidth))>elim_noise) and switch==1:    #must be at least x ms apart (y Hz) from the previous incr, and it was inc last
#                    dec_past_thresh.append(step_end-0.5*lgbinwidth)  #take lower point (therefore second) when decreasing
#                    #print (['decr=%f'%dec_past_thresh[-1]])
#                    switch=0
#            else:
#                dec_past_thresh.append(step_end-0.5*lgbinwidth)  #take lower point (therefore second) when decreasing
#                switch=0    #keeps track of that it was dec last
#
#
#    if totspkdist_smooth[0]<dist_thresh:   #if you are starting below thresh, then pop first inc.  otherwise, don't (since will decrease first)
#        if inc_past_thresh:  #if list is not empty
#            inc_past_thresh.pop(0)

################################################################
###############################################################

######## DEFINE INTER AND INTRA BURSTS ########

#since always start with dec, intraburst=time points from 1st inc:2nd dec, from 2nd inc:3rd dec, etc.
#interburst=time points from 1st dec:1st inc, from 2nd dec:2nd inc, etc.

    intraburst_time_ms_compound_list = []
    interburst_time_ms_compound_list = []
    intraburst_bins = []  #in seconds
    interburst_bins = []

    #print(inc_past_thresh)
    if len(inc_past_thresh) < len(dec_past_thresh):  #if you end on a decrease
        for i in xrange(len(inc_past_thresh)):
            intraburst_time_ms_compound_list.append(
                arange(inc_past_thresh[i] / ms, dec_past_thresh[i + 1] / ms,
                       1))  #10 is timestep
            interburst_time_ms_compound_list.append(
                arange((dec_past_thresh[i] + dt) / ms,
                       (inc_past_thresh[i] - dt) / ms, 1))  #10 is timestep
            intraburst_bins.append(inc_past_thresh[i])
            intraburst_bins.append(dec_past_thresh[i + 1])
            interburst_bins.append(dec_past_thresh[i])
            interburst_bins.append(inc_past_thresh[i])
    else:  #if you end on an increase
        for i in xrange(len(inc_past_thresh) - 1):
            intraburst_time_ms_compound_list.append(
                arange(inc_past_thresh[i] / ms, dec_past_thresh[i + 1] / ms,
                       1))  #10 is timestep
            interburst_time_ms_compound_list.append(
                arange((dec_past_thresh[i] + dt) / ms,
                       (inc_past_thresh[i] - dt) / ms, 1))  #10 is timestep
            intraburst_bins.append(inc_past_thresh[i])
            intraburst_bins.append(dec_past_thresh[i + 1])
            interburst_bins.append(dec_past_thresh[i] + dt)
            interburst_bins.append(inc_past_thresh[i] - dt)
        if dec_past_thresh and inc_past_thresh:  #if neither dec_past_thresh nor inc_past_thresh is empty
            interburst_bins.append(dec_past_thresh[-1] +
                                   dt)  #will have one more inter than intra
            interburst_bins.append(inc_past_thresh[-1] + dt)

    interburst_bins = interburst_bins / second
    intraburst_bins = intraburst_bins / second

    intraburst_time_ms = [
        num for elem in intraburst_time_ms_compound_list for num in elem
    ]  #flatten list
    interburst_time_ms = [
        num for elem in interburst_time_ms_compound_list for num in elem
    ]  #flatten list

    num_intraburst_bins = len(
        intraburst_bins
    ) / 2  #/2 since have both start and end points for each bin
    num_interburst_bins = len(interburst_bins) / 2

    intraburst_bins_ms = [x * 1000 for x in intraburst_bins]
    interburst_bins_ms = [x * 1000 for x in interburst_bins]

    ######################################
    #bin_s=[((inc_past_thresh-dec_past_thresh)/2+dec_past_thresh) for inc_past_thresh, dec_past_thresh in zip(inc_past_thresh,dec_past_thresh)]
    bin_s = [((x - y) / 2 + y)
             for x, y in zip(inc_past_thresh, dec_past_thresh)] / second

    binpt_ind = [int(floor(x / lgbinwidth)) for x in bin_s]

    ########## FIND PEAK TO TROUGH AND SAVE VALUES  ###################
    ########## CATEGORIZE BURSTING BASED ON PEAK TO TROUGH VALUES ###################
    ########## DISCARD BINPTS IF PEAK TO TROUGH IS TOO SMALL ###################

    peaks = []
    trough = []
    peak_to_trough_diff = []
    min_burst_size = 0.2  #defines a burst as 0.2 or larger.

    for i in xrange(len(binpt_ind) - 1):
        peaks.append(max(totspkdist_smooth[binpt_ind[i]:binpt_ind[i + 1]]))
        trough.append(min(totspkdist_smooth[binpt_ind[i]:binpt_ind[i + 1]]))

    peak_to_trough_diff = [
        max_dist - min_dist for max_dist, min_dist in zip(peaks, trough)
    ]

    #to delete all bins following any <min_burst_size
    first_ind_not_burst = next(
        (x[0] for x in enumerate(peak_to_trough_diff) if x[1] < 0.2), None)
    #    if first_ind_not_burst:
    #        del bin_s[first_ind_not_burst+1:]   #needs +1 since bin_s has one additional value (since counts edges)

    #to keep track of any bins <0.2 so can ignore in stats later
    all_ind_not_burst = [
        x[0] for x in enumerate(peak_to_trough_diff) if x[1] < 0.2
    ]  #defines a burst as 0.2 or larger.

    bin_ms = [x * 1000 for x in bin_s]
    binpt_ind = [int(floor(x / lgbinwidth)) for x in bin_s]

    #for moving threshold only
    thresh_plot = []
    thresh_plot = [dist_thresh[x] for x in binpt_ind]

    #for static threshold
    #thresh_plot=[dist_thresh]*len(bin_ms)
    #
    #
    #    bin_s=[((inc_past_thresh-dec_past_thresh)/2+dec_past_thresh) for inc_past_thresh, dec_past_thresh in zip(inc_past_thresh,dec_past_thresh)]
    #    bin_ms=[x*1000 for x in bin_s]
    #    thresh_plot=[]
    #    binpt_ind=[int(floor(x/lgbinwidth)) for x in bin_s]
    #    thresh_plot=[dist_thresh[x] for x in binpt_ind]
    #
    binpts = xrange(int(lgbinwidth * 1000 / 2),
                    int(numlgbins * lgbinwidth * 1000), int(lgbinwidth * 1000))
    totspkhist_list = totspkhist.tolist(
    )  #[val for subl in totspkhist for val in subl]

    #find first index after transient to see if have enough bins to do stats
    bin_ind_no_trans = bisect.bisect(bin_ms, transient)
    intrabin_ind_no_trans = bisect.bisect(intraburst_bins, transient /
                                          1000)  #transient to seconds
    if intrabin_ind_no_trans % 2 != 0:  #index must be even since format is ind0=start_bin, ind1=end_bin, ind2=start_bin, .... .
        intrabin_ind_no_trans += 1
    interbin_ind_no_trans = bisect.bisect(interburst_bins, transient / 1000)
    if interbin_ind_no_trans % 2 != 0:
        interbin_ind_no_trans += 1

    return [
        bin_s, bin_ms, binpts, totspkhist, totspkdist_smooth, dist_thresh,
        totspkhist_list, thresh_plot, binpt_ind, lgbinwidth, numlgbins,
        intraburst_bins, interburst_bins, intraburst_bins_ms,
        interburst_bins_ms, intraburst_time_ms, interburst_time_ms,
        num_intraburst_bins, num_interburst_bins, bin_ind_no_trans,
        intrabin_ind_no_trans, interbin_ind_no_trans
    ]
예제 #42
0
    def __process_results__(self):
        lines = []
        if len(self.experiments) == 0:
            lines.append("no experiments to process")
            return

        # burnin is the same for all chains
        burnin = self.experiments[0].mcmc_chain.mcmc_params.burnin

        quantiles = zeros((len(self.experiments), len(self.ref_quantiles)))
        norm_of_means = zeros(len(self.experiments))
        acceptance_rates = zeros(len(self.experiments))
        #         ess_0 = zeros(len(self.experiments))
        #         ess_1 = zeros(len(self.experiments))
        #         ess_minima = zeros(len(self.experiments))
        #         ess_medians = zeros(len(self.experiments))
        #         ess_maxima = zeros(len(self.experiments))
        times = zeros(len(self.experiments))

        for i in range(len(self.experiments)):
            burned_in = self.experiments[i].mcmc_chain.samples[burnin:, :]

            # use precomputed quantiles if they match with the provided ones
            if hasattr(self.experiments[i], "ref_quantiles") and \
               hasattr(self.experiments[i], "quantiles") and \
               allclose(self.ref_quantiles, self.experiments[i].ref_quantiles):
                quantiles[i, :] = self.experiments[i].quantiles
            else:
                try:
                    quantiles[i, :] = self.experiments[i].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(\
                                      burned_in, self.ref_quantiles)
                except NotImplementedError:
                    print "skipping quantile computations, distribution does", \
                          "not support it."

            # quantiles should be about average error rather than average quantile
            quantiles[i, :] = abs(quantiles[i, :] - self.ref_quantiles)

            dim = self.experiments[
                i].mcmc_chain.mcmc_sampler.distribution.dimension
            norm_of_means[i] = norm(mean(burned_in, 0))
            acceptance_rates[i] = mean(
                self.experiments[i].mcmc_chain.accepteds[burnin:])

            # dump burned in samples to disc
            # sample_filename=self.experiments[0].experiment_dir + self.experiments[0].name + "_burned_in.txt"
            # savetxt(sample_filename, burned_in)

            # store minimum ess for every experiment
            #ess_per_covariate = asarray([RCodaTools.ess_coda(burned_in[:, cov_idx]) for cov_idx in range(dim)])
            #             ess_per_covariate = asarray([0 for _ in range(dim)])
            #             ess_0=ess_per_covariate[0]
            #             ess_1=ess_per_covariate[1]
            #             ess_minima[i] = min(ess_per_covariate)
            #             ess_medians[i] = median(ess_per_covariate)
            #             ess_maxima[i] = max(ess_per_covariate)

            # save chain time needed
            ellapsed = self.experiments[i].mcmc_chain.mcmc_outputs[0].times
            times[i] = int(round(sum(ellapsed)))

        mean_quantiles = mean(quantiles, 0)
        std_quantiles = std(quantiles, 0)

        sqrt_num_trials = sqrt(len(self.experiments))

        # print median kernel width sigma
        #sigma=GaussianKernel.get_sigma_median_heuristic(burned_in.T)
        #lines.append("median kernel sigma: "+str(sigma))

        lines.append("quantiles:")
        for i in range(len(self.ref_quantiles)):
            lines.append(
                str(mean_quantiles[i]) + " +- " +
                str(std_quantiles[i] / sqrt_num_trials))

        lines.append("norm of means:")
        lines.append(
            str(mean(norm_of_means)) + " +- " +
            str(std(norm_of_means) / sqrt_num_trials))

        lines.append("acceptance rate:")
        lines.append(
            str(mean(acceptance_rates)) + " +- " +
            str(std(acceptance_rates) / sqrt_num_trials))

        #         lines.append("ess dimension 0:")
        #         lines.append(str(mean(ess_0)) + " +- " + str(std(ess_0)/sqrt_num_trials))
        #
        #         lines.append("ess dimension 1:")
        #         lines.append(str(mean(ess_1)) + " +- " + str(std(ess_1)/sqrt_num_trials))
        #
        #         lines.append("minimum ess:")
        #         lines.append(str(mean(ess_minima)) + " +- " + str(std(ess_minima)/sqrt_num_trials))
        #
        #         lines.append("median ess:")
        #         lines.append(str(mean(ess_medians)) + " +- " + str(std(ess_medians)/sqrt_num_trials))
        #
        #         lines.append("maximum ess:")
        #         lines.append(str(mean(ess_maxima)) + " +- " + str(std(ess_maxima)/sqrt_num_trials))

        lines.append("times:")
        lines.append(
            str(mean(times)) + " +- " + str(std(times) / sqrt_num_trials))

        # mean as a function of iterations, normalised by time
        step = round(
            (self.experiments[0].mcmc_chain.mcmc_params.num_iterations -
             burnin) / 5)
        iterations = arange(
            self.experiments[0].mcmc_chain.mcmc_params.num_iterations - burnin,
            step=step)

        running_means = zeros(len(iterations))
        running_errors = zeros(len(iterations))
        for i in arange(len(iterations)):
            # norm of mean of chain up
            norm_of_means_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(
                    burnin + iterations[i] + 1 + step), :]
                norm_of_means_yet[j] = norm(mean(samples_yet, 0))

            running_means[i] = mean(norm_of_means_yet)
            error_level = 1.96
            running_errors[i] = error_level * std(norm_of_means_yet) / sqrt(
                len(norm_of_means_yet))

        ioff()
        figure()
        plot(iterations, running_means * mean(times))
        fill_between(iterations, (running_means - running_errors)*mean(times), \
                     (running_means + running_errors)*mean(times), hold=True, color="gray")

        # make sure path to save exists
        try:
            os.makedirs(self.experiments[0].experiment_dir)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise

        savefig(self.experiments[0].experiment_dir + self.experiments[0].name +
                "_running_mean.png")
        close()

        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_Y.txt", \
                running_means*mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_mean_errors.txt", \
                running_errors*mean(times))

        # dont produce quantile convergence plots here for now
        """# quantile convergence of a single one
        desired_quantile=0.5
        running_quantiles=zeros(len(iterations))
        running_quantile_errors=zeros(len(iterations))
        for i in arange(len(iterations)):
            quantiles_yet = zeros(len(self.experiments))
            for j in range(len(self.experiments)):
                samples_yet = self.experiments[j].mcmc_chain.samples[burnin:(burnin + iterations[i] + 1 + step), :]
                
                # just compute one quantile for now
                quantiles_yet[j]=self.experiments[j].mcmc_chain.mcmc_sampler.distribution.emp_quantiles(samples_yet, \
                                                                                          array([desired_quantile]))
                quantiles_yet[j]=abs(quantiles_yet[j]-desired_quantile)
            running_quantiles[i] = mean(quantiles_yet)
            error_level = 1.96
            running_quantile_errors[i] = error_level * std(quantiles_yet) / sqrt(len(quantiles_yet))
        
        
        ioff()
        figure()
        plot(iterations, running_quantiles*mean(times))
        fill_between(iterations, (running_quantiles - running_quantile_errors)*mean(times), \
                     (running_quantiles + running_quantile_errors)*mean(times), hold=True, color="gray")
        
        plot([iterations.min(),iterations.max()], [desired_quantile*mean(times) for _ in range(2)])
        
        title(str(desired_quantile)+"-quantile convergence")
        savefig(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile.png")
        close()
        
        # also store plot X and Y
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_X.txt", \
                iterations)
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_Y.txt", \
                running_quantiles*mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_errors.txt", \
                running_quantile_errors*mean(times))
        savetxt(self.experiments[0].experiment_dir + self.experiments[0].name + "_running_quantile_reference.txt", \
                [desired_quantile*mean(times)])
        """
        # add latex table line
        #         latex_lines = []
        #         latex_lines.append("Sampler & Acceptance & ESS2 & Norm(mean) & ")
        #         for i in range(len(self.ref_quantiles)):
        #             latex_lines.append('%.1f' % self.ref_quantiles[i] + "-quantile")
        #             if i < len(self.ref_quantiles) - 1:
        #                 latex_lines.append(" & ")
        #         latex_lines.append("\\\\")
        #         lines.append("".join(latex_lines))
        #
        #         latex_lines = []
        #         latex_lines.append(self.experiments[0].mcmc_chain.mcmc_sampler.__class__.__name__)
        #         latex_lines.append('$%.3f' % mean(acceptance_rates) + " \pm " + '%.3f$' % (std(acceptance_rates)/sqrt_num_trials))
        #         latex_lines.append('$%.3f' % mean(norm_of_means) + " \pm " + '%.3f$' % (std(norm_of_means)/sqrt_num_trials))
        #         for i in range(len(self.ref_quantiles)):
        #             latex_lines.append('$%.3f' % mean_quantiles[i] + " \pm " + '%.3f$' % (std_quantiles[i]/sqrt_num_trials))
        #
        #
        #         lines.append(" & ".join(latex_lines) + "\\\\")

        return lines
예제 #43
0
파일: try2.py 프로젝트: w20ss11/python_ml
print('-------main-----------')
def main(dataset,labels):
    bool=judgelabelSame(dataset)  # @ReservedAssignment
    if bool :
        print(dataset[0][-1])
    else:
        feat=chooseMaxGain(dataset, labels)
        subdatasets,sublabels=splitdataset(dataset, feat)
        for subdataset in subdatasets:
            for sublabel in sublabels:
                 main(subdataset, sublabel)
print('---------------------------')
dataset=[[1,1,'yes'],
         [1,1,'yes'],
         [1,0,'no'],
         [0,1,'no'],
         [0,1,'no']]
labels=['no surfacing','flippers']

id=chooseMaxGain(dataset, labels)
subdatasets,sublabels=splitdataset(dataset, labels, id)
print('subdatasets:',subdatasets)
print('sublabels:',sublabels)
for i in arange(len(subdatasets)):
    if not judgelabelSame(subdatasets[i]):
        id=chooseMaxGain(subdatasets[i], sublabels)
        subdatasets,sublabels=splitdataset(dataset, labels, id)
        for j in arange(len(subdatasets)):
            if not judgelabelSame(subdatasets[i]):
                print(subdatasets,sublabels)
        
예제 #44
0
#print('y:',y)
Xi=numpy.array([8.19,2.72,6.39,8.71,4.7,2.66,3.78])
Yi=numpy.array([7.01,2.78,6.47,6.71,4.1,4.23,4.05])

w=numpy.array([100,2],dtype='float')
#------------leastsq------------------------
"""
res=scipy.optimize.leastsq(error,w,args=(Xi,Yi,100))
print('res:',res[0])

plt.scatter(Xi,Yi,color='red')
x=linspace(0,10,100,dtype=int)
w=res[0]
plt.plot(x,fun(w,x),'--')
plt.show()""" #res: [ 0.61349535  1.79409255]
#-------------mine-------------------------

alpha=0.03
for i in arange(200):
    z=fun(w,Xi)
    print('error(w, Xi, Yi, i):',error(w, Xi, Yi, i))
    w[0]=w[0]-alpha*sum(error(w, Xi, Yi, i)*Xi)/7
    w[1]=w[1]-alpha*sum(error(w, Xi, Yi, i))/7
    print('w iter:',w)
print('w:',w)
plt.scatter(Xi,Yi,color='red')
x=linspace(0,10,100,dtype=int)
plt.plot(x,fun(w,x),'--')
plt.show()
#log: 0.03 w: [ 0.59562777  1.90639941]
#-----------------------------------------
예제 #45
0
    #print('iter:',iter)
    return fun(w,Xi)-Yi

x1=linspace(6,20,10,dtype=int)
x2=linspace(0,10,10,dtype=int)
x3=linspace(20,31,10,dtype=int)
x=numpy.array([x1,x2,x3])
print('x:',x)
w=numpy.array([15,5,8,10],dtype=float)
y=fun(w,x)
print('y:',y,type(y))

#--------leastsq---------------
#res=scipy.optimize.leastsq(error, [1,1,1,1], args=(x,y,300))
#print('res:',res[0]) #res: [ 15.   5.   8.  10.]

#---------mine-------------------
w=[1,1,1,1]
alpha=0.003
z=fun(w,x)
print('sum((z-y)',sum((z-y)))
for i in arange(300):
    z=fun(w,x)
    print('error:',error(w, x, y, i))
    print('iter:',i)
    w[0]=w[0]-alpha*sum(error(w, x, y, i))/10
    w[1]=w[1]-alpha*sum(error(w, x, y, i)*x1)/10
    w[2]=w[2]-alpha*sum(error(w, x, y, i)*x2)/10
    w[3]=w[3]-alpha*sum(error(w, x, y, i)*x3)/10
    print('w:',w)
예제 #46
0
 def __init__(self, folders, ref_quantiles=arange(0.1, 1, 0.1)):
     ExperimentAggregator.__init__(self, folders)
     self.ref_quantiles = ref_quantiles
예제 #47
0
'''
import os
from numpy.ma.core import arange
def rename(path):
    n = 1;
    for file in os.listdir(path):
        print(file)
        newname = ''+str(n)+'.png'
        nam = os.path.join(path,file)
        if 'ref' in nam :
            print(nam)
        else :
            os.rename(os.path.join(path,file),os.path.join(path,newname))
        print(file,'ok')
        n = n+1
for j in arange(7):
    j=j+3
    path = 'Z:\HDD\dianzikeda_data\shot\class0'+str(j)
    rename(path)
    
#rename("Z:\HDD\dianzikeda_data\shot\class10")

"""rename("X:\\s_class1\\P5")
rename("X:\\s_class2\\P5")
rename("X:\\s_class3\\P5")
rename("X:\\s_class4\\P5")
rename("X:\\s_class5\\P5")"""


"""for i in arange(7):
    i=i+1
예제 #48
0
t1 = time.time()
sumW = zeros(n)

for _ in range(nrRuns):
    sim = simulateLindleyEfficient(lam, mu, n)
    sumW += sim
meanW = sumW / nrRuns
t2 = time.time()
print("Simulation time: %f seconds" % (t2 - t1))

# theoretical steady-state mean waiting time
EW = lam / (mu * (mu - lam))

plt.figure()
plt.plot(arange(1, n + 1), meanW, 'b')
plt.hlines(xmin=0, xmax=n, y=EW, color='red')


# 1. Regular Confidence interval for the mean waiting time
# M is the number of runs,
# N is the number of customers per run,
# k is the length of the warm-up interval (in this case, the number of customers to disregard)
def ciMultipleRuns(M, N, k):
    sumW = 0
    sumW2 = 0
    for _ in range(M):
        sim = simulateLindleyEfficient(lam, mu, N)
        meanWrun = mean(sim[k:N])
        sumW += meanWrun
        sumW2 += meanWrun**2