def analyze(odd_ecc,correct):
    def func(pars):
        a,b,c = pars
        return a*(x**2) + b*x + c 

    def errfunc(pars):
        return y-func(pars)
       #Analysis:
    #Bin the eccentricities: 
        a = np.floor(odd_ecc)
        eccs_used = np.unique(a)
    #Initialize counters: 
        b = np.zeros(len(eccs_used))
        c = np.zeros(len(eccs_used))

    #Loop over the trials and add the correct to one counter and the number of
    #trials to the other: 
        for i in xrange(len(correct)):
            idx = np.where(eccs_used==np.floor(odd_ecc[i]))
            b[idx]+=correct[i]
            c[idx]+=1

        p_correct = b/c

        for i,p in enumerate(p_correct):
            plt.plot(eccs_used[i],p,'o',color='b',markersize=c[i])

    x = []
    y = []

    for i,this_ecc in enumerate(eccs_used):
        x = np.hstack([x,c[i]*[this_ecc]])    
        y = np.hstack([y,c[i]*[p_correct[i]]])
Esempio n. 2
0
def test_dofs(vector_array):
    v = vector_array
    np.random.seed(len(v) + 24 + v.dim)
    for ind in valid_inds(v):
        c = v.copy()
        dofs = c[ind].dofs(np.array([], dtype=np.int))
        assert isinstance(dofs, np.ndarray)
        assert dofs.shape == (v.len_ind(ind), 0)

        c = v.copy()
        dofs = c[ind].dofs([])
        assert isinstance(dofs, np.ndarray)
        assert dofs.shape == (v.len_ind(ind), 0)

        if v.dim > 0:
            for count in (1, 5, 10):
                c_ind = np.random.randint(0, v.dim, count)
                c = v.copy()
                dofs = c[ind].dofs(c_ind)
                assert dofs.shape == (v.len_ind(ind), count)
                c = v.copy()
                dofs2 = c[ind].dofs(list(c_ind))
                assert np.all(dofs == dofs2)
                c = v.copy()
                c.scal(3.)
                dofs2 = c[ind].dofs(c_ind)
                assert np.allclose(dofs * 3, dofs2)
                c = v.copy()
                dofs2 = c[ind].dofs(np.hstack((c_ind, c_ind)))
                assert np.all(dofs2 == np.hstack((dofs, dofs)))
                if hasattr(v, 'data'):
                    assert np.all(dofs == indexed(v.data, ind)[:, c_ind])
Esempio n. 3
0
def moving_something(_fun, tseries, period, day_rs=16, is_days=True):
    """
    Applies a function to a moving window of the time-series:
    ft_ = function([ f(t-N), f(t). f(t+N)])
    """
    # if the time-series is at a day-time step, update the window to a step-size of 16 days
    if is_days:
        p0 = period*day_rs
    else:
        p0 = period

    # find upper and lower bounds of the moving window
    half = p0//2
    tlen = len(tseries)

    twin = [0]*tlen
    for im in range(tlen):
        # find the something for the window that satisfy the edge conditions
        if im < half:
            # fold back onto the end of the time-series
            twin[im] = _fun(np.hstack([tseries[tlen-(half-im):tlen],\
                                        tseries[0:im+half]]))
        elif im > tlen-half:
            # fold back into the beginning of the time-series
            twin[im] = _fun(np.hstack([tseries[im-half:tlen],\
                                        tseries[0:half-(tlen-im)]]))
        else:
            twin[im] = _fun(tseries[im-half:im+half])

    return twin
def get_roc_score(edges_pos, edges_neg, score_matrix, apply_sigmoid=False):

    # Edge case
    if len(edges_pos) == 0 or len(edges_neg) == 0:
        return (None, None, None)

    # Store positive edge predictions, actual values
    preds_pos = []
    pos = []
    for edge in edges_pos:
        if apply_sigmoid == True:
            preds_pos.append(sigmoid(score_matrix[edge[0], edge[1]]))
        else:
            preds_pos.append(score_matrix[edge[0], edge[1]])
        pos.append(1) # actual value (1 for positive)
        
    # Store negative edge predictions, actual values
    preds_neg = []
    neg = []
    for edge in edges_neg:
        if apply_sigmoid == True:
            preds_neg.append(sigmoid(score_matrix[edge[0], edge[1]]))
        else:
            preds_neg.append(score_matrix[edge[0], edge[1]])
        neg.append(0) # actual value (0 for negative)
        
    # Calculate scores
    preds_all = np.hstack([preds_pos, preds_neg])
    labels_all = np.hstack([np.ones(len(preds_pos)), np.zeros(len(preds_neg))])
    roc_score = roc_auc_score(labels_all, preds_all)
    # roc_curve_tuple = roc_curve(labels_all, preds_all)
    ap_score = average_precision_score(labels_all, preds_all)
    
    # return roc_score, roc_curve_tuple, ap_score
    return roc_score, ap_score
Esempio n. 5
0
def torgerson(distances, n_components=2):
    """
    Perform classical mds (Torgerson scaling).

    ..note ::
        If the distances are euclidean then this is equivalent to projecting
        the original data points to the first `n` principal components.

    """
    distances = np.asarray(distances)
    assert distances.shape[0] == distances.shape[1]
    N = distances.shape[0]
    # O ^ 2
    D_sq = distances ** 2

    # double center the D_sq
    rsum = np.sum(D_sq, axis=1, keepdims=True)
    csum = np.sum(D_sq, axis=0, keepdims=True)
    total = np.sum(csum)
    D_sq -= rsum / N
    D_sq -= csum / N
    D_sq += total / (N ** 2)
    B = np.multiply(D_sq, -0.5, out=D_sq)

    U, L, _ = np.linalg.svd(B)
    if n_components > N:
        U = np.hstack((U, np.zeros((N, n_components - N))))
        L = np.hstack((L, np.zeros((n_components - N))))
    U = U[:, :n_components]
    L = L[:n_components]
    D = np.diag(np.sqrt(L))
    return np.dot(U, D)
Esempio n. 6
0
    def __generateInitialSpline(self, splineOrder, timeOffsetPadding, numberOfKnots = None, framerate = None):
        poseSpline = bsplines.BSplinePose(splineOrder, sm.RotationVector())

        # Get the observation times.
        times = np.array([observation.time().toSec() for observation in self.__observations ])
        # get the pose values of the initial transformations at observation time
        curve = np.matrix([ poseSpline.transformationToCurveValue( observation.T_t_c().T() ) for observation in self.__observations]).T
        # make sure all values are well defined
        if np.isnan(curve).any():
            raise RuntimeError("Nans in curve values")
            sys.exit(0)
        # Add 2 seconds on either end to allow the spline to slide during optimization
        times = np.hstack((times[0] - (timeOffsetPadding * 2.0), times, times[-1] + (timeOffsetPadding * 2.0)))
        curve = np.hstack((curve[:,0], curve, curve[:,-1]))

        self.__ensureContinuousRotationVectors(curve)

        seconds = times[-1] - times[0]

        # fixed number of knots
        if (numberOfKnots is not None):
            knots = numberOfKnots
        # otherwise with framerate estimate
        else:
            knots = int(round(seconds * framerate/3))

        print
        print "Initializing a pose spline with %d knots (%f knots per second over %f seconds)" % ( knots, 100, seconds)
        poseSpline.initPoseSplineSparse(times, curve, knots, 1e-4)

        return poseSpline
Esempio n. 7
0
def gen_coastline(lon, lat, bathy, depth=0):
    """
    Given lon, lat, and bathymetry, generate vectors of line segments
    of the coastline. This can be exported to matlab (via savemat) to be
    used with the 'editmask' routine for creating grid masks.

    Input
    -----
    lon : array,
        longitudes of bathymetry locations
    lat : array,
        latitudes of bathymetry locations
    bathy : array,
        bathymetry (negative for ocean, positive for land) values
    depth : float,
        depth to use as the definition of the coast

    Returns
    -------
    lon : ndarray,
        vector of coastlines, separated by nan (matlab-style)
    lat : ndarray,
        vector of coastlines, separated by nan (matlab-style)
    """
    CS = plt.contour(lon, lat, bathy, [depth - 0.25, depth + 0.25])
    lon = list()
    lat = list()
    for col in CS.collections:
        for path in col.get_paths():
            lon.append(path.vertices[:, 0])
            lon.append(np.nan)
            lat.append(path.vertices[:, 1])
            lat.append(np.nan)
    return (np.hstack(lon), np.hstack(lat))
Esempio n. 8
0
	def shift(data,shiftdirection = "center"):
		numshifts = {"right":1,"left":1,"up":1,"down":1}
		
		shiftfuncs = {
			"right": lambda arr:np.hstack((zerocol(outarr),outarr)),
			"left": lambda arr: np.hstack((outarr,zerocol(outarr))),
			"up": lambda arr: np.vstack((outarr,zerorow(outarr))),
			"down": lambda arr: np.vstack((zerorow(outarr),outarr))
		}

		directionpairs =  \
			{"right":"left","left":"right","up":"down","down":"up"}
				
		if shiftdirection != "center":
			numshifts[shiftdirection] += 1
			numshifts[directionpairs[shiftdirection]] -= 1

		outarr = cp.deepcopy(data)
		zerocol = lambda arr: np.zeros((arr.shape[0],1))
		zerorow = lambda arr: np.zeros((1,arr.shape[1]))

		for direction in numshifts.keys():
			for numtimes in xrange(numshifts[direction]):
				outarr = shiftfuncs[direction](outarr)

		return outarr
Esempio n. 9
0
    def __mul__(self, df):
        """
        extract and stack  poles and zeros

        TODO : handling simplification
        """

        b1 = self.b
        a1 = self.a
        b2 = df.b
        a2 = df.a

        pb1 = np.poly1d(b1)
        pa1 = np.poly1d(a1)
        pb2 = np.poly1d(b2)
        pa2 = np.poly1d(a2)

        rpb1 = pb1.r
        rpb2 = pb2.r
        rpa1 = pa1.r
        rpa2 = pa2.r

        F = DF()
        F.p = np.hstack((rpa1, rpa2))
        F.z = np.hstack((rpb1, rpb2))

        F.simplify()

        return F
Esempio n. 10
0
    def test_07_01_make_ijv_outlines(self):
        np.random.seed(70)
        x = cpo.Objects()
        ii, jj = np.mgrid[0:10, 0:20]
        masks = [(ii - ic) ** 2 + (jj - jc) ** 2 < r ** 2
                 for ic, jc, r in ((4, 5, 5), (4, 12, 5), (6, 8, 5))]
        i = np.hstack([ii[mask] for mask in masks])
        j = np.hstack([jj[mask] for mask in masks])
        v = np.hstack([[k + 1] * np.sum(mask) for k, mask in enumerate(masks)])

        x.set_ijv(np.column_stack((i, j, v)), ii.shape)
        x.parent_image = cpi.Image(np.zeros((10, 20)))
        colors = np.random.uniform(size=(3, 3)).astype(np.float32)
        image = x.make_ijv_outlines(colors)
        i1 = [i for i, color in enumerate(colors) if np.all(color == image[0, 5, :])]
        self.assertEqual(len(i1), 1)
        i2 = [i for i, color in enumerate(colors) if np.all(color == image[0, 12, :])]
        self.assertEqual(len(i2), 1)
        i3 = [i for i, color in enumerate(colors) if np.all(color == image[-1, 8, :])]
        self.assertEqual(len(i3), 1)
        self.assertNotEqual(i1[0], i2[0])
        self.assertNotEqual(i2[0], i3[0])
        colors = colors[np.array([i1[0], i2[0], i3[0]])]
        outlines = np.zeros((10, 20, 3), np.float32)
        alpha = np.zeros((10, 20))
        for i, (color, mask) in enumerate(zip(colors, masks)):
            my_outline = outline(mask)
            outlines[my_outline] += color
            alpha[my_outline] += 1
        alpha[alpha == 0] = 1
        outlines /= alpha[:, :, np.newaxis]
        np.testing.assert_almost_equal(outlines, image)
Esempio n. 11
0
def display_layer(X, filename="../images/layer.png"):
    """
    Produces an image, composed of the given N images, patches or neural network weights,
    stored in the array X. Saves it with the given filename.
    :param X: numpy array of size (NxD) — N images, patches or neural network weights
    :param filename: a string, the name of the produced file
    :return: None
    """
    if not isinstance(X, np.ndarray):
        raise TypeError("'X' must be a numpy array")
    N, D = X.shape
    d = get_reshaped_image_size(D)

    if N == 1:
        return X.reshape(d, d, 3)
    divizors = [n for n in range(1, N) if N % n == 0]
    im_sizes = divizors[int(len(divizors) / 2)], int(N / divizors[int(len(divizors) / 2)])
    for i in range(im_sizes[0]):
        # img_row = np.hstack((img_row, np.zeros((d, 1, 3))))
        img_row = np.hstack((np.zeros((d, 1, 3)), np.array(X[i * im_sizes[0], :].reshape(d, d, 3))))
        img_row = np.hstack((img_row, np.zeros((d, 1, 3))))
        for j in range(1, im_sizes[1]):
            img_row = np.hstack((img_row, X[i * im_sizes[1] + j, :].reshape(d, d, 3)))
            img_row = np.hstack((img_row, np.zeros((d, 1, 3))))
        if i == 0:
            img = img_row
        else:
            img = np.vstack((img, img_row))
        img = np.vstack((img, np.zeros((1, img.shape[1], 3))))
    img = np.vstack((np.zeros((1, img.shape[1], 3)), img))
    imsave(filename, img)
    return img
Esempio n. 12
0
    def evaluate_transformation_matrix(self, dynamic_values, constant_values):
        """Returns the numerical transformation matrices for each time step.

        Parameters
        ----------
        dynamic_values : array_like, shape(m,) or shape(n, m)
            The m state values for each n time step.
        constant_values : array_like, shape(p,)
            The p constant parameter values of the system.

        Returns
        -------
        transform_matrix : numpy.array, shape(n, 4, 4)
            A 4 x 4 transformation matrix for each time step.

        """
        #If states is instance of numpy array, well and good.
        #else convert it to one:

        states = np.array(dynamic_values)
        if len(states.shape) > 1:
            n = states.shape[0]
            new = np.zeros((n, 4, 4))
            for i, time_instance in enumerate(states):
                args = np.hstack((time_instance, constant_values))
                new[i, :, :] = self._numeric_transform(*args)
        else:
            n = 1
            args = np.hstack((states, constant_values))
            new = self._numeric_transform(*args)

        self._visualization_matrix = new.reshape(n, 16)
        return self._visualization_matrix
Esempio n. 13
0
File: base.py Progetto: TKCen/nupic
  def getScalars(self, inputData):
    """
    Returns a numpy array containing the sub-field scalar value(s) for
    each sub-field of the inputData. To get the associated field names for each of
    the scalar values, call getScalarNames().

    For a simple scalar encoder, the scalar value is simply the input unmodified.
    For category encoders, it is the scalar representing the category string
    that is passed in. For the datetime encoder, the scalar value is the
    the number of seconds since epoch.

    The intent of the scalar representation of a sub-field is to provide a
    baseline for measuring error differences. You can compare the scalar value
    of the inputData with the scalar value returned from topDownCompute() on a
    top-down representation to evaluate prediction accuracy, for example.

    @param inputData The data from the source. This is typically a object with
                 members
    @returns array of scalar values
    """

    retVals = numpy.array([])

    if self.encoders is not None:
      for (name, encoder, offset) in self.encoders:
        values = encoder.getScalars(self._getInputValue(inputData, name))
        retVals = numpy.hstack((retVals, values))
    else:
      retVals = numpy.hstack((retVals, inputData))

    return retVals
Esempio n. 14
0
    def _train(self, samples):
        """Perform network training.

        Parameter
        ---------
        samples : array-like
          Used for unsupervised training of the SOM.
        """
        # XXX initialize with clever default, e.g. plain of first two PCA
        # components
        self._K = np.random.standard_normal(tuple(self.kshape) + (samples.shape[1],))

        # units weight vector deltas for batch training
        # (height x width x #features)
        unit_deltas = np.zeros(self._K.shape, dtype='float')

        # precompute distance kernel between elements in the Kohonen layer
        # that will remain constant throughout the training
        # (just compute one quadrant, as the distances are symmetric)
        # XXX maybe do other than squared Euclidean?
        dqd = np.fromfunction(lambda x, y: (x**2 + y**2)**0.5,
                             self.kshape, dtype='float')

        # for all iterations
        for it in xrange(1, self.niter + 1):
            # compute the neighborhood impact kernel for this iteration
            # has to be recomputed since kernel shrinks over time
            k = self._compute_influence_kernel(it, dqd)

            # for all training vectors
            for s in samples:
                # determine closest unit (as element coordinate)
                b = self._get_bmu(s)
                # train all units at once by unfolding the kernel (from the
                # single quadrant that is precomputed), cutting it to the
                # right shape and simply multiply it to the difference of target
                # and all unit weights....
                infl = np.vstack((
                        np.hstack((
                            # upper left
                            k[b[0]:0:-1, b[1]:0:-1],
                            # upper right
                            k[b[0]:0:-1, :self.kshape[1] - b[1]])),
                        np.hstack((
                            # lower left
                            k[:self.kshape[0] - b[0], b[1]:0:-1],
                            # lower right
                            k[:self.kshape[0] - b[0], :self.kshape[1] - b[1]]))
                               ))
                unit_deltas += infl[:,:,np.newaxis] * (s - self._K)

            # apply cumulative unit deltas
            self._K += unit_deltas

            if __debug__:
                debug("SOM", "Iteration %d/%d done: ||unit_deltas||=%g" %
                      (it, self.niter, np.sqrt(np.sum(unit_deltas **2))))

            # reset unit deltas
            unit_deltas.fill(0.)
Esempio n. 15
0
    def save(self,filename):
        num_objs = len(self.objects)
        data_size = 0
        
        desc = []
        data = []
        for obj in self.objects:
            if isinstance(obj,self.scalars):
                desc.append(0)
                data.append(obj)
                data_size += 1
            else:
                assert(isinstance(obj,np.ndarray))
                desc.append(len(obj.shape))
                desc.append(obj.shape)
                
                data.append(obj.flatten(order='F'))
                data_size += np.prod(obj.shape)

        desc = np.hstack(desc)
        header_size = 3 + desc.size
        output = np.hstack([num_objs,header_size,data_size]
                           + [desc]
                           + data).astype(np.double)
        assert((header_size + data_size,) == output.shape)
        output.tofile(filename)
Esempio n. 16
0
    def get_subgroup(self, objpath):
        """
        Load a set of image frames from a given subgroup, sorted by the
        ls_z_measured value.
        Returns a pair of (imgdata, metadata).
        """
        node0 = self.h5file.get_node('/', objpath + '/0')

        rowlen = 8
        imgrows = []
        metadata = {'size_x': int(node0.shape[0]), 'size_y': int(node0.shape[1]), 'framedata': []}
        j = 0

        for (i, node) in sorted(self.h5file.get_node('/', objpath)._v_children.items(), key = lambda i: i[1].attrs['ls_z_measured']):
            if len(imgrows) <= j/rowlen:
                imgrows.append(node.read())
            else:
                imgrows[j/rowlen] = numpy.hstack((imgrows[j/rowlen], node.read()))
            metadata['framedata'].append(
                    {'t': int(node.attrs['ls_time']),
                     'n': int(node.attrs['ls_n']),
                     'z_r': float(node.attrs['ls_z_request']),
                     'z': float(node.attrs['ls_z_measured'])})
            j += 1

        # Fully extend the last row
        imgrows[-1] = numpy.hstack((imgrows[-1], numpy.zeros([metadata['size_y'], rowlen * metadata['size_x'] - imgrows[-1].shape[1]])))

        imgdata = numpy.vstack(imgrows)
        self.cache[objpath] = { "imgdata": imgdata, "metadata": metadata }
Esempio n. 17
0
    def _stimcorr_core(self, motionfile, intensityfile, designmatrix, cwd=None):
        """
        Core routine for determining stimulus correlation

        """
        if not cwd:
            cwd = os.getcwd()
        # read in motion parameters
        mc_in = np.loadtxt(motionfile)
        g_in = np.loadtxt(intensityfile)
        g_in.shape = g_in.shape[0], 1
        dcol = designmatrix.shape[1]
        mccol = mc_in.shape[1]
        concat_matrix = np.hstack((np.hstack((designmatrix, mc_in)), g_in))
        cm = np.corrcoef(concat_matrix, rowvar=0)
        corrfile = self._get_output_filenames(motionfile, cwd)
        # write output to outputfile
        file = open(corrfile, 'w')
        file.write("Stats for:\n")
        file.write("Stimulus correlated motion:\n%s\n" % motionfile)
        for i in range(dcol):
            file.write("SCM.%d:" % i)
            for v in cm[i, dcol + np.arange(mccol)]:
                file.write(" %.2f" % v)
            file.write('\n')
        file.write("Stimulus correlated intensity:\n%s\n" % intensityfile)
        for i in range(dcol):
            file.write("SCI.%d: %.2f\n" % (i, cm[i, -1]))
        file.close()
Esempio n. 18
0
def test_elastic_basis_pursuit():
    """
    
    """
    
    xx2d = np.array(np.meshgrid(np.arange(0,50,1),
                                np.arange(0,50,1)))
    bounds = [[0,50], [0,50], [0, None], [0, None]]
    mean1 = [20, 20]
    sigma1 = [10, 10]
    params1 = np.hstack([mean1, sigma1])

    kernel1 = ebp.gaussian_kernel(xx2d, params1)
    mean2 = [30, 40]
    sigma2 = [10, 50]
    params2 = np.hstack([mean2, sigma2])
    kernel2 = ebp.gaussian_kernel(xx2d, params2)
    beta = [0.3, 0.7]
    signal = (beta[0] * kernel1 + beta[1] * kernel2).ravel()
    initial_theta = np.hstack([np.mean(xx2d, axis=(-1, -2)), 1, 1])

    theta, err, r = ebp.elastic_basis_pursuit(xx2d.reshape(-1, signal.shape[0]),
                                              signal,
                                              ebp.leastsq_oracle,
                                              ebp.gaussian_kernel,
                                              initial_theta=initial_theta,
                                              bounds=bounds,
                                              max_iter=1000,
                                              beta_tol=10e-6)
Esempio n. 19
0
def test_kernel_error():
    """

    """
    for xx2d in [np.array(np.meshgrid(np.arange(-100, 100, 5),
                                      np.arange(-100, 100, 5))),

                 np.array(np.meshgrid(np.arange(-100, 100, 5),
                                      np.arange(-100, 100, 5))).reshape(2, -1)]:

        mean1 = [20, 20]
        sigma1 = [10, 10]
        params1 = np.hstack([mean1, sigma1])
        mean2 = [30, 40]
        sigma2 = [10, 50]
        params2 = np.hstack([mean2, sigma2])

        params = [params1, params2]
        betas = [0.3, 0.7]
        
        y = ebp.mixture_of_kernels(xx2d, betas, params,
                                   ebp.gaussian_kernel)

        err = ebp.kernel_err(y, xx2d, betas, params, ebp.gaussian_kernel)

        npt.assert_almost_equal(err, np.zeros(err.shape))
Esempio n. 20
0
 def compute_sketch(self):
     start_time = time.time()
     if self.sketch is not None:
         return self.sketch
     mat_b = np.zeros([self.l + self.b_size, self.m])
     # compute zero valued row list
     zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
     zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
     # repeat inserting each row of matrix A 
     for i in range(0, self.mat.shape[0]):
         # insert a row into matrix B
         mat_b[zero_rows[0], :] = self.mat[i, :]
         # remove zero valued row from the list
         zero_rows.remove(zero_rows[0])
         # if there is no more zero valued row
         if len(zero_rows) == 0:
             # compute SVD of matrix B, we want to find the first l
             self._sketch_func(mat_b)
             # update the zero valued row list
             zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
             zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
     # why do we need this here? 
     # do we need to do a sketch one last time at the end? 
     self._sketch_func(mat_b)
     # get rid of extra non-zero rows when we return 
     self.sketch = mat_b[:self.l, :]
     self.sketching_time = time.time() - start_time
     return self.sketch
Esempio n. 21
0
 def compute_sketch(self):
     start_time = time.time()
     if self.sketch is not None:
         return self.sketch
     # basically, want to init an empty csr matrix 
     if self.randomized and (self.m > 100 * self.l):
         print "using sparse sketch"
         # lets use the sparse version of randomized sketch here 
         return self.compute_sparse_sketch()
     else:
         self._sketch_func = self._fast_rand_sketch
     # what do we do differently here? we need to iterate over the nzrow_inds,
     mat_b = np.zeros([self.l + self.b_size, self.m])
     # other way: np.where(~mat_b.any(axis=1))[0]
     # zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b, axis = 1)])[0].tolist()
     zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
     zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
     # iterate through the nzrow_inds
     for i in self.nzrow_inds:
         mat_b[zero_rows[0], :] = self.mat.getrow(i).todense()
         zero_rows.remove(zero_rows[0])
         if len(zero_rows) == 0:
             print "sketching ", i
             self._sketch_func(mat_b)
             zero_rows = np.nonzero([round(s, 7) == 0.0 for s in np.sum(mat_b[:self.l, :], axis = 1)])[0]
             zero_rows = np.hstack((zero_rows, np.arange(self.l, self.l + self.b_size))).tolist()
     self._sketch_func(mat_b)
     self.sketch = mat_b[:self.l, :]
     self.sketching_time = time.time() - start_time 
     return self.sketch
Esempio n. 22
0
def dobetterstuff(inpath):
    data_files = [f for f in os.listdir(inpath) if f.endswith('.mel.npy')]
    random.shuffle(data_files)
    artists = set([f[:18] for f in data_files])
    artist_string_to_id = dict([(s,i) for i, s in enumerate(artists)])

    def get_split(datafiles___, splitpercent):
        # gen = filtered_stratified_split(datafiles___,
        #                                 sklearn.cross_validation.StratifiedShuffleSplit,
        #                                 [1] * len(datafiles___), n_iterations=1, test_size=splitpercent)
        gen = sklearn.cross_validation.ShuffleSplit(len(datafiles___), 1, splitpercent)
        for i_trs, i_tes in gen:
            return [datafiles___[i] for i in i_trs],  [datafiles___[i] for i in i_tes]

    training_files, test_files =  get_split(data_files, .2)
    training_files, validation_files = get_split(training_files, .2)

    print training_files
    print test_files
    print validation_files

    train_set_y = np.hstack([[artist_string_to_id[f[:18]]] * 129 for f in training_files])
    train_set_x = np.vstack([np.load(os.path.join(inpath, f)) for f in training_files])
    test_set_y = np.hstack([[artist_string_to_id[f[:18]]] * 129 for f in test_files])
    test_set_x = np.vstack([np.load(os.path.join(inpath, f)) for f in test_files])
    validation_set_y = np.hstack([[artist_string_to_id[f[:18]]] * 129 for f in validation_files])
    validation_set_x = np.vstack([np.load(os.path.join(inpath, f)) for f in validation_files])

    datasets = [(train_set_x, train_set_y), (validation_set_x, validation_set_y), (test_set_x, test_set_y)]
    return datasets
Esempio n. 23
0
def load_sdss_data_both_catalogs(hemi):
    lowz = load_sdss_data('lowz', hemi)
    cmass = load_sdss_data('cmass', hemi)
    ra = np.hstack([lowz['ra'],cmass['ra']])
    dec = np.hstack([lowz['dec'],cmass['dec']])
    z = np.hstack([lowz['z'],cmass['z']])        
    return {'ra':ra, 'dec':dec, 'z':z}
def sample_trajectory(M, n_states):
    # Samples trajectories from random nodes
    #  in our domain (M)
    G, W = M.get_graph_inv()
    N = G.shape[0]
    if N >= n_states:
        rand_ind = np.random.permutation(N)
    else:
        rand_ind = np.tile(np.random.permutation(N), (1, 10))
    init_states = rand_ind[0:n_states].flatten()
    goal_s = M.map_ind_to_state(M.targetx, M.targety)
    states = []
    states_xy = []
    states_one_hot = []
    # Get optimal path from graph
    g_dense = W
    g_masked = np.ma.masked_values(g_dense, 0)
    g_sparse = csr_matrix(g_dense)
    d, pred = dijkstra(g_sparse, indices=goal_s, return_predecessors=True)
    for i in range(n_states):
        path = trace_path(pred, goal_s, init_states[i])
        path = np.flip(path, 0)
        states.append(path)
    for state in states:
        L = len(state)
        r, c = M.get_coords(state)
        row_m = np.zeros((L, M.n_row))
        col_m = np.zeros((L, M.n_col))
        for i in range(L):
            row_m[i, r[i]] = 1
            col_m[i, c[i]] = 1
        states_one_hot.append(np.hstack((row_m, col_m)))
        states_xy.append(np.hstack((r, c)))
    return states_xy, states_one_hot
    def phase_step_spike_fq(self, spikes_time, full_step, nb_block, fs):
        stance_spike_fq=[]
        swing_spike_fq=[]
        for step in full_step:
            stance_block_duration = (step[1]-step[0])/nb_block
            swing_block_duration = (step[2]-step[1])/nb_block
            step_stance_count = []
            step_swing_count = []
            for i in range(nb_block):
                step_stance_count.append(0)
                step_swing_count.append(0)

            for spike_time in spikes_time:
                #if stance phase
                if step[0] < spike_time/fs < step[1]:
                    list_block = np.arange(step[0], step[1], stance_block_duration)
                    list_block = np.hstack((list_block, step[1]))
                    for i in range(nb_block):
                        if list_block[i] < spike_time/fs < list_block[i+1]:
                            step_stance_count[i] += 1
                #if swing phase
                elif step[1] < spike_time/fs < step[2]:
                    list_block = np.arange(step[1], step[2], swing_block_duration)
                    list_block = np.hstack((list_block, step[2]))
                    for i in range(nb_block):
                        if list_block[i] < spike_time/fs < list_block[i+1]:
                            step_swing_count[i] += 1
                # elif spike_time/fs > step[2]:
                #     break
            stance_spike_fq.append(np.array(step_stance_count) / stance_block_duration)
            swing_spike_fq.append(np.array(step_swing_count) / swing_block_duration)

        return stance_spike_fq, swing_spike_fq
def run_classify(X_groups_train, y_train, X_groups_validate, y_validate):
    """
    Although this function is given groups, it actually doesn't utilize the groups at all in the criterion
    """

    method_label = "gridsearch_lasso"

    X_validate = np.hstack(X_groups_validate)

    max_power = np.log(50)
    min_power = np.log(1e-4)
    lambda_guesses = np.power(np.e, np.arange(min_power, max_power, (max_power - min_power - 1e-5) / (NUM_LAMBDAS - 1)))
    print method_label, "lambda_guesses", lambda_guesses

    X_train = np.hstack(X_groups_train)
    problem_wrapper = LassoClassifyProblemWrapper(X_train, y_train, [])

    best_cost = 1e5
    best_betas = []
    best_regularization = lambda_guesses[0]

    for l1 in reversed(lambda_guesses):
        betas = problem_wrapper.solve([l1])
        current_cost, _ = testerror_logistic_grouped(X_validate, y_validate, betas)
        if best_cost > current_cost:
            best_cost = current_cost
            best_betas = betas
            best_regularization = l1
            print method_label, "best_cost so far", best_cost, "best_regularization", best_regularization
            sys.stdout.flush()

    print method_label, "best_validation_error", best_cost
    print method_label, "best lambdas:", best_regularization

    return best_betas, best_cost
 def test_fuzz(self):
     # try a bunch of crazy inputs
     rfuncs = (
             np.random.uniform,
             np.random.normal,
             np.random.standard_cauchy,
             np.random.exponential)
     ntests = 100
     for i in range(ntests):
         rfunc = random.choice(rfuncs)
         target_norm_1 = random.expovariate(1.0)
         n = random.randrange(2, 16)
         A_original = rfunc(size=(n,n))
         E_original = rfunc(size=(n,n))
         A_original_norm_1 = scipy.linalg.norm(A_original, 1)
         scale = target_norm_1 / A_original_norm_1
         A = scale * A_original
         E = scale * E_original
         M = np.vstack([
             np.hstack([A, E]),
             np.hstack([np.zeros_like(A), A])])
         expected_expm = scipy.linalg.expm(A)
         expected_frechet = scipy.linalg.expm(M)[:n, n:]
         observed_expm, observed_frechet = expm_frechet(A, E)
         assert_allclose(expected_expm, observed_expm)
         assert_allclose(expected_frechet, observed_frechet)
Esempio n. 28
0
 def offsetPlane(plane, x, y):
     """
     Takes a numpy 2D array and returns the same plane offset by x and y,
     adding rows and columns of 0 values
     """
     height, width = plane.shape
     dataType = plane.dtype
     # shift x by cropping, creating a new array of columns and stacking
     # horizontally
     if abs(x) > 0:
         newCols = zeros((height, abs(x)), dataType)
         x1 = max(0, 0 - x)
         x2 = min(width, width - x)
         crop = plane[0:height, x1:x2]
         if x > 0:
             plane = hstack((newCols, crop))
         else:
             plane = hstack((crop, newCols))
     # shift y by cropping, creating a new array of rows and stacking
     # vertically
     if abs(y) > 0:
         newRows = zeros((abs(y), width), dataType)
         y1 = max(0, 0 - y)
         y2 = min(height, height - y)
         crop = plane[y1:y2, 0:width]
         if y > 0:
             plane = vstack((newRows, crop))
         else:
             plane = vstack((crop, newRows))
     return plane
Esempio n. 29
0
    def _plot_traj(self, z, axes, units):
        """Plots spacecraft trajectory.

        Args:
            - z (``tuple``, ``list``, ``numpy.ndarray``): Decision chromosome.
            - axes (``matplotlib.axes._subplots.Axes3DSubplot``): 3D axes to use for the plot
            - units (``float``, ``int``): Length unit by which to normalise data.

        Examples:
            >>> prob.extract(pykep.trajopt.indirect_or2or).plot_traj(pop.champion_x)
        """

        # times
        t0 = pk.epoch(0)
        tf = pk.epoch(z[0])

        # Mean Anomalies
        M0 = z[1] - self.elem0[1] * np.sin(z[1])
        Mf = z[2] - self.elemf[1] * np.sin(z[2])

        elem0 = np.hstack([self.elem0[:5], [M0]])
        elemf = np.hstack([self.elemf[:5], [Mf]])

        # Keplerian points
        kep0 = pk.planet.keplerian(t0, elem0)
        kepf = pk.planet.keplerian(tf, elemf)

        # planets
        pk.orbit_plots.plot_planet(
            kep0, t0=t0, units=units, ax=axes, color=(0.8, 0.8, 0.8))
        pk.orbit_plots.plot_planet(
            kepf, t0=tf, units=units, ax=axes, color=(0.8, 0.8, 0.8))
Esempio n. 30
0
 def setUp(self):
     db =  pysal.open(pysal.examples.get_path("baltim.dbf"),'r')
     self.ds_name = "baltim.dbf"
     self.y_name = "PRICE"
     self.y = np.array(db.by_col(self.y_name)).T
     self.y.shape = (len(self.y),1)
     self.x_names = ["NROOM","AGE","SQFT"]
     self.x = np.array([db.by_col(var) for var in self.x_names]).T
     ww = pysal.open(pysal.examples.get_path("baltim_q.gal"))
     self.w = ww.read()
     ww.close()
     self.w_name = "baltim_q.gal"
     self.w.transform = 'r'
     self.regimes = db.by_col("CITCOU")
     #Artficial:
     n = 256
     self.n2 = n/2
     self.x_a1 = np.random.uniform(-10,10,(n,1))
     self.x_a2 = np.random.uniform(1,5,(n,1))
     self.q_a = self.x_a2 + np.random.normal(0,1,(n,1))
     self.x_a = np.hstack((self.x_a1,self.x_a2))
     self.y_a = np.dot(np.hstack((np.ones((n,1)),self.x_a)),np.array([[1],[0.5],[2]])) + np.random.normal(0,1,(n,1))
     latt = int(np.sqrt(n))
     self.w_a = pysal.lat2W(latt,latt)
     self.w_a.transform='r'
     self.regi_a = [0]*(n/2) + [1]*(n/2)
     self.w_a1 = pysal.lat2W(latt/2,latt)
     self.w_a1.transform='r'
Esempio n. 31
0
    ['ApplicantIncome', 'CoapplicantIncome', 'LoanAmount', 'Loan_Amount_Term'])

ohe = OneHotEncoder(sparse=False)
mms = MinMaxScaler()
enc = {}
ans = 0
for col in X:
    if any(continuous == col):
        enc[col] = mms.fit_transform(
            np.array(X_ohe[col]).reshape(-1, 1).astype(np.float64))
        ans += enc[col].shape[1]
    else:
        tmp = ohe.fit_transform(X_ohe[col].factorize()[0].reshape(
            -1, 1)).astype(np.int64)
        enc[col] = tmp
        ans += enc[col].shape[1]

X_fin = np.hstack([
    enc['Gender'], enc['Married'], enc['Dependents'], enc['Education'],
    enc['Self_Employed'], enc['ApplicantIncome'], enc['CoapplicantIncome'],
    enc['LoanAmount'], enc['Loan_Amount_Term'], enc['Credit_History'],
    enc['Property_Area']
])

# "#### 設問5:ローン審査の予測モデルとして、以下のパイプラインを構成して下さい。\n",
# "- 標準化とロジスティック回帰\n",
# "- 標準化とランダムフォレスト\n",
# "- 標準化と主成分分析とランダムフォレスト"

# num_pipeline =
Esempio n. 32
0
def ocean_30_vs_90_with_colorbar():
    dirs = "../result_h/mean_vector/ocean_currents_30_vs_90_with_colorbar/"
    if not os.path.exists(dirs):
        os.makedirs(dirs)

    start_list.pop()
    for i, start in enumerate(start_list):
        print("******************  {}/{}  *******************".format(
            i + 1, M - 1))
        helmert_30_30_fname = "../data/csv_Helmert_30/Helmert_30_" + str(
            start)[:6] + ".csv"
        data_30 = pd.read_csv(helmert_30_30_fname)
        data_30_vec = [
            np.array(data_30["ocean_u"]),
            np.array(data_30["ocean_v"])
        ]
        helmert_90_90_fname = "../data/csv_Helmert_both_90/Helmert_both_90_" + str(
            start)[:6] + ".csv"
        data_90 = pd.read_csv(helmert_90_90_fname)
        data_90_vec = [
            np.array(data_90["ocean_u_90"]),
            np.array(data_90["ocean_v_90"])
        ]

        vector_list = [data_30_vec, data_90_vec]
        name_list = ["_ocean_30", "_ocean_90"]
        for j in range(2):
            m = Basemap(lon_0=180,
                        boundinglat=50,
                        resolution='l',
                        projection='npstere')
            m.drawcoastlines(color='0.15')
            m.fillcontinents(color='#555555')
            lon = np.array(latlon_ex.Lon)
            lat = np.array(latlon_ex.Lat)
            x, y = m(lon, lat)
            x1 = np.reshape(x, (145, 145), order='F')
            y1 = np.reshape(y, (145, 145), order='F')
            dx1 = (x1[1, 0] - x1[0, 0]) / 2
            dy1 = (y1[0, 1] - y1[0, 0]) / 2

            x2 = np.linspace(x1[0, 0], x1[144, 0], 145)
            y2 = np.linspace(y1[0, 0], y1[0, 144], 145)
            xx, yy = np.meshgrid(x2, y2)
            xx, yy = xx.T, yy.T

            vector_u = np.ma.masked_invalid(vector_list[j][0])
            vector_v = np.ma.masked_invalid(vector_list[j][1])
            vector_speed = np.sqrt(vector_u * vector_u + vector_v * vector_v)

            data_non_wind = vector_speed
            data_non_wind = np.ma.masked_invalid(data_non_wind)
            data1 = np.reshape(data_non_wind, (145, 145), order='F')

            xx = np.hstack([xx, xx[:, 0].reshape(145, 1)])
            xx_ex = np.vstack([xx, (xx[144, :] + (xx[1, 0] - xx[0, 0]))])
            yy = np.vstack([yy, yy[0, :]])
            yy_ex = np.hstack([
                (yy[:, 0].reshape(146, 1) + (yy[0, 0] - yy[0, 1])), yy
            ])

            m.pcolormesh(xx_ex - dx1,
                         yy_ex + dy1,
                         data1,
                         cmap=plt.cm.jet,
                         vmax=0.2,
                         vmin=0)
            m.colorbar(location='bottom')
            m.quiver(x, y, vector_u, vector_v, color="k")
            save_name = dirs + str(start)[:6] + name_list[j] + ".png"
            print(save_name)
            plt.savefig(save_name, dpi=450)
            plt.close()
Esempio n. 33
0
def boundary_conditions(x, t, z=None, nbsym=2):
    """
    Extend the signal beyond it's bounds w.r.t mirror symmetry.

    :param x: Signal to be mirrored.
    :param t: Timestamps of the signal
    :param z: Signal on whose extrema the interpolation is evaluated. (By \
        default this is just ``x``)
    :param nbsym: Number of points added to each end of the signal.
    :type x: array-like
    :type t: array-like
    :type z: array-like
    :type nbsym: int
    :return: timestamps and values of extended extrema, ordered as (minima \
        timestamps, maxima timestamps, minima values, maxima values.)
    :rtype: tuple
    """
    indmax = argrelmax(x)[0]
    indmin = argrelmin(x)[0]
    lx = x.shape[0] - 1
    if indmin.shape[0] + indmax.shape[0] < 3:
        raise ValueError("Not enough extrema.")

    if indmax[0] < indmin[0]:
        if x[0] > x[indmin[0]]:
            lmax = indmax[1:np.min([indmax.shape[0], nbsym + 1])][::-1]
            lmin = indmin[:np.min([indmin.shape[0], nbsym])][::-1]
            lsym = indmax[0]
        else:
            lmax = indmax[1:np.min([indmax.shape[0], nbsym])][::-1]
            lmin = indmin[:np.min([indmin.shape[0], nbsym - 1])][::-1]
            lmin = np.hstack((lmin, [1]))
            lsym = 1
    else:
        if x[0] < x[indmax[0]]:
            lmax = indmax[:np.min([indmax.shape[0], nbsym])][::-1]
            lmin = indmin[1:np.min([indmin.shape[0], nbsym + 1])][::-1]
            lsym = indmin[0]
        else:
            lmax = indmax[:np.min([indmin.shape[0], nbsym - 1])][::-1]
            lmax = np.hstack((lmax, [1]))
            lmin = indmin[:np.min([indmax.shape[0], nbsym])][::-1]
            lsym = 1

    if indmax[-1] < indmin[-1]:
        if x[-1] < x[indmax[-1]]:
            rmax = indmax[(max([indmax.shape[0] - nbsym + 1, 1]) - 1):][::-1]
            rmin = indmin[(max([indmin.shape[0] - nbsym, 1]) - 1):-1][::-1]
            rsym = indmin[-1]
        else:
            rmax = indmax[max(indmax.shape[0] - nbsym + 1, 0):indmax.shape[0]][::-1]
            rmax = np.hstack(([lx], rmax))
            rmin = indmin[max(indmin.shape[0] - nbsym, 0):][::-1]
            rsym = lx
    else:
        if x[-1] > x[indmin[-1]]:
            rmax = indmax[max(indmax.shape[0] - nbsym - 1, 0):-1][::-1]
            rmin = indmin[max(indmin.shape[0] - nbsym, 0):][::-1]
            rsym = indmax[-1]
        else:
            rmax = indmax[max(indmax.shape[0] - nbsym, 0):][::-1]
            rmin = indmin[max(indmin.shape[0] - nbsym + 1, 0):][::-1]
            rmin = np.hstack(([lx], rmin))
            rsym = lx

    tlmin = 2 * t[lsym] - t[lmin]
    tlmax = 2 * t[lsym] - t[lmax]
    trmin = 2 * t[rsym] - t[rmin]
    trmax = 2 * t[rsym] - t[rmax]

    # In case symmetrized parts do not extend enough
    if (tlmin[0] > t[0]) or (tlmax[0] > t[1]):
        if lsym == indmax[0]:
            lmax = indmax[:np.min((indmax.shape[0], nbsym))][::-1]
        else:
            lmin = indmin[:np.min((indmin.shape[0], nbsym))][::-1]
        if lsym == 1:
            raise Exception("Bug")
        lsym = 1
        tlmin = 2 * t[lsym] - t[lmin]
        tlmax = 2 * t[lsym] - t[lmax]

    if (trmin[-1] < t[lx]) or (trmax[-1] < t[lx]):
        if rsym == indmax.shape[0]:
            rmax = indmax[np.max([indmax.shape[0] - nbsym + 1,
                                 1]):indmax.shape[0]][::-1]
        else:
            rmin = indmin[np.max([indmax.shape[0] - nbsym + 1,
                                 1]):indmin.shape[0]][::-1]

        if rsym == lx:
            raise Exception("bug")
        rsym = lx
        trmin = 2 * t[rsym] - t[rmin]
        trmax = 2 * t[rsym] - t[rmax]

    if z is None:
        z = x
    zlmax = z[lmax]
    zlmin = z[lmin]
    zrmax = z[rmax]
    zrmin = z[rmin]

    tmin = map(np.array, [tlmin, t[indmin], trmin])
    tmax = map(np.array, [tlmax, t[indmax], trmax])
    zmin = map(np.array, [zlmin, z[indmin], zrmin])
    zmax = map(np.array, [zlmax, z[indmax], zrmax])

    tmin, tmax, zmin, zmax = map(np.hstack, [tmin, tmax, zmin, zmax])
    return tmin, tmax, zmin, zmax
Esempio n. 34
0
def run_env(budget_para):
    # 训练
    print('data loading')
    test_data = pd.read_csv(data_type['data_path'] + data_type['campaign_id'] +
                            str(data_type['fraction_type']) + '/test_data.csv',
                            header=None).drop([0])
    test_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
        = test_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
        int)
    test_data.iloc[:, config['data_pctr_index']] \
        = test_data.iloc[:, config['data_pctr_index']].astype(
        float)
    test_data.iloc[:, config['data_fraction_index']] \
        = test_data.iloc[:, config['data_fraction_index']].astype(
        int)
    test_data = test_data.values

    test_all_sample = pd.read_csv(
        data_type['data_path'] + data_type['campaign_id'] +
        str(data_type['fraction_type']) + '/test_all_sample.csv',
        header=None).drop([0])
    test_all_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
        = test_all_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
        int)
    test_all_sample.iloc[:, config['data_pctr_index']] \
        = test_all_sample.iloc[:, config['data_pctr_index']].astype(
        float)
    test_all_sample.iloc[:, config['data_fraction_index']] \
        = test_all_sample.iloc[:, config['data_fraction_index']].astype(
        int)
    test_all_sample = test_all_sample.values

    test_down_sample = pd.read_csv(
        data_type['data_path'] + data_type['campaign_id'] +
        str(data_type['fraction_type']) + '/test_down_sample.csv',
        header=None).drop([0])
    test_down_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
        = test_down_sample.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
        int)
    test_down_sample.iloc[:, config['data_pctr_index']] \
        = test_down_sample.iloc[:, config['data_pctr_index']].astype(
        float)
    test_down_sample.iloc[:, config['data_fraction_index']] \
        = test_down_sample.iloc[:, config['data_fraction_index']].astype(
        int)
    test_down_sample = test_down_sample.values

    train_data = pd.read_csv(data_type['data_path'] +
                             data_type['campaign_id'] +
                             str(data_type['fraction_type']) + '/train_' +
                             data_type['type'] + '.csv')
    train_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2] \
        = train_data.iloc[:, config['data_clk_index']:config['data_marketprice_index'] + 2].astype(
        int)
    train_data.iloc[:, config['data_pctr_index']] \
        = train_data.iloc[:, config['data_pctr_index']].astype(
        float)
    train_data.iloc[:, config['data_fraction_index']] \
        = train_data.iloc[:, config['data_fraction_index']].astype(
        int)
    train_data = train_data.values

    # config['train_budget'] = np.sum(train_data[:, config['data_marketprice_index']])
    config['train_budget'] = 32000000
    budget = config['train_budget'] * budget_para

    # config['test_budget'] = np.sum(test_data[:, config['data_marketprice_index']])
    config['test_budget'] = 32000000

    original_ctr = np.sum(
        train_data[:, config['data_clk_index']]) / len(train_data)
    total_clks = np.sum(train_data[:, config['data_clk_index']])
    real_hour_clks = []
    for i in range(data_type['fraction_type']):
        real_hour_clks.append(
            np.sum(train_data[train_data[:, config['data_fraction_index']] ==
                              i][:, config['data_clk_index']]))

    td_error, action_loss = 0, 0
    eCPC = choose_eCPC(data_type['campaign_id'], original_ctr)

    e_results = []
    e_actions = []
    test_records = []

    e_all_actions = []
    test_all_records = []

    e_down_actions = []
    test_down_records = []

    is_learn = False

    fraction_type = data_type['fraction_type']
    exploration_rate = config['exploration_rate']
    for episode in range(config['train_episodes']):
        e_clks = [0 for i in range(fraction_type)]  # episode各个时段所获得的点击数,以下类推
        e_profits = [0 for i in range(fraction_type)]
        e_reward = [0 for i in range(fraction_type)]
        e_cost = [0 for i in range(fraction_type)]

        e_true_value = [0 for i in range(fraction_type)]
        e_miss_true_value = [0 for i in range(fraction_type)]
        e_win_imp_with_clk_value = [0 for i in range(fraction_type)]
        e_win_imp_without_clk_cost = [0 for i in range(fraction_type)
                                      ]  # 各个时段浪费在没有点击的曝光上的预算
        e_lose_imp_with_clk_value = [0 for i in range(fraction_type)]
        e_clk_aucs = [0 for i in range(fraction_type)]
        e_clk_no_win_aucs = [0 for i in range(fraction_type)]
        e_lose_imp_without_clk_cost = [0 for i in range(fraction_type)]
        e_no_clk_aucs = [0 for i in range(fraction_type)]
        e_no_clk_no_win_aucs = [0 for i in range(fraction_type)]

        actions = [0 for i in range(fraction_type)]
        init_action = 0
        next_action = 0

        state_ = np.array([])

        break_time_slot = 0
        real_clks = [0 for i in range(fraction_type)]
        bid_nums = [0 for i in range(fraction_type)]
        imps = [0 for i in range(fraction_type)]

        ou_noise = OrnsteinUhlenbeckNoise(mu=np.zeros(1))

        # 状态包括:当前CTR,
        for t in range(fraction_type):
            auc_datas = train_data[train_data[:, config['data_fraction_index']]
                                   == t]

            if t == 0:
                state = np.array(
                    [1, 0, 0, 0]
                )  # current_time_slot, budget_left_ratio, cost_t_ratio, ctr_t, win_rate_t
                action = RL.choose_action(state)
                action = np.clip(np.random.normal(action, exploration_rate),
                                 -0.99, 0.99)
                init_action = action
                bids = auc_datas[:, config['data_pctr_index']] * eCPC / (
                    1 + init_action)
                bids = np.where(bids >= 300, 300, bids)
            else:
                state = state_
                action = next_action
                bids = auc_datas[:, config['data_pctr_index']] * eCPC / (
                    1 + action)
                bids = np.where(bids >= 300, 300, bids)

            actions[t] = action

            win_auctions = auc_datas[
                bids >= auc_datas[:, config['data_marketprice_index']]]
            no_win_auctions = auc_datas[
                bids <= auc_datas[:, config['data_marketprice_index']]]
            e_cost[t] = np.sum(win_auctions[:,
                                            config['data_marketprice_index']])
            e_profits[t] = np.sum(
                win_auctions[:, config['data_pctr_index']] * eCPC -
                win_auctions[:, config['data_marketprice_index']])

            e_true_value[t] = np.sum(
                win_auctions[:, config['data_pctr_index']] * eCPC)
            e_miss_true_value[t] = np.sum(
                no_win_auctions[:, config['data_pctr_index']] * eCPC)
            with_clk_win_auctions = win_auctions[
                win_auctions[:, config['data_clk_index']] == 1]
            e_win_imp_with_clk_value[t] = np.sum(
                with_clk_win_auctions[:, config['data_pctr_index']] * eCPC)
            e_win_imp_without_clk_cost[t] = np.sum(
                win_auctions[win_auctions[:, config['data_clk_index']] ==
                             0][:, config['data_marketprice_index']])
            with_clk_no_win_auctions = no_win_auctions[
                no_win_auctions[:, config['data_clk_index']] == 1]
            e_lose_imp_with_clk_value[t] = np.sum(
                with_clk_no_win_auctions[:, config['data_pctr_index']] * eCPC)

            e_clks[t] = np.sum(win_auctions[:, config['data_clk_index']],
                               dtype=int)
            imps[t] = len(win_auctions)
            real_clks[t] = np.sum(auc_datas[:, config['data_clk_index']],
                                  dtype=int)
            bid_nums[t] = len(auc_datas)

            e_clk_aucs[t] = len(
                auc_datas[auc_datas[:, config['data_clk_index']] == 1])
            e_clk_no_win_aucs[t] = len(with_clk_no_win_auctions)

            e_no_clk_aucs[t] = len(
                auc_datas[auc_datas[:, config['data_clk_index']] == 0])

            without_clk_no_win_auctions = no_win_auctions[
                no_win_auctions[:, config['data_clk_index']] == 0]
            e_lose_imp_without_clk_cost[t] = np.sum(
                without_clk_no_win_auctions[:,
                                            config['data_marketprice_index']])
            e_no_clk_no_win_aucs[t] = len(without_clk_no_win_auctions)

            market_prices_t = auc_datas[:, config['data_marketprice_index']]

            bid_win_t = bids[
                bids >= auc_datas[:, config['data_marketprice_index']]]
            market_price_win_t = market_prices_t[
                bids >= auc_datas[:, config['data_marketprice_index']]]

            no_win_imps_market_prices_t = np.sum(
                no_win_auctions[:, config['data_marketprice_index']])
            if np.sum(e_cost) >= budget:
                break_time_slot = t
                temp_cost = 0
                temp_lose_cost = 0
                temp_win_auctions = 0
                e_clks[t] = 0
                e_profits[t] = 0

                e_true_value[t] = 0
                e_miss_true_value[t] = 0

                e_win_imp_without_clk_cost[t] = 0
                e_lose_imp_with_clk_value[t] = 0
                real_clks[t] = 0
                imps[t] = 0
                bid_nums[t] = 0

                e_win_imp_with_clk_value[t] = 0
                e_clk_aucs[t] = 0
                e_lose_imp_without_clk_cost[t] = 0
                e_no_clk_aucs[t] = 0
                e_no_clk_no_win_aucs[t] = 0

                bids_t = []
                market_prices_t = []

                bid_win_t = []
                market_price_win_t = []
                no_win_imps_market_prices_t = 0
                for i in range(len(auc_datas)):
                    if temp_cost >= (budget - np.sum(e_cost[:t])):
                        break
                    current_data = auc_datas[i, :]
                    temp_clk = int(current_data[config['data_clk_index']])
                    temp_market_price = current_data[
                        config['data_marketprice_index']]
                    if t == 0:
                        temp_action = init_action
                    else:
                        temp_action = next_action
                    bid = current_data[config['data_pctr_index']] * eCPC / (
                        1 + temp_action)
                    bid = bid if bid <= 300 else 300
                    real_clks[t] += temp_clk
                    bid_nums[t] += 1

                    if temp_clk == 1:
                        e_clk_aucs[t] += temp_clk
                    else:
                        e_no_clk_aucs[t] += 1
                    bids_t.append(bid)
                    market_prices_t.append(temp_market_price)
                    if bid >= temp_market_price:
                        if temp_clk == 0:
                            e_win_imp_without_clk_cost[t] += temp_market_price
                        else:
                            e_win_imp_with_clk_value[t] += current_data[
                                config['data_pctr_index']] * eCPC
                        bid_win_t.append(bid)
                        market_price_win_t.append(temp_market_price)

                        e_profits[t] += (
                            current_data[config['data_pctr_index']] * eCPC -
                            temp_market_price)
                        e_true_value[t] += current_data[
                            config['data_pctr_index']] * eCPC
                        e_clks[t] += temp_clk
                        imps[t] += 1
                        temp_cost += temp_market_price
                        temp_win_auctions += 1
                    else:
                        e_miss_true_value[t] += current_data[
                            config['data_pctr_index']] * eCPC
                        temp_lose_cost += temp_market_price
                        no_win_imps_market_prices_t += temp_market_price
                        if temp_clk == 1:
                            e_clk_no_win_aucs[t] += 1
                            e_lose_imp_with_clk_value[t] += current_data[
                                config['data_pctr_index']] * eCPC
                        else:
                            e_no_clk_no_win_aucs[t] += 1
                e_cost[t] = temp_cost
                ctr_t = e_clks[
                    t] / temp_win_auctions if temp_win_auctions > 0 else 0
                win_rate_t = temp_win_auctions / bid_nums[t]
            else:
                ctr_t = np.sum(
                    win_auctions[:, config['data_clk_index']]) / len(
                        win_auctions) if len(win_auctions) > 0 else 0
                win_rate_t = len(win_auctions) / len(auc_datas) if len(
                    auc_datas) > 0 else 0
            budget_left_ratio = (budget - np.sum(e_cost[:t + 1])) / budget
            budget_left_ratio = budget_left_ratio if budget_left_ratio >= 0 else 0
            time_left_ratio = (fraction_type - 1 - t) / fraction_type
            avg_time_spend = budget_left_ratio / time_left_ratio if time_left_ratio > 0 else 0
            cost_t_ratio = e_cost[t] / budget
            if t == 0:
                state_ = np.array(
                    [avg_time_spend, cost_t_ratio, ctr_t, win_rate_t])
            else:
                state_ = np.array(
                    [avg_time_spend, cost_t_ratio, ctr_t, win_rate_t])
            action_ = RL.choose_action(state_)

            action_ = np.clip(np.random.normal(action_, exploration_rate),
                              -0.99, 0.99)
            next_action = action_

            reward_t = adjust_reward(
                len(auc_datas), e_true_value, e_miss_true_value, bid_win_t,
                market_price_win_t, e_win_imp_with_clk_value, e_cost,
                e_win_imp_without_clk_cost, real_clks,
                e_lose_imp_with_clk_value, e_clk_aucs, e_clk_no_win_aucs,
                e_lose_imp_without_clk_cost, e_no_clk_aucs,
                e_no_clk_no_win_aucs, no_win_imps_market_prices_t, budget,
                total_clks, t)
            reward = reward_t
            e_reward[t] = reward
            transition = np.hstack(
                (state.tolist(), action, reward, state_.tolist()))
            RL.store_transition(transition)

            if np.sum(e_cost) >= budget:
                break

        if (episode > 0) and ((episode + 1) % config['observation_episode']
                              == 0):
            e_result = [
                np.sum(e_reward),
                np.sum(e_profits), budget,
                np.sum(e_cost),
                int(np.sum(e_clks)),
                int(np.sum(real_clks)),
                np.sum(bid_nums),
                np.sum(imps),
                np.sum(e_cost) / np.sum(imps) if np.sum(imps) > 0 else 0,
                break_time_slot, td_error, action_loss
            ]
            e_results.append(e_result)

            # 在原始论文中,每感知一次环境就要对模型进行一次训练
            # 然而频繁地学习在未充分感知环境的情况下,会使模型陷入局部(当前)最优
            # 因此可以每感知N次再对模型训练n次,这样会使得模型更稳定,并加快学习速度
            is_learn = True
            exploration_rate *= 0.999
            if is_learn:  # after observing config['observation_size'] times, for config['learn_iter'] learning time
                for m in range(config['learn_iter']):
                    td_e, a_loss = RL.learn()
                    td_error, action_loss = td_e, a_loss
                    RL.soft_update(RL.Actor, RL.Actor_)
                    RL.soft_update(RL.Critic, RL.Critic_)
                    if m == config['learn_iter'] - 1:
                        is_learn = False

            actions_df = pd.DataFrame(data=actions)
            actions_df.to_csv(log_path + '/result_reward_1/' +
                              str(fraction_type) + '/train_actions_' +
                              str(budget_para) + '.csv')

            hour_clks = {
                'clks': e_clks,
                'no_bid_clks': np.subtract(real_hour_clks, e_clks).tolist(),
                'real_clks': real_hour_clks
            }
            hour_clks_df = pd.DataFrame(data=hour_clks)
            hour_clks_df.to_csv(log_path + '/result_reward_1/' +
                                str(fraction_type) + '/train_hour_clks_' +
                                str(budget_para) + '.csv')
            print(
                'episode {}, reward={}, profits={}, budget={}, cost={}, clks={}, real_clks={}, bids={}, '
                'imps={}, cpm={}, break_time_slot={}, td_error={}, action_loss={}\n'
                .format(
                    episode + 1, np.sum(e_reward), np.sum(e_profits), budget,
                    np.sum(e_cost), int(np.sum(e_clks)),
                    int(np.sum(real_clks)), np.sum(bid_nums), np.sum(imps),
                    np.sum(e_cost) / np.sum(imps) if np.sum(imps) > 0 else 0,
                    break_time_slot, td_error, action_loss))

            test_result, test_actions, test_hour_clks = test_env(
                config['test_budget'] * budget_para, budget_para, test_data,
                eCPC)
            test_records.append(test_result)
            e_actions.append(test_actions)

            test_all_result, test_all_actions, test_hour_clks = test_env(
                config['test_budget'] * budget_para, budget_para,
                test_all_sample, eCPC)
            test_all_records.append(test_all_result)
            e_all_actions.append(test_all_actions)

            test_down_result, test_down_actions, test_hour_clks = test_env(
                config['test_budget'] * budget_para, budget_para,
                test_down_sample, eCPC)
            test_down_records.append(test_down_result)
            e_down_actions.append(test_down_actions)

    e_results_df = pd.DataFrame(data=e_results,
                                columns=[
                                    'reward', 'profits', 'budget', 'cost',
                                    'clks', 'real_clks', 'bids', 'imps', 'cpm',
                                    'break_time_slot', 'td_error',
                                    'action_loss'
                                ])
    e_results_df.to_csv(log_path + '/result_reward_1/' + str(fraction_type) +
                        '/train_episode_results_' + str(budget_para) + '.csv')

    e_actions_df = pd.DataFrame(data=e_actions)
    e_actions_df.to_csv(log_path + '/result_reward_1/' + str(fraction_type) +
                        '/test_episode_actions_' + str(budget_para) + '.csv')

    ## test_data
    test_records_df = pd.DataFrame(data=test_records,
                                   columns=[
                                       'profits', 'budget', 'cost', 'clks',
                                       'real_clks', 'bids', 'imps', 'cpm',
                                       'break_time_slot'
                                   ])

    test_records_df.to_csv(log_path + '/result_reward_1/' +
                           str(fraction_type) + '/test_episode_results_' +
                           str(budget_para) + '.csv')

    e_actions_df = pd.DataFrame(data=e_actions)
    e_actions_df.to_csv(log_path + '/result_reward_1/' + str(fraction_type) +
                        '/test_episode_actions_' + str(budget_para) + '.csv')

    # test_all_sample
    test_all_records_df = pd.DataFrame(data=test_all_records,
                                       columns=[
                                           'profits', 'budget', 'cost', 'clks',
                                           'real_clks', 'bids', 'imps', 'cpm',
                                           'break_time_slot'
                                       ])
    test_all_records_df.to_csv(log_path + '/result_reward_1/' +
                               str(fraction_type) +
                               '/all_sample/test_episode_results_' +
                               str(budget_para) + '.csv')

    e_all_actions_df = pd.DataFrame(data=e_all_actions)
    e_all_actions_df.to_csv(log_path + '/result_reward_1/' +
                            str(fraction_type) +
                            '/all_sample/test_episode_actions_' +
                            str(budget_para) + '.csv')

    # test_down_sample
    test_down_records_df = pd.DataFrame(data=test_down_records,
                                        columns=[
                                            'profits', 'budget', 'cost',
                                            'clks', 'real_clks', 'bids',
                                            'imps', 'cpm', 'break_time_slot'
                                        ])
    test_down_records_df.to_csv(log_path + '/result_reward_1/' +
                                str(fraction_type) +
                                '/down_sample/test_episode_results_' +
                                str(budget_para) + '.csv')

    e_down_actions_df = pd.DataFrame(data=e_down_actions)
    e_down_actions_df.to_csv(log_path + '/result_reward_1/' +
                             str(fraction_type) +
                             '/down_sample/test_episode_actions_' +
                             str(budget_para) + '.csv')
Esempio n. 35
0
def execute(context):
    df_persons = context.stage("synthesis.population.sociodemographics")[[
        "person_id", "hts_person_id", "age", "sex", "residence_area_index"
    ]]

    df_trips = pd.DataFrame(context.stage("data.hts.cleaned")[1], copy=True)

    df_trips = df_trips[[
        "person_id", "trip_id", "departure_time", "arrival_time", "mode",
        "preceeding_purpose", "following_purpose"
    ]]
    df_trips.to_csv("%s/HTS/trips.csv" % context.config("data_path"))
    assert (len(df_trips) == len(df_trips.dropna()))

    df_trips = df_trips.sort_values(by=["person_id", "trip_id"])

    df_trip_counts = df_trips[[
        "person_id"
    ]].groupby("person_id").size().reset_index(name="count")
    df_trips["trip_id"] = np.hstack(
        [np.arange(n) for n in df_trip_counts["count"].values])

    df_trips.columns = [
        "hts_person_id", "trip_id", "departure_time", "arrival_time", "mode",
        "preceeding_purpose", "following_purpose"
    ]

    # Merge trips to persons
    df_trips = pd.merge(df_persons, df_trips)

    df_trips.loc[df_trips["arrival_time"] < df_trips["departure_time"],
                 "arrival_time"] += 24.0 * 3600.0
    df_trips.loc[:,
                 "travel_time"] = df_trips.loc[:,
                                               "arrival_time"] - df_trips.loc[:,
                                                                              "departure_time"]

    df_trips = df_trips[[
        "person_id", "trip_id", "departure_time", "arrival_time",
        "travel_time", "mode", "preceeding_purpose", "following_purpose",
        "age", "hts_person_id", "residence_area_index", "sex"
    ]]

    df_trips = df_trips.sort_values(by=["person_id", "trip_id"])

    # Diversify departure times
    counts = df_trips[[
        "person_id", "trip_id"
    ]].groupby("person_id").size().reset_index(name="count")["count"].values

    interval = df_trips[[
        "person_id", "departure_time"
    ]].groupby("person_id").min().reset_index()["departure_time"].values
    interval = np.minimum(
        1800.0, interval
    )  # If first departure time is just 5min after midnight, we only add a deviation of 5min

    offset = np.random.random(size=(len(counts), )) * interval * 2.0 - interval
    offset = np.repeat(offset, counts)

    df_trips["departure_time"] += offset
    df_trips["arrival_time"] += offset
    df_trips["departure_time"] = np.round(df_trips["departure_time"])
    df_trips["arrival_time"] = np.round(df_trips["arrival_time"])

    return df_trips
Esempio n. 36
0
import cv2
 
# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required = True, help = "Path to the image")
args = vars(ap.parse_args())


# load the image, clone it for output, and then convert it to grayscale
image = cv2.imread(args["image"])
output = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# detect circles in the image
circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100)
 
# ensure at least some circles were found
if circles is not None:
	# convert the (x, y) coordinates and radius of the circles to integers
	circles = np.round(circles[0, :]).astype("int")
 
	# loop over the (x, y) coordinates and radius of the circles
	for (x, y, r) in circles:
		# draw the circle in the output image, then draw a rectangle
		# corresponding to the center of the circle
		cv2.circle(output, (x, y), r, (0, 255, 0), 4)
		cv2.rectangle(output, (x - 5, y - 5), (x + 5, y + 5), (0, 128, 255), -1)
 
	# show the output image
	cv2.imshow("output", np.hstack([image, output]))
	cv2.waitKey(0)
Esempio n. 37
0
    def _train_one_epoch(self, epoch):
        # Train
        try:
            self.net.train()
            _learning_rate_ = self._adjust_learning_rate(epoch)
            Tools.print('Epoch: [{}] lr={} name={}'.format(
                epoch, _learning_rate_, Config.name))

            self.produce_class.reset()
            avg_loss, avg_loss_1, avg_loss_2 = AverageMeter(), AverageMeter(
            ), AverageMeter()
            for batch_idx, (inputs, _,
                            indexes) in enumerate(self.train_loader):
                batch_size, _, c, w, h = inputs.shape
                a = np.random.beta(1, 1, [batch_size, 1])
                b = np.hstack([a, 1 - a])
                ab = np.tile(b[..., None, None, None], [c, w, h])

                x1_add_x2 = torch.mean(inputs * ab, dim=1,
                                       keepdim=True).float()
                now_inputs = torch.cat([inputs, x1_add_x2],
                                       dim=1).view(-1, c, w, h)
                now_inputs = Config.cuda(now_inputs)

                self.optimizer.zero_grad()

                out_logits, out_l2norm, out_softmax = self.net(now_inputs)
                out_dim = out_softmax.shape[-1]
                now_out_logits = out_logits.view(batch_size, -1, out_dim)
                now_out_l2norm = out_l2norm.view(batch_size, -1, out_dim)
                now_out_softmax = out_softmax.view(batch_size, -1, out_dim)

                softmax_w = np.hstack([a, 1 - a, -np.ones_like(a)])
                softmax_w = torch.tensor(
                    np.tile(softmax_w[..., None], [
                        out_dim,
                    ])).float()
                now_softmax_w = Config.cuda(softmax_w)
                out_error = torch.sum(now_out_softmax * now_softmax_w, dim=1)
                loss1 = torch.mean(torch.sum(torch.pow(out_error, 2), dim=1))
                loss1 *= Config.mix_ratio

                targets = self.produce_class.get_label(indexes[0])
                targets2 = self.produce_class.get_label(indexes[1])
                self.produce_class.cal_label(now_out_l2norm[:, 0, :],
                                             indexes[0])
                loss2 = self.criterion(now_out_logits[:, 0, :], targets)

                if Config.has_ic_2:
                    loss2_2 = self.criterion(now_out_logits[:, 1, :], targets2)
                    loss2 = loss2 + loss2_2
                    pass

                loss = (loss1 + loss2) if Config.has_mix else loss2
                loss.backward()
                self.optimizer.step()

                avg_loss.update(loss.item(), inputs.size(0))
                avg_loss_1.update(loss1.item(), inputs.size(0))
                avg_loss_2.update(loss2.item(), inputs.size(0))

                # if batch_idx % 100 == 0:
                #     Tools.print('Train: [{}] {}/{} Loss: {:.4f}/{:.4f} '
                #                 'Loss 1: {:.4f}/{:.4f} Loss 2: {:.4f}/{:.4f}'.format(
                #         epoch, batch_idx, len(self.train_loader), avg_loss.avg, avg_loss.val,
                #         avg_loss_1.avg, avg_loss_1.val, avg_loss_2.avg, avg_loss_2.val))
                pass

            Tools.print("Train: [{}] {}/{}".format(epoch,
                                                   self.produce_class.count,
                                                   self.produce_class.count_2))
            Tools.print('Train: [{}] {} Loss: {:.4f}/{:.4f} '
                        'Loss 1: {:.4f}/{:.4f} Loss 2: {:.4f}/{:.4f}'.format(
                            epoch, len(self.train_loader), avg_loss.avg,
                            avg_loss.val, avg_loss_1.avg, avg_loss_1.val,
                            avg_loss_2.avg, avg_loss_2.val))
        finally:
            pass

        # Test
        try:
            Tools.print("Test:  [{}] .......".format(epoch))
            _acc = self.test(epoch=epoch)
            if _acc > self.best_acc:
                Tools.print('Saving..')
                state = {
                    'net': self.net.state_dict(),
                    'acc': _acc,
                    'epoch': epoch
                }
                torch.save(state, Config.checkpoint_path)
                self.best_acc = _acc
                pass
            Tools.print('Test:  [{}] best accuracy: {:.2f}'.format(
                epoch, self.best_acc * 100))
        finally:
            pass

        pass
Esempio n. 38
0
ax.set_ylabel('age')
plt.savefig(r'', bbox_inches='tight')

df.groupby(['month', 'y']).size().unstack().plot(kind='bar', stacked=True)
plt.gca().set_ylabel('count')
plt.savefig(r'', bbox_inches='tight')

# df['pdays'].plot.hist(bins=50)
# plt.gca().set_ylabel('count')
# plt.savefig(r'', bbox_inches='tight')
df['pdays'].quantile(
    q=[0, 0.005, 0.025, 0.1, 0.25, 0.5, 0.75, 0.9, 0.975, 0.995, 1])
df.boxplot(column='pdays')

###################### ANN Results ###################################
ax = pd.DataFrame(np.hstack(
    (labels, tf.nn.sigmoid(model(df.values.astype(np.float32))).numpy())),
                  columns=['result', 'predicted']).boxplot(column='predicted',
                                                           by='result')
plt.gca().set_title('')
plt.gca().set_ylabel('predicted')
plt.savefig(r'D:\Personal\presentations\WF\pred_boxplot.png',
            bbox_inches='tight')

sns.set(style="whitegrid")
ax = sns.boxplot(x="result",
                 y="predicted",
                 data=pd.DataFrame(np.hstack(
                     (labels, tf.nn.sigmoid(model(df.values.astype(
                         np.float32))).numpy())),
                                   columns=['result', 'predicted']),
                 showfliers=False)
Esempio n. 39
0
def runSingleFold(data_list, file_idx, fold_number):
    print '| ---- ---- Fold #{} ---- ----'.format(fold_number)

    number_of_views = len(data_list)

    (training_blocks, tuning_blocks,
     testing_blocks) = util.configure_blocks(fold_number)

    data_pre_processor = DataPreProcessor(data_list, file_idx, training_blocks,
                                          False)
    training_data_per_view, training_labels_per_view = data_pre_processor.process(
    )

    data_pre_processor = DataPreProcessor(data_list, file_idx, tuning_blocks,
                                          False)
    tuning_data_per_view, tuning_labels_per_view = data_pre_processor.process()

    data_pre_processor = DataPreProcessor(data_list, file_idx, testing_blocks,
                                          False)
    testing_data_per_view, testing_labels_per_view = data_pre_processor.process(
    )

    for i in range(number_of_views):
        training_data = np.ndarray(shape=(0, np.shape(
            training_data_per_view[i])[0]),
                                   dtype=np.float)
        training_labels = np.array([], dtype=np.int)

        if use_full_phones:
            training_data = training_data_per_view[i].transpose()
            training_labels = training_labels_per_view[i]
        else:
            for j in range(len(training_labels_per_view[i])):
                if (training_labels_per_view[i][j] in vowel_labels):
                    training_data = np.vstack(
                        (training_data,
                         training_data_per_view[i].transpose()[j, :]))
                    training_labels = np.hstack(
                        (training_labels, int(training_labels_per_view[i][j])))

        param_msg = None

        # Start tuning/testing
        if classification_model == ClassificationModel.Kernel_SVM_RBF:
            max_accuracy = 0.0
            optimal_gamma = 0.0

            for j in [3e-08, 3.5e-08, 4e-08, 4.5e-08]:
                model = svm.SVC(decision_function_shape='ovo',
                                kernel='rbf',
                                gamma=j,
                                C=2)
                model.fit(training_data, training_labels)
                accuracy = getAccuracy(model, tuning_data_per_view[i],
                                       tuning_labels_per_view[i])
                if accuracy > max_accuracy:
                    max_accuracy = accuracy
                    optimal_gamma = j

            param_msg = 'Optimal gamma value: {}'.format(optimal_gamma)

            model = svm.SVC(decision_function_shape='ovo',
                            kernel='rbf',
                            gamma=optimal_gamma,
                            C=2)
        else:
            max_accuracy = 0.0
            optimal_neighbors = 0

            for j in [28, 32, 36, 40]:
                model = neighbors.KNeighborsClassifier(j, weights='distance')
                model.fit(training_data, training_labels)
                accuracy = getAccuracy(model, tuning_data_per_view[i],
                                       tuning_labels_per_view[i])
                if accuracy > max_accuracy:
                    max_accuracy = accuracy
                    optimal_neighbors = j

            param_msg = 'Optimal number of neighbors: {}'.format(
                optimal_neighbors)

            model = neighbors.KNeighborsClassifier(optimal_neighbors,
                                                   weights='distance')

        model.fit(training_data, training_labels)
        accuracy = getAccuracy(model, testing_data_per_view[i],
                               testing_labels_per_view[i])

        print '| Accuracy for view {}: {:.3f}'.format(
            i + 1, accuracy) + ', ' + param_msg

    print '|'
Esempio n. 40
0
 # key bindings
 if k == 27:         # esc to exit
     break
 elif k == ord('0'): # BG drawing
     print(" mark background regions with left mouse button \n")
     value = DRAW_BG
 elif k == ord('1'): # FG drawing
     print(" mark foreground regions with left mouse button \n")
     value = DRAW_FG
 elif k == ord('2'): # PR_BG drawing
     value = DRAW_PR_BG
 elif k == ord('3'): # PR_FG drawing
     value = DRAW_PR_FG
 elif k == ord('s'): # save image
     bar = np.zeros((img.shape[0],5,3),np.uint8)
     res = np.hstack((img2,bar,img,bar,output))
     cv2.imwrite('grabcut_output.png',res)
     print(" Result saved as image \n")
 elif k == ord('r'): # reset everything
     print("resetting \n")
     rect = (0,0,1,1)
     drawing = False
     rectangle = False
     rect_or_mask = 100
     rect_over = False
     value = DRAW_FG
     img = img2.copy()
     mask = np.zeros(img.shape[:2],dtype = np.uint8) # mask initialized to PR_BG
     output = np.zeros(img.shape,np.uint8)           # output image to be shown
 elif k == ord('n'): # segment the image
     print(""" For finer touchups, mark foreground and background after pressing keys 0-3
Esempio n. 41
0
        dec['cdsvm'].append(dec1)
        auc['cdsvm'].append(roc_auc_score(yt,dec1))

    acc_all['cdsvm'].append(acc['cdsvm'])
    auc_all['cdsvm'].append(auc['cdsvm'])

acc_std = {}
acc_mean = {}
auc_std = {}
auc_mean = {}

acc_all['cdsvm'] = np.array(acc_all['cdsvm'])
acc_mean['cdsvm'] = np.mean(acc_all['cdsvm'],axis=1).reshape((len(src_full),1))
acc_std['cdsvm'] = np.std(acc_all['cdsvm'],axis=1).reshape((len(src_full),1))
auc_all['cdsvm'] = np.array(auc_all['cdsvm'])
auc_mean['cdsvm'] = np.mean(auc_all['cdsvm'],axis=1).reshape((len(src_full),1))
auc_std['cdsvm'] = np.std(auc_all['cdsvm'],axis=1).reshape((len(src_full),1))


source_list = np.array(src_list).T

cdsvm_result = np.hstack((acc_all['cdsvm'], acc_mean['cdsvm'], acc_std['cdsvm'],
                          auc_all['cdsvm'], auc_mean['cdsvm'], auc_std['cdsvm']))
cdsvm_df = pd.DataFrame(cdsvm_result, index = src_str_list)
cdsvm_df.to_csv("%sfold_ica+cdsvm_result_%s_%s.csv"%(kfold, target[0], target[1]))


for i in dec:
    dec[i].append(yt)
    pd.DataFrame(np.array(dec[i])).to_csv('%sfold_dec_%s_%s_%s.csv'%(kfold,i,target[0], target[1]))
  def evaluate_recall(self, candidate_boxes=None, thresholds=None,
                      area='all', limit=None):
    """Evaluate detection proposal recall metrics.

    Returns:
        results: dictionary of results with keys
            'ar': average recall
            'recalls': vector recalls at each IoU overlap threshold
            'thresholds': vector of IoU overlap thresholds
            'gt_overlaps': vector of all ground-truth overlaps
    """
    # Record max overlap value for each gt box
    # Return vector of overlap values
    areas = {'all': 0, 'small': 1, 'medium': 2, 'large': 3,
             '96-128': 4, '128-256': 5, '256-512': 6, '512-inf': 7}
    area_ranges = [[0 ** 2, 1e5 ** 2],  # all
                   [0 ** 2, 32 ** 2],  # small
                   [32 ** 2, 96 ** 2],  # medium
                   [96 ** 2, 1e5 ** 2],  # large
                   [96 ** 2, 128 ** 2],  # 96-128
                   [128 ** 2, 256 ** 2],  # 128-256
                   [256 ** 2, 512 ** 2],  # 256-512
                   [512 ** 2, 1e5 ** 2],  # 512-inf
                   ]
    assert area in areas, 'unknown area range: {}'.format(area)
    area_range = area_ranges[areas[area]]
    gt_overlaps = np.zeros(0)
    num_pos = 0
    for i in range(self.num_images):
      # Checking for max_overlaps == 1 avoids including crowd annotations
      # (...pretty hacking :/)
      max_gt_overlaps = self.roidb[i]['gt_overlaps'].toarray().max(axis=1)
      gt_inds = np.where((self.roidb[i]['gt_classes'] > 0) &
                         (max_gt_overlaps == 1))[0]
      gt_boxes = self.roidb[i]['boxes'][gt_inds, :]
      gt_areas = self.roidb[i]['seg_areas'][gt_inds]
      valid_gt_inds = np.where((gt_areas >= area_range[0]) &
                               (gt_areas <= area_range[1]))[0]
      gt_boxes = gt_boxes[valid_gt_inds, :]
      num_pos += len(valid_gt_inds)

      if candidate_boxes is None:
        # If candidate_boxes is not supplied, the default is to use the
        # non-ground-truth boxes from this roidb
        non_gt_inds = np.where(self.roidb[i]['gt_classes'] == 0)[0]
        boxes = self.roidb[i]['boxes'][non_gt_inds, :]
      else:
        boxes = candidate_boxes[i]
      if boxes.shape[0] == 0:
        continue
      if limit is not None and boxes.shape[0] > limit:
        boxes = boxes[:limit, :]

      overlaps = bbox_overlaps(boxes.astype(np.float),
                               gt_boxes.astype(np.float))

      _gt_overlaps = np.zeros((gt_boxes.shape[0]))
      for j in range(gt_boxes.shape[0]):
        # find which proposal box maximally covers each gt box
        argmax_overlaps = overlaps.argmax(axis=0)
        # and get the iou amount of coverage for each gt box
        max_overlaps = overlaps.max(axis=0)
        # find which gt box is 'best' covered (i.e. 'best' = most iou)
        gt_ind = max_overlaps.argmax()
        gt_ovr = max_overlaps.max()
        assert (gt_ovr >= 0)
        # find the proposal box that covers the best covered gt box
        box_ind = argmax_overlaps[gt_ind]
        # record the iou coverage of this gt box
        _gt_overlaps[j] = overlaps[box_ind, gt_ind]
        assert (_gt_overlaps[j] == gt_ovr)
        # mark the proposal box and the gt box as used
        overlaps[box_ind, :] = -1
        overlaps[:, gt_ind] = -1
      # append recorded iou coverage level
      gt_overlaps = np.hstack((gt_overlaps, _gt_overlaps))

    gt_overlaps = np.sort(gt_overlaps)
    if thresholds is None:
      step = 0.05
      thresholds = np.arange(0.5, 0.95 + 1e-5, step)
    recalls = np.zeros_like(thresholds)
    # compute recall for each iou threshold
    for i, t in enumerate(thresholds):
      recalls[i] = (gt_overlaps >= t).sum() / float(num_pos)
    # ar = 2 * np.trapz(recalls, thresholds)
    ar = recalls.mean()
    return {'ar': ar, 'recalls': recalls, 'thresholds': thresholds,
            'gt_overlaps': gt_overlaps}
S1 = np.eye(2)
S2 = np.array([[1, 0.95], [0.95, 1]])
m1 = np.array([0.75, 0]).reshape(-1, 1)
m2 = np.array([-0.75, 0])
xx = np.repeat(m1, n1).reshape(2, n1)
yy = np.repeat(m2, n2).reshape(2, n2)
x1 = np.linalg.cholesky(S1).T @ np.random.randn(2,n1) + xx
x2 = np.linalg.cholesky(S2).T @ np.random.randn(2,n2) + yy
x = np.concatenate([x1.T, x2.T])
y1 = -np.ones(n1).reshape(-1, 1)
y2 = np.ones(n2).reshape(-1, 1)
y = np.concatenate([y1, y2])
q = np.linspace(-4, 4, 81)
r = np.linspace(-4, 4, 81)
t1, t2 = np.meshgrid(q, r)
tgrid = np.hstack([t1.reshape(-1, 1), t2.reshape(-1, 1)])

class GPClassificationModel(AbstractVariationalGP):
    def __init__(self, train_x):
        variational_distribution = CholeskyVariationalDistribution(train_x.size(0))
        variational_strategy = VariationalStrategy(self, train_x, variational_distribution)
        super(GPClassificationModel, self).__init__(variational_strategy)
        self.mean_module = gpytorch.means.ConstantMean()
        self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel())
    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        latent_pred = gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
        return latent_pred

def train(model, init_lengthscale, init_sigmaf_2):
Esempio n. 44
0
    decoder = Decoder(input_size = 300)

    print(" [*] Decoding result")
    conf_pred = np.stack([one_hot(target, 9).numpy() for target in conf_target])
    conf_pred = conf_pred.reshape(-1, 9)
    output = decoder((torch.FloatTensor(conf_pred), loc_target))

    all_boxes = [[[] for _ in range(1)] for _ in range(9)]
    for j in range(1, 9):
        dets = output[0, j, :]
        mask = dets[:, 0].gt(0.).expand(5, dets.size(0)).t()
        dets = torch.masked_select(dets, mask).view(-1, 5)
        if dets.dim() == 0:
            continue
        boxes = dets[:, 1:]
        scores = dets[:, 0].cpu().numpy()
        cls_dets = np.hstack((boxes.cpu().numpy(), scores[:, np.newaxis])).astype(np.float32, copy = False)
        all_boxes[j][0] = cls_dets

    #print(" [*] label:")
    #print(output[:, 0])
    print(" [*] box:")
    print(output)







def train():
    # data load
    voca, sentence_index = data_load()
    voca_num = len(voca)

    # write vocabrary lists
    pickle.dump(voca, open("vocabrary_word2vec.bn", "wb"))

    print("vocabrary num:", voca_num)
    print("e.x.", voca[:5])
    
    # model
    model = Word2Vec(voca_num, dim=hidden_dim).to(device)

    # minibatch index
    mbi = 0

    data_num = len(sentence_index)
    train_ind = np.arange(data_num)
    np.random.seed(0)
    np.random.shuffle(train_ind)

    # loss function
    loss_fn = torch.nn.NLLLoss()
    
    # each learning rate and iteration
    for lr, ite in train_factors:
        print("lr", lr, " ite", ite)

        # optimizer
        if opt == "SGD":
            optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9)
        elif opt == "Adam":
            optimizer = torch.optim.Adam(model.parameters(), lr=lr)
        else:
            raise Exception("invalid optimizer:", opt)
        
        # each iteration
        for ite in range(ite):
            # get minibatch index
            if mbi + mb > data_num:
                mb_ind = copy(train_ind[mbi:])
                np.random.shuffle(train_ind)
                mb_ind = np.hstack((mb_ind, train_ind[:(mb-(data_num-mbi))]))
            else:
                mb_ind = train_ind[mbi: mbi+mb]
                mbi += mb

            # get minibatch
            X_inds = [sentence_index[i] for i in mb_ind]

            loss = 0
            accuracy = 0.
            total_len = 0

            # each data of minibatch
            for mb_index in range(mb):
                # 1 data of minibatch
                Xs = np.array(X_inds[mb_index]).reshape([-1, 1])

                input_X = np.zeros([1, voca_num])
                input_X[:, Xs[C]] = 1
                input_X = torch.tensor(input_X, dtype=torch.float).to(device)
                
                # reset graph
                optimizer.zero_grad()
            
                # data length
                total_len += x_length

                # forward network
                ys = model(input_X)

                # target label index
                t_inds = [_i for _i in range(x_length) if _i != C]

                # each target label
                for i, y in zip(t_inds, ys):
                    # target label
                    t = torch.tensor(Xs[i], dtype=torch.long).to(device)

                    # get loss
                    loss += loss_fn(torch.log(y), t)

                    # count accuracy
                    if y.argmax() == t:
                        accuracy += 1

                """
                # each target label
                for i in range(x_length):
                    # forward network
                    y = model(input_X)

                    # target label
                    t = torch.tensor(Xs[i], dtype=torch.long).to(device)
                    #t = torch.tensor(Xs[i], dtype=torch.long).to(device).view(-1, voca_num)

                    # get loss
                    loss += loss_fn(torch.log(y), t)

                    # count accuracy
                    if y.argmax() == t:
                        accuracy += 1
                """

            # loss backward
            loss.backward()

            # update weight
            optimizer.step()
            
            # get loss
            loss = loss.item() / total_len
            accuracy = accuracy / total_len

            if (ite + 1) % 10 == 0:
                print("iter :", ite+1, ",loss >>:", loss, "accuracy:", accuracy)

    torch.save(model.state_dict(), 'word2vec.pt')
Esempio n. 46
0
def match_prediction():
    match, stat = load_data()
    teams = calc_score()
    teams_train = {}

    for team in teams:
        teams_train[team] = {'x': [], 'y': []}
        for i in range(len(teams[team]) - TRAINSIZE):
            teams_train[team]['x'].append(teams[team][i:i + TRAINSIZE])
            teams_train[team]['y'].append(teams[team][i + TRAINSIZE])
        teams_train[team]['x'] = np.array(teams_train[team]['x'])
        teams_train[team]['y'] = np.array(teams_train[team]['y'])

    x_train = []
    y_train = []

    for a in teams_train.values():
        x_train.append(a['x'])
        y_train.append(a['y'])

    x_train = np.vstack(x_train)
    y_train = np.hstack(y_train)
    #
    x_all = np.concatenate((x_train, y_train.reshape(-1, 1)), axis=1)
    sc = StandardScaler()
    x_all = sc.fit_transform(x_all)
    x_train2 = x_all[:, :TRAINSIZE]
    y_train2 = x_all[:, TRAINSIZE]

    #x_train = teams_train['26']['x']
    #y_train = teams_train['26']['y']

    model = Sequential()
    model.add(LSTM(32, input_shape=(TRAINSIZE, 1), return_sequences=True))
    model.add(Dropout(0.1))
    #model.add(LSTM(32, return_sequences=True))
    #model.add(Dropout(0.1))
    model.add(LSTM(32, return_sequences=False))
    model.add(Dropout(0.1))
    model.add(Dense(1, activation='linear'))
    start = time.time()
    adam = keras.optimizers.Adam(learning_rate=0.0005,
                                 beta_1=0.9,
                                 beta_2=0.999,
                                 amsgrad=False)
    nadam = keras.optimizers.Nadam(learning_rate=0.002,
                                   beta_1=0.9,
                                   beta_2=0.999)
    sgd = keras.optimizers.SGD(learning_rate=0.001,
                               momentum=0.0,
                               nesterov=False)
    model.compile(loss="mse", optimizer=adam)
    print("Compilation Time : ", time.time() - start)
    model.fit(x_train2.reshape(x_train2.shape + (1, )),
              y_train2,
              batch_size=16,
              epochs=10,
              validation_split=0.05)

    plt.figure()
    predict = model.predict(x_train2.reshape(x_train2.shape + (1, )))
    plt.plot(y_train2)
    plt.plot(predict)

    class_data, class_label = make_class_data(model, match, teams)

    s4teams = []
    for a in list(match.values())[-380:]:
        if a['home_team_id'] not in s4teams:
            s4teams.append(a['home_team_id'])

    s4_teams = {}
    for a in teams:
        if a in s4teams:
            if len(teams[a]) == 38:
                s4_teams[a] = [46, 46, 46, 46, 46] + teams[a]
            else:
                s4_teams[a] = teams[a][-43:]

    teamsi = dict.fromkeys(s4_teams.keys(), 0)
    test_data = []
    test_label = []
    for a in list(match.values())[-380:]:
        home = a['home_team_id']
        away = a['away_team_id']
        if int(a['full_time_score'].split(':')[0]) > int(
                a['full_time_score'].split(':')[1]):
            y = [1, 0, 0]
        elif int(a['full_time_score'].split(':')[0]) == int(
                a['full_time_score'].split(':')[1]):
            y = [0, 1, 0]
        else:
            y = [0, 0, 1]
        ihome = teamsi[home]
        iaway = teamsi[away]
        h = np.array(s4_teams[home][ihome:ihome + TRAINSIZE] + [0]).reshape(
            -1, 1)
        a = np.array(s4_teams[away][iaway:iaway + TRAINSIZE] + [0]).reshape(
            -1, 1)
        h = sc.transform(h.reshape(1, -1))
        a = sc.transform(a.reshape(1, -1))
        h_score = model.predict(
            h.reshape(-1, 1)[:TRAINSIZE].reshape(1, TRAINSIZE, 1))
        a_score = model.predict(
            a.reshape(-1, 1)[:TRAINSIZE].reshape(1, TRAINSIZE, 1))
        test_data.append([h_score[0], a_score[0]])
        test_label.append(y)
        teamsi[home] += 1
        teamsi[away] += 1
    test_data = np.array(test_data)
    test_label = np.array(test_label)

    clf = RandomForestClassifier()
    clf = clf.fit(class_data[:1300].reshape(1300, 2), class_label[:1300])
    score = clf.score(test_data.reshape(380, 2), test_label)
    print(score)
def exp_hog_parameters(args, var, to_log=True):
    # define feature parameters
    cell_per_block  = var['cell_per_block'] # 2
    color_space     = var['color_space']    # can be RGB, HSV, LUV, HLS, YUV, YCrCb
    hist_bins       = var['hist_bins']      # 32  # number of histogram bins
    hist_feat       = var['hist_feat']      # histogram features on or off
    hog_channel     = var['hog_channel']    # 'ALL' # can be 0, 1, 2, or 'ALL'
    hog_feat        = var['hog_feat']       # HOG features on or off
    orient          = var['orient']         # 8
    overlap         = var['overlap']        # 0.5
    pix_per_cell    = var['pix_per_cell']   # 8
    scale           = var['scale']          # 1.0
    spatial_feat    = var['spatial_feat']   # True, spatial features on or off
    spatial_size    = var['spatial_size']   # (32,32)  # spatial binning dimensions
    x_start_stop    = var['x_start_stop']   # [None, None]
    y_start_stop    = var['y_start_stop']   # [400, 656]
    xy_window       = var['xy_window']      # (128, 128)

    # list_all_images
    cars, notcars = list_all_images(args)

    # choose random car/notcar indices
    flag_random = False
    if flag_random:
        car_ind = np.random.randint(0, len(cars))
        notcar_ind = np.random.randint(0, len(notcars))
    else:
        car_ind, notcar_ind = 2734, 7868

    # read in car / notcar images
    car_image = mpimg.imread(cars[car_ind])
    notcar_image = mpimg.imread(notcars[notcar_ind])

    num_img = 5
    flag_random = False
    if flag_random:
        cars_image_to_plot = [[cars[index].split('\\')[-1][:-4], cars[index]] for index in
                              [random.randint(0, len(cars)) for i in range(num_img)]]
        notcars_image_to_plot = [[notcars[index].split('\\')[-1][:-4], notcars[index]] for index in
                                 [random.randint(0, len(notcars)) for i in range(num_img)]]
    else:
        cars_image_to_plot = [[index, cars[index]] for index in
                              [random.randint(0, len(cars)) for i in range(num_img)]]
        notcars_image_to_plot = [[index, notcars[index]] for index in
                                 [random.randint(0, len(notcars)) for i in range(num_img)]]

    car_features, car_hog_image = single_img_features(car_image, color_space=color_space, spatial_size=spatial_size,
                                                      hist_bins=hist_bins, orient=orient, pix_per_cell=pix_per_cell,
                                                      cell_per_block=cell_per_block, hog_channel=hog_channel,
                                                      spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat,
                                                      vis=True)

    notcar_features, notcar_hog_image = single_img_features(notcar_image, color_space=color_space,
                                                            spatial_size=spatial_size,
                                                            hist_bins=hist_bins, orient=orient,
                                                            pix_per_cell=pix_per_cell,
                                                            cell_per_block=cell_per_block, hog_channel=hog_channel,
                                                            spatial_feat=spatial_feat, hist_feat=hist_feat,
                                                            hog_feat=hog_feat,
                                                            vis=True)

    t            = time.time()
    n_samples    = 1000
    random_idxs  = np.random.randint(0, len(cars), n_samples)
    test_cars    = np.array(cars)[random_idxs]
    test_noncars = np.array(notcars)[random_idxs]

    car_features = extract_features(test_cars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins,
                                    orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
                                    hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)

    notcar_features = extract_features(test_noncars, color_space=color_space, spatial_size=spatial_size, hist_bins=hist_bins,
                                    orient=orient, pix_per_cell=pix_per_cell, cell_per_block=cell_per_block,
                                    hog_channel=hog_channel, spatial_feat=spatial_feat, hist_feat=hist_feat, hog_feat=hog_feat)

    t_feature_computation = round(time.time() - t, 2)
    print(t_feature_computation, 'Seconds to compute features...')
    X = np.vstack((car_features, notcar_features)).astype(np.float64)
    # fit a per_column scaler
    X_scaler = StandardScaler().fit(X)
    # apply the scaler to X
    scaled_X = X_scaler.transform(X)

    # define the labels vector
    y = np.hstack(( np.ones(len(car_features)), np.zeros(len(notcar_features)) ))

    # split up data into randomized training and test sets
    rand_state = np.random.randint(0, 100)
    X_train, X_test, y_train, y_test = train_test_split(scaled_X, y, test_size=0.1, random_state=rand_state)

    print('Using:', orient, 'orientations,', pix_per_cell, 'pixels per cell,', cell_per_block,
          'cells per block,', hist_bins, 'histogram bins, and', spatial_size, 'spatial sampling')
    print('Feature vector length:', len(X_train[0]))

    # use a linear SVC
    svc = LinearSVC()
    # check the training time for the SVC
    t   = time.time()

    svc.fit(X_train, y_train) # https://stackoverflow.com/questions/40524790/valueerror-this-solver-needs-samples-of-at-least-2-classes-in-the-data-but-the
    t_train = round(time.time()-t, 2)
    print(t_train, 'Seconds to train SVC...')
    # check the score of the SVC
    accuracy = round(svc.score(X_test, y_test), 4)
    print('Test Accuracy of SVC = ', accuracy)

    log = [ cell_per_block, color_space, hist_bins,
            hist_feat, hog_channel, orient,
            pix_per_cell, spatial_feat, spatial_size,
            accuracy, len(X_train[0]), t_feature_computation, t_train, t_feature_computation+t_train  ]

    # log = [ var['cell_per_block'], var['color_space'], var['hist_bins'],
    #         var['hist_feat'], var['hog_channel'], var['orient'],
    #         var['pix_per_cell'], var['spatial_feat'], var['spatial_size'],
    #         accuracy, len(X_train[0]), t_feature_computation, t_train, t_feature_computation+t_train  ]

    if to_log: log_write(args, log)
Esempio n. 48
0
print(
    len(overlap_idx), 'out of',
    len(missed_idx[(len(missed_idx) - 1) - iter_idx]),
    'misclassifications were overlapped between logistic regression and ResNet9\n\n'
)

missed_df = pd.DataFrame(X_test[overlap_idx], columns=df.columns)
missed_df.insert(loc=0, column='labels', value=y_test[overlap_idx])

# ## Stacking

# ### Create Training Set for Meta-Learner

X_train_meta = np.hstack(
    (log_y_prob.reshape(len(log_y_prob),
                        1), rfc_y_prob.reshape(len(rfc_y_prob), 1),
     knn_y_prob.reshape(len(knn_y_prob),
                        1), res_y_prob.reshape(len(res_y_prob), 1)))
y_train_meta = y_test

pos = []
neg = []

for i in range(len(y_train_meta)):
    if y_train_meta[i] > 0:
        pos.append(X_train_meta[i, :])
    else:
        neg.append(X_train_meta[i, :])

pos = np.asarray(pos)
neg = np.asarray(neg)
Esempio n. 49
0
    ret, frame = cap.read()

    M = frame.shape[0]
    N = frame.shape[1]

    frame = cv2.resize(frame, (int(ratio*N),int(ratio*M)))
    filterSz = 5
    frameBl = cv2.blur(frame,(filterSz,filterSz))
    labImg = cv2.cvtColor(frameBl, cv2.COLOR_BGR2LAB)
    hsvImg = cv2.cvtColor(frameBl, cv2.COLOR_BGR2HSV)

    [Blue,G,R] = cv2.split(frame)
    [L,A,B] = cv2.split(labImg)
    [H,S,V] = cv2.split(hsvImg)

    rgbMerge = np.hstack((R,G,Blue))
    labMerge = np.hstack((L,A,B))
    hsvMerge = np.hstack((H,S,V))

    finMerge = np.vstack((rgbMerge,labMerge, hsvMerge))





    cv2.imshow('frame-Binary',finMerge)


    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    reg2 = LinearRegression().fit(W2, Y)
    print("linear regression: {:.3f}".format((int_x[1] - int_x[0]) * reg1.coef_[0] * reg2.coef_[0][0]))

    ####################### instrumental variable ##################
    X, Y, Z = data[:, 0][:], data[:, 1], data[:, 2]
    array_YZ = np.vstack((Y, Z))
    cov_YZ = np.cov(array_YZ)[0][1]
    array_XZ = np.vstack((X, Z))
    cov_XZ = np.cov(array_XZ)[0][1]
    print("instrumental variable: {:.3f}".format((int_x[1] - int_x[0]) * cov_YZ / cov_XZ))

    ###################### propensity score ########################
    X, Y, Z = data[:, 0][:, np.newaxis], data[:, 1][:, np.newaxis], data[:, 2][:, np.newaxis]
    W1, W2 = data[:, 3][:, np.newaxis], data[:, 4][:, np.newaxis]
    #### step 1 ####
    fan = np.hstack((np.ones_like(X), Z, W1, W2))
    reg1 = LinearRegression(fit_intercept=False).fit(fan, X)
    Beta = reg1.coef_.transpose()
    variance = np.mean((X - np.dot(fan, Beta))**2)
    R = normal(X, X - np.dot(fan, Beta), variance)
    #### step 2 ####
    inpt = np.hstack((np.ones_like(X), X,  R, X*R))
    reg2 = LinearRegression(fit_intercept=False).fit(inpt, Y)
    Alpha = reg2.coef_.transpose()
    #### step3 ####
    t0 = int_x[0] * np.ones_like(X)
    r0 = normal(t0, np.dot(fan, Beta), variance)
    inpt0 = np.hstack((np.ones_like(X), t0, r0, t0*r0))
    y0 = np.mean(np.dot(inpt0, Alpha))

    t1 = int_x[1] * np.ones_like(X)
Esempio n. 51
0
def get_deflection(self, *args, label, index, indices, field_name):
    """Get the vector field, the field name and the corresponding pyvista mesh in format adapted to a glyph plot.

    Parameters
    ----------
    self : MeshSolution
        a MeshSolution object
    *args: list of strings
        List of axes requested by the user, their units and values (optional)
    label : str
        a label
    index : int
        an index
    indices : list
        list of the points to extract (optional)
    field_name : str
        title of the field to display on plot

    Returns
    -------
    vect_field : ndaray
        vector field to plot
    field_normal_amp : ndarray
        scalar field which is the normal/radial component of vect_field (to adjust the colormap)
    field_name : str
        name of the field
    mesh_pv : UnstructuredGrid
        pyvista mesh to plot
    """
    # Get mesh_pv and field

    # Try to get normal field, else use radial
    try:
        mesh_pv, field_normal_amp, field_name = self.get_mesh_field_pv(
            *args,
            label=label,
            index=index,
            indices=indices,
            field_name=field_name,
            is_normal=True,
        )
    except:
        mesh_pv, field_normal_amp, field_name = self.get_mesh_field_pv(
            *args,
            label=label,
            index=index,
            indices=indices,
            field_name=field_name,
            is_radial=True,
        )

    _, vect_field, _ = self.get_mesh_field_pv(
        *args,
        label=label,
        index=index,
        indices=indices,
        field_name=field_name,
    )

    solution = self.get_solution(
        label=label,
        index=index,
    )

    axes_list = solution.get_axes_list(*args)

    # Add third dimension if needed
    ind = axes_list[0].index("component")
    if axes_list[1][ind] == 2:
        vect_field = hstack((vect_field, zeros((vect_field.shape[0], 1))))

    if field_name is None:
        if label is not None:
            field_name = label
        elif self.get_solution(index=index).label is not None:
            field_name = self.get_solution(index=index).label
        else:
            field_name = "Field"

    return vect_field, field_normal_amp, field_name, mesh_pv
Esempio n. 52
0
def uniform_init(x_low, x_high, y_low, y_high, v_x, v_y, n):
    return np.hstack(
        (np.random.uniform(x_low, x_high,
                           (n, 1)), np.random.uniform(y_low, y_high, (n, 1)),
         np.random.uniform(-v_x, v_x,
                           (n, 1)), np.random.uniform(-v_y, v_y, (n, 1))))
Esempio n. 53
0
def loadBlender(fname_blender: str):
    """
    Compile and return the fragments and collision shapes from ``fname``.

    The ``fname`` file must specify a Blender file.

    :param str fname: Blender file name
    :return: (frags dict, cshapes dict)
    """
    # Use Blender itself to load the Blender file. This will trigger a script
    # inside Blender that creates a JSON file with custom scene data. The name
    # of that file is returned in 'fname_az'.
    fname_az = processBlenderFile(fname_blender)

    # Load the collision shape data and geometry.
    pp = pyassimp.postprocess
    azFile = json.loads(open(fname_az, 'rb').read().decode('utf8'))
    scene = pyassimp.load(fname_blender, pp.aiProcess_Triangulate)
    del fname_az, pp

    # Sanity check: both must contain information about the exact same objects.
    assert set(azFile.keys()) == {_.name for _ in scene.meshes}

    # Load all the materials.
    materials = loadMaterials(scene, fname_blender)

    # Compile fragments and collision shapes for Azrael.
    frags, cshapes = {}, {}
    for mesh in scene.meshes:
        # Get the material for this mesh.
        material = materials[mesh.materialindex]

        # Iterate over the faces and rebuild the vertices.
        azData = azFile[mesh.name]
        vert = np.zeros(len(mesh.faces) * 9)
        for idx, face in enumerate(mesh.faces):
            tmp = [mesh.vertices[_] for _ in face]
            start, stop = idx * 9, (idx + 1) * 9
            vert[start:stop] = np.hstack(tmp)
            del start, stop, tmp, idx, face

        # Copy the UV coordinates, if there are any.
        if len(mesh.texturecoords) > 0:
            uv = np.zeros(len(mesh.faces) * 6)
            for idx, face in enumerate(mesh.faces):
                tmp = [mesh.texturecoords[0][_, :2] for _ in face]
                start, stop = 6 * idx, 6 * (idx + 1)
                uv[start:stop] = np.hstack(tmp)
                del start, stop, tmp, idx, face

            # Sanity check.
            assert (len(uv) // 2 == len(vert) // 3)
        else:
            uv = 0.5 * np.ones(2 * (len(vert) // 3))
            uv = []

        # Azrael does not allow 'dots', yet Bullet uses it prominently to name
        # objects. As a quick fix, simply replace the dots with something else.
        # fixme: should be redundant once #149 (https://trello.com/c/wcHX3qGd)
        # is implemented.
        azname = mesh.name.replace('.', 'x')

        # Unpack the position and orientation of the mesh in world coordinates.
        pos, rot, dim = azData['pos'], azData['rot'], np.array(azData['dimensions'])

        # Unpack the interior points and put them into any kind of byte string.
        # This will be attached as yet another file to the fragment data.
        interior_points = json.dumps(azData['interior_points']).encode('utf8')

        # Create a RAW fragment.
        scale = 1
        rgb, width, height = material['RGB'], material['width'], material['height']
        frag = demolib.getFragMetaRaw(vert, uv, rgb, scale, pos, rot)
        frag.files['interior_points'] = interior_points
        frags[azname] = frag

        # Construct the BOX collision shape based on the Blender dimensions.
        hlen_x, hlen_y, hlen_z = (dim / 2).tolist()
        box = aztypes.CollShapeBox(hlen_x, hlen_y, hlen_z)

        # Construct the CollshapeMeta data.
        cshapes[azname] = aztypes.CollShapeMeta('box', pos, rot, box)
        del azData, vert, dim, scale, rgb, width, height, hlen_x, hlen_y, hlen_z
    return frags, cshapes
plt.xlabel("Step number")
plt.ylabel("MCMC chains of c")
plt.tight_layout()

plt.show()


# computing Medians for true values
medians = np.median(samples, axis=0)

a_true, b_true, c_true = np.average(medians,axis=0)  # computing the the value of the parameters by averaging over the 50 chains

data = np.zeros([4000 * 50, 3])

for i in range(3):
    data[:, i] = np.hstack(samples[:, :, i])

# We can plot the posterior PDF using the corner library.

fig = corner.corner(samples.reshape(40 * 5000, 3), labels=["a", "b", "c"], quantiles=[0.16, 0.5, 0.84],
                    truths=[a_true, b_true, c_true], show_titles=True)
plt.show()

X = np.linspace(0, 300, 1000)

for i in range(200):
    r = np.random.randint(0, high=4000 * 50)
    a = data[r, 0]
    b = data[r, 1]
    c = data[r, 2]
    plt.plot(X, a * X ** 2 + b * X + c)
Esempio n. 55
0
# 勾勒轮廓,为了接下来处理
cv2.drawContours(image_,contours,2,(0, 0, 255),2)
cv2.drawContours(image_,contours,4,(0, 0, 255),2)
cv2.drawContours(image_,contours,6,(0, 0, 255),2)

cv2.imshow("demo",image_)
cv2.waitKey(0)


# # 2 自定义用hough解决
contour = image_[:,:,2]
mask = cv2.inRange(contour,254,255)
cv2.imshow("demo",mask)
cv2.waitKey(0)
mask_canny = cv2.Canny(mask,100,30)
dst = np.hstack((mask_canny,mask))
cv2.namedWindow("circle")
cv2.createTrackbar('param1','circle',120,200,nothing)
cv2.createTrackbar('param2','circle',12,200,nothing)
cv2.createTrackbar('minRadius','circle',22,30,nothing)
cv2.createTrackbar('maxRadius','circle',43,100,nothing)

while (True):
    image1 = np.copy(img)
    # 137 33 24 44
    param1 = cv2.getTrackbarPos('param1', 'circle')
    param2 = cv2.getTrackbarPos('param2', 'circle')
    minRadius = cv2.getTrackbarPos('minRadius', 'circle')
    maxRadius = cv2.getTrackbarPos('maxRadius', 'circle')
    circle = cv2.HoughCircles(mask,cv2.HOUGH_GRADIENT,1,5,param1=param1,param2=param2,minRadius=minRadius,maxRadius=maxRadius)
Esempio n. 56
0
 def get_current_visual(self, n):
     real_A = tensor2img(self.real_A, n)
     real_B = tensor2img(self.real_B, n)
     fake_B = tensor2img(self.fake_B, n)
     return np.hstack([real_A, real_B, fake_B])
Esempio n. 57
0
    def compute_rand_index_ijv(self, gt_ijv, test_ijv, shape):
        '''Compute the Rand Index for an IJV matrix

        This is in part based on the Omega Index:
        Collins, "Omega: A General Formulation of the Rand Index of Cluster
        Recovery Suitable for Non-disjoint Solutions", Multivariate Behavioral
        Research, 1988, 23, 231-242

        The basic idea of the paper is that a pair should be judged to
        agree only if the number of clusters in which they appear together
        is the same.
        '''
        #
        # The idea here is to assign a label to every pixel position based
        # on the set of labels given to that position by both the ground
        # truth and the test set. We then assess each pair of labels
        # as agreeing or disagreeing as to the number of matches.
        #
        # First, add the backgrounds to the IJV with a label of zero
        #
        gt_bkgd = numpy.ones(shape, bool)
        gt_bkgd[gt_ijv[:, 0], gt_ijv[:, 1]] = False
        test_bkgd = numpy.ones(shape, bool)
        test_bkgd[test_ijv[:, 0], test_ijv[:, 1]] = False
        gt_ijv = numpy.vstack([
            gt_ijv,
            numpy.column_stack([numpy.argwhere(gt_bkgd),
                                numpy.zeros(numpy.sum(gt_bkgd), gt_bkgd.dtype)])])
        test_ijv = numpy.vstack([
            test_ijv,
            numpy.column_stack([numpy.argwhere(test_bkgd),
                                numpy.zeros(numpy.sum(test_bkgd), test_bkgd.dtype)])])
        #
        # Create a unified structure for the pixels where a fourth column
        # tells you whether the pixels came from the ground-truth or test
        #
        u = numpy.vstack([
            numpy.column_stack([gt_ijv, numpy.zeros(gt_ijv.shape[0], gt_ijv.dtype)]),
            numpy.column_stack([test_ijv, numpy.ones(test_ijv.shape[0], test_ijv.dtype)])])
        #
        # Sort by coordinates, then by identity
        #
        order = numpy.lexsort([u[:, 2], u[:, 3], u[:, 0], u[:, 1]])
        u = u[order, :]
        # Get rid of any duplicate labelings (same point labeled twice with
        # same label.
        #
        first = numpy.hstack([[True], numpy.any(u[:-1, :] != u[1:, :], 1)])
        u = u[first, :]
        #
        # Create a 1-d indexer to point at each unique coordinate.
        #
        first_coord_idxs = numpy.hstack([
            [0],
            numpy.argwhere((u[:-1, 0] != u[1:, 0]) |
                           (u[:-1, 1] != u[1:, 1])).flatten() + 1,
            [u.shape[0]]])
        first_coord_counts = first_coord_idxs[1:] - first_coord_idxs[:-1]
        indexes = centrosome.index.Indexes([first_coord_counts])
        #
        # Count the number of labels at each point for both gt and test
        #
        count_test = numpy.bincount(indexes.rev_idx, u[:, 3]).astype(numpy.int64)
        count_gt = first_coord_counts - count_test
        #
        # For each # of labels, pull out the coordinates that have
        # that many labels. Count the number of similarly labeled coordinates
        # and record the count and labels for that group.
        #
        labels = []
        for i in range(1, numpy.max(count_test) + 1):
            for j in range(1, numpy.max(count_gt) + 1):
                match = ((count_test[indexes.rev_idx] == i) &
                         (count_gt[indexes.rev_idx] == j))
                if not numpy.any(match):
                    continue
                #
                # Arrange into an array where the rows are coordinates
                # and the columns are the labels for that coordinate
                #
                lm = u[match, 2].reshape(numpy.sum(match) / (i + j), i + j)
                #
                # Sort by label.
                #
                order = numpy.lexsort(lm.transpose())
                lm = lm[order, :]
                #
                # Find indices of unique and # of each
                #
                lm_first = numpy.hstack([
                    [0],
                    numpy.argwhere(numpy.any(lm[:-1, :] != lm[1:, :], 1)).flatten() + 1,
                    [lm.shape[0]]])
                lm_count = lm_first[1:] - lm_first[:-1]
                for idx, count in zip(lm_first[:-1], lm_count):
                    labels.append((count,
                                   lm[idx, :j],
                                   lm[idx, j:]))
        #
        # We now have our sets partitioned. Do each against each to get
        # the number of true positive and negative pairs.
        #
        max_t_labels = reduce(max, [len(t) for c, t, g in labels], 0)
        max_g_labels = reduce(max, [len(g) for c, t, g in labels], 0)
        #
        # tbl is the contingency table from Table 4 of the Collins paper
        # It's a table of the number of pairs which fall into M sets
        # in the ground truth case and N in the test case.
        #
        tbl = numpy.zeros(((max_t_labels + 1), (max_g_labels + 1)))
        for i, (c1, tobject_numbers1, gobject_numbers1) in enumerate(labels):
            for j, (c2, tobject_numbers2, gobject_numbers2) in \
                    enumerate(labels[i:]):
                nhits_test = numpy.sum(
                    tobject_numbers1[:, numpy.newaxis] ==
                    tobject_numbers2[numpy.newaxis, :])
                nhits_gt = numpy.sum(
                    gobject_numbers1[:, numpy.newaxis] ==
                    gobject_numbers2[numpy.newaxis, :])
                if j == 0:
                    N = c1 * (c1 - 1) / 2
                else:
                    N = c1 * c2
                tbl[nhits_test, nhits_gt] += N

        N = numpy.sum(tbl)
        #
        # Equation 13 from the paper
        #
        min_JK = min(max_t_labels, max_g_labels) + 1
        rand_index = numpy.sum(tbl[:min_JK, :min_JK] * numpy.identity(min_JK)) / N
        #
        # Equation 15 from the paper, the expected index
        #
        e_omega = numpy.sum(numpy.sum(tbl[:min_JK, :min_JK], 0) *
                            numpy.sum(tbl[:min_JK, :min_JK], 1)) / N ** 2
        #
        # Equation 16 is the adjusted index
        #
        adjusted_rand_index = (rand_index - e_omega) / (1 - e_omega)
        return rand_index, adjusted_rand_index
Esempio n. 58
0
for i in eigen_pairs:
    print(i[0])

##Get the k value for reduction
sum_eigVal = np.sum(eigen_val)
arry = []
for i in eigen_val:
    if (i / sum_eigVal > 0.02):
        arry.append(i)
type(arry)
dim = len(arry)

#TODO
matrix_res = []
for i in range(0, dim):
    matrix_res.append(np.hstack(eigen_pairs[i][1].reshape(cols, 1)))

print('Matrix W:\n', matrix_res)
type(matrix_res)
matrix1 = np.asmatrix(matrix_res).T

pca_reduced = matrix1.T.dot(samples)

print(pca_reduced)
pca_reduced.shape
pca_reduced = pca_reduced.T
pca_reduced.shape
pca_reduced = pd.DataFrame(pca_reduced)
type(pca_reduced)
fig = plt.figure()
fig.add_subplot(1, 1, 1)
Esempio n. 59
0
    def measure_objects(self, workspace):
        image_set = workspace.image_set
        object_name_GT = self.object_name_GT.value
        objects_GT = workspace.get_objects(object_name_GT)
        iGT, jGT, lGT = objects_GT.ijv.transpose()
        object_name_ID = self.object_name_ID.value
        objects_ID = workspace.get_objects(object_name_ID)
        iID, jID, lID = objects_ID.ijv.transpose()
        ID_obj = 0 if len(lID) == 0 else max(lID)
        GT_obj = 0 if len(lGT) == 0 else max(lGT)

        xGT, yGT = objects_GT.shape
        xID, yID = objects_ID.shape
        GT_pixels = numpy.zeros((xGT, yGT))
        ID_pixels = numpy.zeros((xID, yID))
        total_pixels = xGT * yGT

        GT_pixels[iGT, jGT] = 1
        ID_pixels[iID, jID] = 1

        GT_tot_area = len(iGT)
        if len(iGT) == 0 and len(iID) == 0:
            intersect_matrix = numpy.zeros((0, 0), int)
        else:
            #
            # Build a matrix with rows of i, j, label and a GT/ID flag
            #
            all_ijv = numpy.column_stack(
                    (numpy.hstack((iGT, iID)),
                     numpy.hstack((jGT, jID)),
                     numpy.hstack((lGT, lID)),
                     numpy.hstack((numpy.zeros(len(iGT)), numpy.ones(len(iID))))))
            #
            # Order it so that runs of the same i, j are consecutive
            #
            order = numpy.lexsort((all_ijv[:, -1], all_ijv[:, 0], all_ijv[:, 1]))
            all_ijv = all_ijv[order, :]
            # Mark the first at each i, j != previous i, j
            first = numpy.where(numpy.hstack(
                    ([True],
                     ~ numpy.all(all_ijv[:-1, :2] == all_ijv[1:, :2], 1),
                     [True])))[0]
            # Count # at each i, j
            count = first[1:] - first[:-1]
            # First indexer - mapping from i,j to index in all_ijv
            all_ijv_map = centrosome.index.Indexes([count])
            # Bincount to get the # of ID pixels per i,j
            id_count = numpy.bincount(all_ijv_map.rev_idx,
                                   all_ijv[:, -1]).astype(int)
            gt_count = count - id_count
            # Now we can create an indexer that has NxM elements per i,j
            # where N is the number of GT pixels at that i,j and M is
            # the number of ID pixels. We can then use the indexer to pull
            # out the label values for each to populate a sparse array.
            #
            cross_map = centrosome.index.Indexes([id_count, gt_count])
            off_gt = all_ijv_map.fwd_idx[cross_map.rev_idx] + cross_map.idx[0]
            off_id = all_ijv_map.fwd_idx[cross_map.rev_idx] + cross_map.idx[1] + \
                     id_count[cross_map.rev_idx]
            intersect_matrix = scipy.sparse.coo_matrix(
                    (numpy.ones(len(off_gt)),
                     (all_ijv[off_id, 2], all_ijv[off_gt, 2])),
                    shape=(ID_obj + 1, GT_obj + 1)).toarray()[1:, 1:]

        gt_areas = objects_GT.areas
        id_areas = objects_ID.areas
        FN_area = gt_areas[numpy.newaxis, :] - intersect_matrix
        all_intersecting_area = numpy.sum(intersect_matrix)

        dom_ID = []

        for i in range(0, ID_obj):
            indices_jj = numpy.nonzero(lID == i)
            indices_jj = indices_jj[0]
            id_i = iID[indices_jj]
            id_j = jID[indices_jj]
            ID_pixels[id_i, id_j] = 1

        for i in intersect_matrix:  # loop through the GT objects first
            if len(i) == 0 or max(i) == 0:
                id = -1  # we missed the object; arbitrarily assign -1 index
            else:
                id = numpy.where(i == max(i))[0][0]  # what is the ID of the max pixels?
            dom_ID += [id]  # for ea GT object, which is the dominating ID?

        dom_ID = numpy.array(dom_ID)

        for i in range(0, len(intersect_matrix.T)):
            if len(numpy.where(dom_ID == i)[0]) > 1:
                final_id = numpy.where(intersect_matrix.T[i] == max(intersect_matrix.T[i]))
                final_id = final_id[0][0]
                all_id = numpy.where(dom_ID == i)[0]
                nonfinal = [x for x in all_id if x != final_id]
                for n in nonfinal:  # these others cannot be candidates for the corr ID now
                    intersect_matrix.T[i][n] = 0
            else:
                continue

        TP = 0
        FN = 0
        FP = 0
        for i in range(0, len(dom_ID)):
            d = dom_ID[i]
            if d == -1:
                tp = 0
                fn = id_areas[i]
                fp = 0
            else:
                fp = numpy.sum(intersect_matrix[i][0:d]) + numpy.sum(intersect_matrix[i][(d + 1)::])
                tp = intersect_matrix[i][d]
                fn = FN_area[i][d]
            TP += tp
            FN += fn
            FP += fp

        TN = max(0, total_pixels - TP - FN - FP)

        def nan_divide(numerator, denominator):
            if denominator == 0:
                return numpy.nan
            return float(numerator) / float(denominator)

        accuracy = nan_divide(TP, all_intersecting_area)
        recall = nan_divide(TP, GT_tot_area)
        precision = nan_divide(TP, (TP + FP))
        F_factor = nan_divide(2 * (precision * recall), (precision + recall))
        true_positive_rate = nan_divide(TP, (FN + TP))
        false_positive_rate = nan_divide(FP, (FP + TN))
        false_negative_rate = nan_divide(FN, (FN + TP))
        true_negative_rate = nan_divide(TN, (FP + TN))
        shape = numpy.maximum(numpy.maximum(
                numpy.array(objects_GT.shape), numpy.array(objects_ID.shape)),
                numpy.ones(2, int))
        rand_index, adjusted_rand_index = self.compute_rand_index_ijv(
                objects_GT.ijv, objects_ID.ijv, shape)
        m = workspace.measurements
        m.add_image_measurement(self.measurement_name(FTR_F_FACTOR), F_factor)
        m.add_image_measurement(self.measurement_name(FTR_PRECISION),
                                precision)
        m.add_image_measurement(self.measurement_name(FTR_RECALL), recall)
        m.add_image_measurement(self.measurement_name(FTR_TRUE_POS_RATE),
                                true_positive_rate)
        m.add_image_measurement(self.measurement_name(FTR_FALSE_POS_RATE),
                                false_positive_rate)
        m.add_image_measurement(self.measurement_name(FTR_TRUE_NEG_RATE),
                                true_negative_rate)
        m.add_image_measurement(self.measurement_name(FTR_FALSE_NEG_RATE),
                                false_negative_rate)
        m.add_image_measurement(self.measurement_name(FTR_RAND_INDEX),
                                rand_index)
        m.add_image_measurement(self.measurement_name(FTR_ADJUSTED_RAND_INDEX),
                                adjusted_rand_index)

        def subscripts(condition1, condition2):
            x1, y1 = numpy.where(GT_pixels == condition1)
            x2, y2 = numpy.where(ID_pixels == condition2)
            mask = set(zip(x1, y1)) & set(zip(x2, y2))
            return list(mask)

        TP_mask = subscripts(1, 1)
        FN_mask = subscripts(1, 0)
        FP_mask = subscripts(0, 1)
        TN_mask = subscripts(0, 0)

        TP_pixels = numpy.zeros((xGT, yGT))
        FN_pixels = numpy.zeros((xGT, yGT))
        FP_pixels = numpy.zeros((xGT, yGT))
        TN_pixels = numpy.zeros((xGT, yGT))

        def maskimg(mask, img):
            for ea in mask:
                img[ea] = 1
            return img

        TP_pixels = maskimg(TP_mask, TP_pixels)
        FN_pixels = maskimg(FN_mask, FN_pixels)
        FP_pixels = maskimg(FP_mask, FP_pixels)
        TN_pixels = maskimg(TN_mask, TN_pixels)
        if self.wants_emd:
            emd = self.compute_emd(objects_ID, objects_GT)
            m.add_image_measurement(
                    self.measurement_name(FTR_EARTH_MOVERS_DISTANCE), emd)

        if self.show_window:
            workspace.display_data.true_positives = TP_pixels
            workspace.display_data.true_negatives = TN_pixels
            workspace.display_data.false_positives = FP_pixels
            workspace.display_data.false_negatives = FN_pixels
            workspace.display_data.statistics = [
                (FTR_F_FACTOR, F_factor),
                (FTR_PRECISION, precision),
                (FTR_RECALL, recall),
                (FTR_FALSE_POS_RATE, false_positive_rate),
                (FTR_FALSE_NEG_RATE, false_negative_rate),
                (FTR_RAND_INDEX, rand_index),
                (FTR_ADJUSTED_RAND_INDEX, adjusted_rand_index)
            ]
            if self.wants_emd:
                workspace.display_data.statistics.append(
                        (FTR_EARTH_MOVERS_DISTANCE, emd))
#Performing meanshift on flatImg
ms1.fit(flat_image)

#(r,g,b) vectors corresponding to the different clusters after meanshift
labels1=ms1.labels_


#Remaining colors after meanshift
cluster_centers1 = ms1.cluster_centers_


#Finding and diplaying the number of clusters
labels_unique1 = np.unique(labels1)
n_clusters_1 = len(labels_unique1)
#print("number of estimated clusters : %d" % n_clusters_1)

# Displaying segmented image
seg1 = cluster_centers1[np.reshape(labels1, originShape[:2])]
segImage1 = seg1.astype(np.uint8)

imageR = cv2.resize(image, (250, 300))
segImage1_R =  cv2.resize(segImage1, (250, 300))
print("Time takend to complete Segmentation is {0}seconds.".format(time.time()-start))

#Result = np.hstack((imageR,segImage1_R,segImage2_R,segImage3_R,segImage4_R))
Result = np.hstack((imageR,segImage1_R))

cv2.imshow('Image', Result)
cv2.waitKey(0)
cv2.destroyAllWindows()