Ejemplo n.º 1
0
    def learn_dictionary (self, images, npatches=50000, niter=1000, njobs=-1):
        """ Learn a Sparse Code dictionary for this ScSPM.

        This method trains a sparse codes dictionary for the ScSPM descriptor
        object. This only needs to be run once before multiple calls to the
        extract() method can be made.

        Arguments:
            images: list, a list of paths to images to use for training.
            npatches: int (default 50000) number of SIFT patches to extract from
                the images to use for training the dictionary.
            niter: int (default 1000), the number of iterations of dictionary
                learning (Lasso) to perform.
            njobs: int (default -1), the number of threads to use. -1 means the
                number of threads will be equal to the number of cores.

        """

        # Get SIFT training patches 
        print('Getting training patches...')
        patches = sw.training_patches(images, npatches, self.psize, self.maxdim,
                                        verbose=True)
        patches = pch.norm_patches(patches)
        print('{0} patches requested, {1} patches found.'.format(npatches,
                patches.shape[0]))
        time.sleep(3) # Give people a chance to see this message
          
        # Learn dictionary
        print('Learning dictionary...')
        self.dic = trainDL(np.asfortranarray(patches.T, np.float64), mode=0,
                       K=self.dsize, lambda1=0.15, iter=niter, numThreads=njobs)
        print('done.')
Ejemplo n.º 2
0
def run_multiprocessing_queue_spams_trainDL(out_queue, *args, **kwargs):
    """
        Designed to run spams.trainDL in a separate process.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Queue to return the resulting dictionary.


        Args:
            out_queue(multiprocessing.Queue):       what will take the returned dictionary from spams.trainDL.
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                          a dictionary of keyword arguments to pass to spams.trainDL.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    out_queue.put(result)
Ejemplo n.º 3
0
def call_spams_trainDL(*args, **kwargs):
    """
        Encapsulates call to spams.trainDL. Ensures copy of results occur just
        in case. Designed to be like the multiprocessing calls.

        Args:
            *args(list):                            a list of position
                                                    arguments to pass to
                                                    spams.trainDL.

            **kwargs(dict):                         a dictionary of keyword
                                                    arguments to pass to
                                                    spams.trainDL.

        Note:
            For legacy.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    result = result.copy()

    return(result)
def getstainMat(I,param,i_0):
	#I : Patch for W estimation
	V,VforW=BLtrans(I,i_0)   #Beer-Lambert law
	#step 2: Sparse NMF factorization (Learning W; V=WH)
	out = suppress_stdout()
	Ws = spams.trainDL(np.asfortranarray(np.transpose(VforW)),**param)
	suppress_stdout(out)
	return Ws
Ejemplo n.º 5
0
def get_staincolor_sparsenmf(v):
    blockPrint()
    D = spams.trainDL(np.transpose(v), **param)
    enablePrint()

    a_arg = np.argsort(np.transpose(D)[:, 1])
    # print(np.transpose(np.transpose(D)[a_arg]))
    return np.transpose(np.transpose(D)[a_arg])
Ejemplo n.º 6
0
 def __init__(self, X, n_components, alpha, save=None):
     X1 = np.asfortranarray(X.T)
     self.X = X1
     self.lambda1 = alpha
     self.d = spams.trainDL(X1, K=n_components, mode=3, modeD=0, numThreads=-1, lambda1=alpha, return_model=False)
     self.d = np.asfortranarray(self.d)
     if save:
         np.save(save, self.d)
def test_trainDL_Memory():
    img_file = 'lena.png'
    try:
        img = Image.open(img_file)
    except:
        print("Cannot load image %s : skipping test" %img_file)
        return None
    I = np.array(img) / 255.
    if I.ndim == 3:
        A = np.asfortranarray(I.reshape((I.shape[0],I.shape[1] * I.shape[2])))
        rgb = True
    else:
        A = np.asfortranarray(I)
        rgb = False

    m = 8
    n = 8
    X = spams.im2col_sliding(A,m,n,rgb)

    X = X - np.tile(np.mean(X,0),(X.shape[0],1))
    X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)),(X.shape[0],1)))
    X = np.asfortranarray(X[:,np.arange(0,X.shape[1],10)],dtype = myfloat)

    param = { 'K' : 200, # learns a dictionary with 100 elements
          'lambda1' : 0.15, 'numThreads' : 4,
          'iter' : 100}

    ############# FIRST EXPERIMENT  ##################
    tic = time.time()
    D = spams.trainDL_Memory(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)

    print('Evaluating cost function...')
    lparam = _extract_lasso_param(param)
    alpha = spams.lasso(X,D = D,**lparam)
    xd = X - D * alpha
    R = np.mean(0.5 * (xd * xd).sum(axis=0) + param['lambda1'] * np.abs(alpha).sum(axis=0))
    print("objective function: %f" %R)
    #* ? DISPLAY

    ############# SECOND EXPERIMENT  ##################
    tic = time.time()
    D = spams.trainDL(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)
    print('Evaluating cost function...')
    alpha = spams.lasso(X,D = D,**lparam)
    xd = X - D * alpha
    R = np.mean(0.5 * (xd * xd).sum(axis=0) + param['lambda1'] * np.abs(alpha).sum(axis=0))
    print("objective function: %f" %R)

    #* ? DISPLAY

    return None
    def learn_D(self, segment_list, k, a=None, batch=False, iter=-5):

        # Horizontal train list
        temp = numpy.hstack(segment_list)
        if a == None:
            a = 1.0 / math.sqrt(temp.shape[0])
        # Learn dictionary
        D = spams.trainDL(numpy.asfortranarray(temp), K=k, lambda1=a, batch=batch, iter=iter, posAlpha=True)
        return D
Ejemplo n.º 9
0
def test_trainDL_Memory():
    img_file = 'lena.png'
    try:
        img = Image.open(img_file)
    except:
        print("Cannot load image %s : skipping test" %img_file)
        return None
    I = np.array(img) / 255.
    if I.ndim == 3:
        A = np.asfortranarray(I.reshape((I.shape[0],I.shape[1] * I.shape[2])))
        rgb = True
    else:
        A = np.asfortranarray(I)
        rgb = False

    m = 8;n = 8;
    X = spams.im2col_sliding(A,m,n,rgb)

    X = X - np.tile(np.mean(X,0),(X.shape[0],1))
    X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)),(X.shape[0],1)))
    X = np.asfortranarray(X[:,np.arange(0,X.shape[1],10)],dtype = myfloat)

    param = { 'K' : 200, # learns a dictionary with 100 elements
          'lambda1' : 0.15, 'numThreads' : 4,
          'iter' : 100}

    ############# FIRST EXPERIMENT  ##################
    tic = time.time()
    D = spams.trainDL_Memory(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)

    print('Evaluating cost function...')
    lparam = _extract_lasso_param(param)
    alpha = spams.lasso(X,D = D,**lparam)
    xd = X - D * alpha
    R = np.mean(0.5 * (xd * xd).sum(axis=0) + param['lambda1'] * np.abs(alpha).sum(axis=0))
    print("objective function: %f" %R)
    #* ? DISPLAY

    ############# SECOND EXPERIMENT  ##################
    tic = time.time()
    D = spams.trainDL(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)
    print('Evaluating cost function...')
    alpha = spams.lasso(X,D = D,**lparam)
    xd = X - D * alpha
    R = np.mean(0.5 * (xd * xd).sum(axis=0) + param['lambda1'] * np.abs(alpha).sum(axis=0))
    print("objective function: %f" %R)

    #* ? DISPLAY

    return None
Ejemplo n.º 10
0
 def DicL(i,output_dir,dicl_dim,migp_dim):
     #load migp data, a subject-by-feature matrix
     ff = output_dir+'/DicL' + str(dicl_dim) + '/DLdata_mod'+('%02d' % (i+1))+'.npy'
     if os.path.exists(ff) == 0:
         dd=np.load(output_dir+'/MIGP' + str(migp_dim) + '/PCAdata_mod'+('%02d' % (i+1))+'.npy') 
         D = spams.trainDL(np.asfortranarray(dd.T,dtype ='float64'),**param) #this is dicl_dim-by-migp_dim    
         np.save(ff,D)
     else:
         D = np.load(ff)
     return D   
Ejemplo n.º 11
0
def getDictionary(Img, patch_size, **param):
    I = np.array(Img) / 255.
    A = np.asfortranarray(I)
    rgb = False
    X = spams.im2col_sliding(A, patch_size, patch_size, rgb)
    X = im2col(Img, (patch_size, patch_size))
    X = X - np.tile(np.mean(X, 0), (X.shape[0], 1))
    X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)), (X.shape[0], 1)), dtype=float)
    D = spams.trainDL(X, **param)
    return D
Ejemplo n.º 12
0
def get_staincolor_sparsenmf(v, **param):

    # Params
    # Learning W through sparse NMF
    Ws = spams.trainDL(v.T, **param)

    # Arranging H stain color vector as first column and then the second column
    # vectors as E stain color vector
    Ws = Ws.T
    Ws = Ws[Ws[:, 1].argsort()]

    return Ws.T
Ejemplo n.º 13
0
def learn_dict(vecs, factor):
    print 'X shape: ' + str(vecs.shape)
    X = vecs
    n_components = len(X)
    X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)),(X.shape[0],1)),dtype = 'float32')
    param = { 'K' : factor*n_components, # learns a dictionary with K elements
          'lambda1' : 0.15, 'numThreads' : -1, 'batchsize' : 50,
          'iter' : 1000}
    
    D = spams.trainDL(X,**param)
    print 'Q shape: ' + str(D.shape)
    return D
def getstainMat(I, param):
    #Source Input
    #I : Patch for W estimation
    V, VforW = BLtrans(I)  #Beer-Lambert law
    #step 2: Sparse NMF factorization (Learning W; V=WH)
    out = suppress_stdout()
    Ws = spams.trainDL(np.asfortranarray(np.transpose(VforW)), **param)
    suppress_stdout(out)
    #Label the columns to be Hematoxylin and eosin
    tmp = np.transpose(Ws)
    Ws = np.transpose(tmp[tmp[:, 2].argsort(), ])
    return Ws
Ejemplo n.º 15
0
 def fit(x, num):
     return spams.trainDL(
         K=num,
         numThreads=2,
         X=np.asfortranarray(x.T),
         mode=[4, 2][self.use_lasso],
         lambda1=[self.l0_max, self.l1_dictionary][self.use_lasso],
         iter=self.num_iterations,
         verbose=self.verbose,
         posAlpha=self.positive_coefficients,
         batchsize=self.minibatch_size,
     )
Ejemplo n.º 16
0
def dictionary_learning(patch_feature, lambda1=1, dictionary_size=100, batchsize=100,
                       posD=True):
    # input shape (feature size, sample size)
    X_patch = np.asfortranarray(patch_feature)
    param = { 'K' : dictionary_size, # learns a dictionary with 400 elements
             "mode":0,
              'lambda1' : lambda1, 'numThreads' : -1,
             "batchsize":batchsize,
             'posD':posD,
             "verbose":False
            }
    D = spams.trainDL(X_patch,**param)
    return D
def get_staincolor_sparsenmf(v, param):
    #print(param,v.T.shape)
    #print np.where(v.T<0)
    #parame = { 'K' :2,'lambda1': 0.02,'numThreads' :4,'posAlpha':True}
    Ws = spams.trainDL(v.T, **param)
    #print('here')
    #model = NMF(n_components=param['K'], init='random', random_state=2,alpha=0.02, l1_ratio=1)
    ##print(len(v<0))
    #Ws = model.fit_transform(v, y=None, W=None, H=None)
    #print("W_stain shape",Ws.shape)
    #print Ws.shape
    Ws = Ws[:, np.argsort(-Ws[1, :])]
    return Ws
Ejemplo n.º 18
0
def train_SC(database, param):

    num = database.shape[0]
    X = database.reshape([num, 80, 59])
    X = np.swapaxes(X, axis1=0, axis2=2)

    D_list = []
    pathces_num = X.shape[1]
    for i in range(pathces_num):
        x = np.asfortranarray(X[:, i, :])
        D = spams.trainDL(x, **param)
        D_list.append(D)
    return D_list
Ejemplo n.º 19
0
def alignfirst_dico(dataset,
                    N0,
                    J,
                    init=None,
                    save=False,
                    directory=None,
                    verbose=False):
    '''Performs (real) dictionary learning on the dataset, after it is optimally rotated along its mean.
    Relies on the SPAMS toolbox of Mairal et al. '''
    K1 = len(dataset)
    dataset = align_rot(dataset)
    dataset_r = multi_complex2real(dataset)
    X = sqrtPsi @ dataset_r.T
    X = np.asfortranarray(X)  # necessary for using spams toolbox
    D = spams.trainDL(X,
                      K=J,
                      D=init,
                      mode=3,
                      modeD=0,
                      lambda1=N0,
                      verbose=verbose)
    A = spams.omp(X, D=D, L=N0)
    Ad = np.array(A.todense()).reshape(J, K)
    D_c = multi_real2complex((sqrtPsi_inv @ D).T).T

    drawMany(D_c.T, show=False)
    plt.title('Align-first dictionary  N0 = {}  J = {}'.format(N0, J))
    if save:
        plt.savefig(directory + '/dico_alignfirst.png', dpi=200)
    plt.show()

    if verbose:
        DA = D_c @ A
        for k in test_k_set:
            display_res(dataset, DA, k, save=save, directory=directory)

    diffs = dataset.T - D_c @ Ad
    if K1 < 10000:
        E = np.diag(diffs.T.conj() @ Phi @ diffs).sum().real
    else:
        E = 0
        for k in range(K):
            E += (diffs[:, k].conj() @ Phi @ diffs[:, k]).real
    print('final loss : ', E)
    print('RMSE :', np.sqrt(E / K))
    if save:
        text_file = open(directory + '/readme_alignfirst.txt', 'a')
        text_file.write('Final loss: {}\n'.format(E))
        text_file.write('Final RMSE: {}\n'.format(np.sqrt(E / K)))
        text_file.close()
    return D_c, Ad, E
Ejemplo n.º 20
0
def online_learning(Xdev,
                    Ydev,
                    X,
                    Y,
                    covar,
                    display=False,
                    K=1,
                    method='mean'):
    """ uses online dictionary learning on the dev matrices
        to build and estimate Y_hat from the given X """

    X_init = np.array(Xdev, dtype=np.float64, order="FORTRAN")
    Y_init = np.array(Ydev, dtype=np.float64, order="FORTRAN")

    # f here is the number of features
    # F is the magnitude spectrum frequency number
    (f, T) = X_init.shape
    (F, T) = Y_init.shape

    # Learning a rank-f  model
    # initialize the model ?
    A = np.array(np.eye(F), dtype=np.float64, order="FORTRAN")
    B = np.array(X_init, dtype=np.float64, order="FORTRAN")
    prev_model = {'A': A, 'B': B, 'iter': T}
    # or not
    # model = None

    (D, model) = spams.trainDL(Y_init,
                               return_model=True,
                               model=prev_model,
                               iter=40,
                               lambda1=1,
                               posAlpha=False,
                               K=f)

    print D.shape, X.shape, model['A'].shape
    Y_hat = np.dot(D, np.dot(model['A'], X))
    print Y_hat.shape
    if display:
        plt.figure()
        plt.subplot(121)
        plt.imshow(np.log(Y))
        plt.title('Reponse originale')
        plt.colorbar()
        plt.subplot(122)
        plt.imshow(np.log(Y_hat))
        plt.colorbar()
        plt.title('Reponse predite')
        plt.show()

    return Y_hat, D
Ejemplo n.º 21
0
 def getW(self, V):
     W = spams.trainDL(np.asfortranarray(V),
                       K=self.STAIN_NUM,
                       lambda1=self.LAMBDA1,
                       iter=self.ITER,
                       mode=2,
                       modeD=0,
                       posAlpha=True,
                       posD=True,
                       verbose=False)
     W = W / np.linalg.norm(W, axis=0)[None, :]
     if (W[0, 0] < W[0, 1]):
         W = W[:, [1, 0]]
     return W
    def learn_D(self, segment_list, k, a=None, batch=False, iter=-5):

        # Horizontal train list
        temp = numpy.hstack(segment_list)
        if a == None:
            a = 1.0 / math.sqrt(temp.shape[0])
        # Learn dictionary
        D = spams.trainDL(numpy.asfortranarray(temp),
                          K=k,
                          lambda1=a,
                          batch=batch,
                          iter=iter,
                          posAlpha=True)
        return D
Ejemplo n.º 23
0
def getstainMat(I, param, i_0):
    #Beer-Lambert transform
    V, VforW = BLtrans(I, i_0)

    out = suppress_stdout()

    #Sparse NMF (Learning W; V=WH)
    #W is learnt only using VforW, i.e. by ignoring the white pixels
    #change VforW to V for W-estimation using all pixels
    Ws = spams.trainDL(np.asfortranarray(np.transpose(VforW)), **param)

    suppress_stdout(out)

    return Ws
Ejemplo n.º 24
0
def selectByChai2016(nGuide, fileName, parallel, nFrame, initD=None):
    if parallel:
        X, hairHeader, Data = SCGetMatrixAndHeaderMP(fileName, readEachFrame, nFrame) # X: len(u_s) x nHair, float64
    else:
        X, hairHeader, Data = SCGetMatrixAndHeader(fileName, readEachFrame, nFrame) # X: len(u_s) x nHair, float64

    lambda1 = para.lambda1
    Us = np.asfortranarray(X, 'd')

    params = {'lambda1': lambda1, 'lambda2': 0, 'return_model': True, 'model': None, 'posAlpha': True}
    D, ABi = spams.trainDL(Us, D=initD, K=nGuide, iter=100, batchsize=10, **params)  # D: len(u_s) x nGuide

    guide, nGuide = pickGuideHair(D, X)

    print "Got %d guide hairs" % nGuide
    return guide, nGuide
Ejemplo n.º 25
0
def get_stain_matrix(I, threshold=0.8, lamda=0.1):
    """
    Get 2x3 stain matrix. First row H and second row E
    :param I:
    :param threshold:
    :param lamda:
    :return:
    """
    mask = ut.notwhite_mask(I, thresh=threshold).reshape((-1,))
    OD = ut.RGB_to_OD(I).reshape((-1, 3))
    OD = OD[mask]
    dictionary = spams.trainDL(OD.T, K=2, lambda1=lamda, mode=2, modeD=0, posAlpha=True, posD=True, verbose=False).T
    if dictionary[0, 0] < dictionary[1, 0]:
        dictionary = dictionary[[1, 0], :]
    dictionary = ut.normalize_rows(dictionary)
    return dictionary
Ejemplo n.º 26
0
    def learn_semantic_atoms(self,
                             matrix,
                             corpus_file,
                             squared_norms,
                             w2i,
                             params,
                             initial_D=None,
                             file_id=None):
        if file_id is not None and os.path.exists(
                '{}.dict.gz'.format(file_id)):
            D = np.loadtxt('{}.dict.gz'.format(file_id))
            return np.asfortranarray(D)

        D = np.asfortranarray(spams.trainDL(matrix.T, D=initial_D, **params))
        if file_id is not None:
            np.savetxt('{}.dict.gz'.format(file_id), D)
        return D
Ejemplo n.º 27
0
def sample_test(driverNum, sampleNum, w_len = 40, k = 500):
	path = '/media/shih/新增磁碟區/ZiWen_packup/drivers/drivers'
	bow_sp = bow.BoW_sp()

	folder = os.listdir(path)
	sample = [int(i) for i in folder]
	#print sample
	sample.remove(driverNum)
	
	#print [sample, type(sample)]
	rand_driver = [random.choice(sample) for i in range(0,sampleNum)]
	#print [sample, type(sample)]
	#rand_driver = [folder[i] for i in sample ]
	#print rand_driver
	    
	a = 1.0/math.sqrt(w_len)
	trajectory = list()
	    #Iteratively process drivers data
	for i,driver in enumerate(rand_driver):		
		# Load 200 trajectory for random drivers
		
		j = random.randint(1,200)
		#print os.path.join(path, str(driver), str(j)+'.csv')
		temp = numpy.genfromtxt(os.path.join(path, str(driver), str(j)+'.csv'), 
		                        delimiter=',', skip_header=True)
		
		#print j
		
		temp = dataTransformation.trip_diff(temp)
		trajectory.append(numpy.asarray(temp))
	#print [len(trajectory)]
	trajectory = bow_sp.segment(trajectory,w_len=40)	
	print [len(trajectory), numpy.shape(numpy.hstack(trajectory))]
	D = spams.trainDL(numpy.asfortranarray(numpy.hstack(trajectory)), 
                          K=k, lambda1=a, posAlpha=True, iter=-3)
	#print [len(trajectory), len(trajectory[0])]

	sparseCode = bow_sp.coding_series(trajectory, D, a=a, iter=-3)
	#numpy.savetxt('test_result.csv', sparseCode, delimiter=',')
	#sparseCode = sparseCode/ numpy.linalg.norm(sparseCode)	
	# no need to normalize, trajectory always different length

	#raw_input("Press Enter to continue...")    
	return sparseCode
Ejemplo n.º 28
0
    def get_stain_matrix(I, threshold=0.8, lamda=0.1):
        """
        Get 2x3 stain matrix. First row H and second row E.
        See the original paper for details.
        Also see spams docs.

        :param I: Image RGB uint8.
        :param threshold:
        :param lamda:
        :return:
        """
        mask = mu.notwhite_mask(I, thresh=threshold).reshape((-1,))
        OD = mu.RGB_to_OD(I).reshape((-1, 3))
        OD = OD[mask]
        dictionary = spams.trainDL(OD.T, K=2, lambda1=lamda, mode=2, modeD=0, posAlpha=True, posD=True, verbose=False).T
        if dictionary[0, 0] < dictionary[1, 0]:
            dictionary = dictionary[[1, 0], :]
        dictionary = mu.normalize_rows(dictionary)
        return dictionary
Ejemplo n.º 29
0
    def get_stain_matrix(self, source_image):
        """ OD = SV. Get V

        Parameters
        ----------
        source_image: array_like
                      np.unit8 array of rgb values

        Returns
        -------

        stain_matrix: array_like
                      2 x M matrix for a N x M matrix

        """
        OD = RGB2OD(source_image)
        OD = OD.reshape((-1, 3))
        if self.maskout_white:
            nonwhite_mask = get_nonwhite_mask(source_image,
                                              self.nonwhite_threshold).reshape(
                                                  (-1, ))
            OD = OD[nonwhite_mask]
        OD = OD[(OD > self.beta).any(axis=1), :]
        self.OD = OD
        """
        Objective ||X-Da||_2^ + lambda1 * ||a||_1 + lambda2 ||a||_2^2 +
        """
        param = {
            "K": 2,
            "lambda1": self.lambda1,
            "lambda2": self.lambda2,
            "gamma1": self.gamma1,
            "mode": 2,
            "modeD": 1,
            "posD": True,
            "posAlpha": True,
            "verbose": False,
        }
        stain_matrix = spams.trainDL(OD.T, **param).T
        if stain_matrix[0, 0] < stain_matrix[1, 0]:
            stain_matrix = stain_matrix[[1, 0], :]
        return stain_matrix
Ejemplo n.º 30
0
def call_spams_trainDL(*args, **kwargs):
    """
        Encapsulates call to spams.trainDL. Ensures copy of results occur just in case.
        Designed to be like the multiprocessing calls.

        Args:
            *args(list):                            a list of position arguments to pass to spams.trainDL.
            **kwargs(dict):                         a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            For legacy.
    """

    # It is not needed outside of calling spams.trainDL.
    # Also, it takes a long time to load this module.
    import spams

    result = spams.trainDL(*args, **kwargs)
    result = result.copy()

    return (result)
Ejemplo n.º 31
0
def selectByChai2016New(nGuide, fileName, parallel, nFrame, initD=None):
    if parallel:
        X, hairHeader, Data = SCGetMatrixAndHeaderMP(fileName, readEachFrameNoDir, nFrame) # X: len(u_s) x nHair, float64
    else:
        X, hairHeader, Data = SCGetMatrixAndHeader(fileName, readEachFrameNoDir, nFrame) # X: len(u_s) x nHair, float64

    offset = hairHeader.factor * 3

    X0 = X[:offset, :]
    X = X[offset:, :] - np.tile(X0, (nFrame-1, 1))

    lambda1 = para.lambda1
    Us = np.asfortranarray(X, 'd')

    params = {'lambda1': lambda1, 'lambda2': 0, 'return_model': True, 'model': None, 'posAlpha': True}
    D, ABi = spams.trainDL(Us, D=initD, K=nGuide, iter=100, batchsize=10, **params)  # D: len(u_s) x nGuide

    norm = lambda x: np.linalg.norm(x)
    guide, nGuide = pickGuideHair(D, X, norm, norm)

    print "Got %d guide hairs" % nGuide
    return guide, nGuide
Ejemplo n.º 32
0
    def learn_dictionary(self, images, npatches=50000, niter=1000, njobs=-1):
        """ Learn a Sparse Code dictionary for this ScSPM.

        This method trains a sparse codes dictionary for the ScSPM descriptor
        object. This only needs to be run once before multiple calls to the
        extract() method can be made.

        Arguments:
            images: list, a list of paths to images to use for training.
            npatches: int (default 50000) number of SIFT patches to extract from
                the images to use for training the dictionary.
            niter: int (default 1000), the number of iterations of dictionary
                learning (Lasso) to perform.
            njobs: int (default -1), the number of threads to use. -1 means the
                number of threads will be equal to the number of cores.

        """

        # Get SIFT training patches
        print('Getting training patches...')
        patches = sw.training_patches(images,
                                      npatches,
                                      self.psize,
                                      self.maxdim,
                                      verbose=True)
        patches = pch.norm_patches(patches)
        print('{0} patches requested, {1} patches found.'.format(
            npatches, patches.shape[0]))
        time.sleep(3)  # Give people a chance to see this message

        # Learn dictionary
        print('Learning dictionary...')
        self.dic = trainDL(np.asfortranarray(patches.T, np.float64),
                           mode=0,
                           K=self.dsize,
                           lambda1=0.15,
                           iter=niter,
                           numThreads=njobs)
        print('done.')
Ejemplo n.º 33
0
    def learn_D(segment_list, k, lambda1=None, batch=False, iter1=-5):
        """Learn dictionary from given series with input parameters
        Args:
            segment_list(list): each item contains m subsequences which is
                sliced from original time series instance
            k(int): size of dictionary
            lambda1(float): lambda conefficient in sparse coding,
                |X-D*a|^2 + lambda*|a|^1
                For more information, see spams packages:
                http://spams-devel.gforge.inria.fr/doc-python/html/doc_spams004.html#sec5
            batch(bool): online learning or batch learning for sparse coding
                For more information, see spams packages:
                http://spams-devel.gforge.inria.fr/doc-python/html/doc_spams004.html#sec5
            iter1(int): learning iterations
                For more information, see spams packages:
                http://spams-devel.gforge.inria.fr/doc-python/html/doc_spams004.html#sec5
        Returns:
            D(numpy 2d-array): learning dictionary
        """
        # Horizontal train list
        temp = numpy.hstack(segment_list)
        if lambda1 == None:
            lambda1 = 1.0 / math.sqrt(temp.shape[0])
        # Log learning informatino
        log_msg = "learning dictionary with lambda: %f" % (lambda1)
        print(log_msg)
        logger.info(log_msg)
        # Learn dictionary

        print("About to train")
        D = spams.trainDL(numpy.asarray(temp, dtype=numpy.float64),
                          K=2,
                          lambda1=lambda1,
                          batch=batch,
                          iter=iter1,
                          posAlpha=True)

        print("Trained")
        return D
Ejemplo n.º 34
0
    def fit(self, X, y=None):
        ''' Fit a NMF model using the spams package

        Parameters
        ----------
        X : array, shape (n_samples, n_features)
            Data matrix to be fitted by the model

        y : ignored

        Returns
        -------
        self
        '''
        # Set the seed for numpy.random
        np.random.seed(self.random_state)

        # Create bootstrapped X
        if self.bootstrap:
            n_samples = X.shape[0]
            bootstrap_X = X[np.random.choice(n_samples,
                                             n_samples,
                                             replace=True)]
        else:
            bootstrap_X = X

        # Compute the initialization dictionary
        initialization = initialguess(bootstrap_X.T, self.n_components)

        # Use spams to compute the PPs
        Dsolution = spams.trainDL(
            # Data matrix
            # we flip X because spams requires features as rows
            np.asfortranarray(bootstrap_X.T),
            # Initial guess as provided by initialguess()
            D=initialization,
            **self.arguments)
        self.components_ = Dsolution.T
        return self
Ejemplo n.º 35
0
def extract_components(mov_tot, n_components=6, normalize_std=True, max_iter_DL=-30, method_factorization='nmf', **kwargs):
    """
    From optical flow images can extract spatial and temporal components

    Parameters:
    ----------
    mov_tot: ndarray (can be 3 or 4D)
        contains the optical flow values, either in cartesian or polar, either one (3D) or both (4D coordinates)
        the input is generated by the compute_optical_flow function

    n_components: int
        number of components to look for

    normalize_std: bool
        whether to normalize each oof the optical flow components

    normalize_output_traces: boolean
        whether to normalize the behavioral traces so that they match the units in the movie

    Returns:
    -------
    spatial_filter: ndarray
        set of spatial inferred filters

    time_trace:ndarray
        set of time components

    norm_fact: ndarray
        used notmalization factors

    """

    if mov_tot.ndim == 4:
        if normalize_std:
            norm_fact = np.nanstd(mov_tot, axis=(1, 2, 3))
            mov_tot = old_div(
                mov_tot, norm_fact[:, np.newaxis, np.newaxis, np.newaxis])
        else:
            norm_fact = np.array([1., 1.])
        c, T, d1, d2 = np.shape(mov_tot)

    else:
        norm_fact = 1
        T, d1, d2 = np.shape(mov_tot)
        c = 1

    tt = time.time()
    newm = np.reshape(mov_tot, (c * T, d1 * d2))

    if method_factorization == 'nmf':
        nmf = NMF(n_components=n_components, **kwargs)

        time_trace = nmf.fit_transform(newm)
        spatial_filter = nmf.components_
        spatial_filter = np.concatenate(
            [np.reshape(sp, (d1, d2))[np.newaxis, :, :] for sp in spatial_filter], axis=0)

    elif method_factorization == 'dict_learn':
        import spams
        newm = np.asfortranarray(newm, dtype=np.float32)
        time_trace = spams.trainDL(
            newm, K=n_components, mode=0, lambda1=1, posAlpha=True, iter=max_iter_DL)

        spatial_filter = spams.lasso(newm, D=time_trace, return_reg_path=False, lambda1=0.01,
                                     mode=spams.spams_wrap.PENALTY, pos=True)

        spatial_filter = np.concatenate([np.reshape(
            sp, (d1, d2))[np.newaxis, :, :] for sp in spatial_filter.toarray()], axis=0)

    time_trace = [np.reshape(ttr, (c, T)).T for ttr in time_trace.T]

    el_t = time.time() - tt
    print(el_t)
    return spatial_filter, time_trace, norm_fact
Ejemplo n.º 36
0
def test_trainDL():
    img_file = 'boat.png'
    try:
        img = Image.open(img_file)
    except:
        print("Cannot load image %s : skipping test" %img_file)
        return None
    I = np.array(img) / 255.
    if I.ndim == 3:
        A = np.asfortranarray(I.reshape((I.shape[0],I.shape[1] * I.shape[2])))
        rgb = True
    else:
        A = np.asfortranarray(I)
        rgb = False

    m = 8;n = 8;
    X = spams.im2col_sliding(A,m,n,rgb)

    X = X - np.tile(np.mean(X,0),(X.shape[0],1))
    X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)),(X.shape[0],1)),dtype = myfloat)
    param = { 'K' : 100, # learns a dictionary with 100 elements
              'lambda1' : 0.15, 'numThreads' : 4, 'batchsize' : 400,
              'iter' : 1000}

    ########## FIRST EXPERIMENT ###########
    tic = time.time()
    D = spams.trainDL(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)

    ##param['approx'] = 0
    # save dictionnary as dict.png
    _objective(X,D,param,'dict')

    #### SECOND EXPERIMENT ####
    print("*********** SECOND EXPERIMENT ***********")

    X1 = X[:,0:X.shape[1]//2]
    X2 = X[:,X.shape[1]//2 -1:]
    param['iter'] = 500
    tic = time.time()
    (D,model) = spams.trainDL(X1,return_model = True,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f\n' %t)

    _objective(X,D,param,'dict1')

    # Then reuse the learned model to retrain a few iterations more.
    param2 = param.copy()
    param2['D'] = D
    tic = time.time()
    (D,model) = spams.trainDL(X2,return_model = True,model = model,**param2)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)
    _objective(X,D,param,'dict2')

    #################### THIRD & FOURTH EXPERIMENT ######################
    # let us add sparsity to the dictionary itself

    print('*********** THIRD EXPERIMENT ***********')
    param['modeParam'] = 0
    param['iter'] = 1000
    param['gamma1'] = 0.3
    param['modeD'] = 1

    tic = time.time()
    D = spams.trainDL(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)
    _objective(X,D,param)

    #* DISPLAY
    print('*********** FOURTH EXPERIMENT ***********')
    param['modeParam'] = 0
    param['iter'] = 1000
    param['gamma1'] = 0.3
    param['modeD'] = 3

    tic = time.time()
    D = spams.trainDL(X,**param)
    tac = time.time()
    t = tac - tic
    print('time of computation for Dictionary Learning: %f' %t)
    _objective(X,D,param)

    return None
Ejemplo n.º 37
0
for m,n in itertools.product(range(20,51,5),range(100,1001,100)):
	counter = 0
	result = 0
    # Iteratively process drivers' data
	for driver in sorted(folder):
		print '---------------|'+str(driver)+'|-------------------'		
		# Load 200 trajectory for each drivers
		trajectory = list()
		for j in range(1, 201):
			temp = numpy.genfromtxt(os.path.join(path, str(driver), str(j)+'.csv'), 
			                        delimiter=',', skip_header=True)						
			trajectory.append(temp)
		trajectory = bow_inter.diff(trajectory)	
		trajectory_seg = bow_inter.slice(trajectory,w_len=m)
		
		D = spams.trainDL(numpy.asfortranarray(trajectory_seg), 
	                          K=n, lambda1=a, posAlpha=True, iter=-3)
		#print [len(trajectory), len(trajectory[0])]

		driverCode = bow_inter.bow(trajectory, D, a=a, w_len=m)
		#numpy.savetxt('result.csv', driverCode, delimiter=',')
		#driverCode = driverCode/ numpy.linalg.norm(driverCode)	
		# no need to normalize, trajectory always different length


		# ======================== sample negative data ==========================
		sample = [int(i) for i in folder]		
		sample.remove(int(driver))
		# Load 200 trajectory for random drivers
		rand_driver = [random.choice(sample) for i in range(0,200)]
		    		
		trajectory = list()
Ejemplo n.º 38
0
submatrix = np.asfortranarray(np.zeros((n, submatrix_size), dtype = np.float32))

D = None
for k in range(submatrix_iterations):
	print("iteration on submatrices " + str(k))
	submatrix_nonzero_indices = global_nonzero_indices[(submatrix_size * k):(submatrix_size * (k + 1))]

	print("read data ")
	for i in range(len(Kmer_Hash_Count_Files)):
		submatrix[i,:] = np.memmap(Kmer_Hash_Count_Files[i], dtype='float32', mode='r')[submatrix_nonzero_indices]

	print("matrix normalization")
	submatrix = submatrix / np.sqrt(np.sum(submatrix * submatrix, 0))

	D = spams.trainDL(submatrix, D = D, K = args.cluster_number, lambda1 = args.lambda1, lambda2 = args.lambda2, 
		posAlpha = True, posD = True, rho = 1.0, iter = args.iter_nb, batchsize = args.batchsize, numThreads = cpu)



np.save(args.outputdir + "cluster_index.npy", D.T)
print(D)


print("cluster kmers")

cluster_cols = np.zeros(2**hash_size, dtype = 'uint16')
submatrix_size = int(memory_limit / (4 * (args.cluster_number + n)))
submatrix_iterations =  int(nzi / submatrix_size) + 1
submatrix_size = min(nzi, submatrix_size)

Ejemplo n.º 39
0
def denoise(data, block_size, overlap, param_alpha, param_D, variance, n_iter=10,
            mask=None, dtype=np.float64):

    # no overlapping blocks for training
    no_over = (0, 0, 0, 0)
    X = im2col_nd(data, block_size, no_over)

    # Solving for D
    param_alpha['pos'] = True
    param_alpha['mode'] = 2
    param_alpha['lambda1'] = 1.2 / np.sqrt(np.prod(block_size))

    param_D['verbose'] = False
    param_D['posAlpha'] = True
    param_D['posD'] = True
    param_D['mode'] = 2
    param_D['lambda1'] = 1.2 / np.sqrt(np.prod(block_size))
    param_D['K'] = int(2*np.prod(block_size))
    param_D['iter'] = 150
    param_D['batchsize'] = 500

    if 'D' in param_alpha:
        param_D['D'] = param_alpha['D']

    # mask_col = im2col_nd(mask, block_size[:3], no_over[:3])
    mask_col = im2col_nd(np.broadcast_to(mask[..., None], data.shape), block_size, no_over)
    train_idx = np.sum(mask_col, axis=0) > mask_col.shape[0]/2

    train_data = X[:, train_idx]
    train_data = np.asfortranarray(train_data[:, np.any(train_data != 0, axis=0)], dtype=dtype)
    train_data /= np.sqrt(np.sum(train_data**2, axis=0, keepdims=True), dtype=dtype)
    param_alpha['D'] = spams.trainDL(train_data, **param_D)
    param_alpha['D'] /= np.sqrt(np.sum(param_alpha['D']**2, axis=0, keepdims=True, dtype=dtype))
    param_D['D'] = param_alpha['D']

    del train_data

    n_cores = param_alpha['numThreads']
    param_alpha['numThreads'] = 1
    param_D['numThreads'] = 1

    time_multi = time()
    pool = Pool(processes=n_cores)

    arglist = [(data[:, :, k:k+block_size[2]], mask[:, :, k:k+block_size[2]], variance[:, :, k:k+block_size[2]], block_size_subset, overlap_subset, param_alpha_subset, param_D_subset, dtype_subset, n_iter_subset)
               for k, block_size_subset, overlap_subset, param_alpha_subset, param_D_subset, dtype_subset, n_iter_subset
               in zip(range(data.shape[2] - block_size[2] + 1),
                      repeat(block_size),
                      repeat(overlap),
                      repeat(param_alpha),
                      repeat(param_D),
                      repeat(dtype),
                      repeat(n_iter))]

    data_denoised = pool.map(processer, arglist)
    pool.close()
    pool.join()

    param_alpha['numThreads'] = n_cores
    param_D['numThreads'] = n_cores

    print('Multiprocessing done in {0:.2f} mins.'.format((time() - time_multi) / 60.))

    # Put together the multiprocessed results
    data_subset = np.zeros_like(data)
    divider = np.zeros_like(data, dtype=np.int16)
    ones = np.ones_like(data_denoised[0], dtype=np.int16)

    for k in range(len(data_denoised)):
        data_subset[:, :, k:k+block_size[2]] += data_denoised[k]
        divider[:, :, k:k+block_size[2]] += ones

    data_subset /= divider
    return data_subset
Ejemplo n.º 40
0
             'lambda1': Lambda, # number of threads
             'numThreads': -1,
             'batchsize': min(1024, n), # positive dictionary
             'posD': True, # positive dictionary
             'iter': 500, # number of iteration
             'modeD': 0,
             'verbose': 0, # print out update information?
             #'pos': 1, # positive alpha
             'posAlpha': 1, # positive alpha
             'gamma1': gamma1, # penalizing parameter on the dictionary patterns
             'D': np.asfortranarray(D0) # set initial values
    }

    X = np.asfortranarray(X, dtype=float)
    # print X.shape
    Dtemplate = spams.trainDL(X, **param)

    # for each fixed dictionary K, we will repeat dictionary
    # learning for 100 times, each with a different initial value
    test_cases = 10
    R = np.zeros((test_cases, ))
    for i in xrange(0, test_cases):
        if randomStart == 1:
            D0 = util.dictLearnInit(X, K, 'random', 0)
        param['D'] = np.asfortranarray(D0)
        lparam = {'lambda1': Lambda,
                  'pos': True,
                  'mode': 2,
                  'numThreads': -1
        }
        D = spams.trainDL(X, **param)
    rgb = False

m = 8;n = 8;
#m = 2;n = 2;
X = spams.im2col_sliding(A,m,n,rgb)

X = X - np.tile(np.mean(X,0),(X.shape[0],1))
X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)),(X.shape[0],1)),dtype = myfloat)
param = { 'K' : 100, # learns a dictionary with 100 elements
          'lambda1' : 0.15, 'numThreads' : 4, 'batchsize' : 400,
          'iter' : 10}
paramL = {'lambda1' : 0.15, 'numThreads' : 4}

########## FIRST EXPERIMENT ###########
tic = time.time()
D = spams.trainDL(X,**param)
tac = time.time()
t = tac - tic
print 'time of computation for Dictionary Learning: %f' %t
print "DTYPE %s" %str(D.dtype)
#param['approx'] = 0
print 'Evaluating cost function...'
alpha = spams.lasso(X,D,**paramL)
print "XX X %s, D %s, alpha %s" %(str(X.shape),str(D.shape),str(alpha.shape))
y = X
if(alpha.shape[1] > 1000):
    alpha = alpha[:,0:1000]
    y = X[:,0:1000]
#Da = spams.calcXAt(D,ssp.csc_matrix(alpha.T))
a = alpha.todense()
print "XXa %s" %str(a.shape)
Ejemplo n.º 42
0
def run_multiprocessing_array_spams_trainDL(result_array_type,
                                            result_array,
                                            X_array_type,
                                            X_array,
                                            D_is_arg=False,
                                            D_array_type=None,
                                            D_array=None,
                                            *args,
                                            **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the
        result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation
        faults have been discovered in later parts of the Python code dependent
        on whether SPAMS has run or not. It is suspected that spams may
        interfere with the interpreter. Thus, it should be sandboxed (run in a
        different Python interpreter) so that it doesn't damage what happens in
        this one.

        This particular version uses a multiprocessing.Array to share memory to
        return the resulting dictionary.


        Args:
            result_array_type(numpy.ctypeslib.ndpointer):   Unused will drop.
                                                            A pointer type with
                                                            properties needed
                                                            by result_array.

            result_array(multiprocessing.RawArray):         shared memory array
                                                            to store results
                                                            in.

            X_array_type(numpy.ctypeslib.ndpointer):        Unused will drop.
                                                            a pointer type with
                                                            properties needed
                                                            by X_array.

            X_array(numpy.ndarray):                         currently uses
                                                            numpy ndarray as
                                                            input.

            D_is_arg(bool):                                 Whether D either is
                                                            an arg and/or
                                                            should be an arg.

            D_array_type(numpy.ctypeslib.ndpointer):        Unused will drop.
                                                            a pointer type with
                                                            properties needed
                                                            by D_array.

            D_array(numpy.ndarray):                         currently uses
                                                            numpy ndarray as
                                                            the initial
                                                            dictionary.

            *args(list):                                    a list of position
                                                            arguments to pass
                                                            to spams.trainDL.

            **kwargs(dict):                                 a dictionary of
                                                            keyword arguments
                                                            to pass to
                                                            spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.
    """

    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    # Also, it is not needed outside of calling this function.
    import spams

    with npctypes.shared.as_ndarray(X_array) as X:
        with npctypes.shared.as_ndarray(result_array) as result:
            if D_array is not None:
                with npctypes.shared.as_ndarray(D_array) as D:
                    if D_is_arg:
                        args[3] = D
                    else:
                        kwargs["D"] = D

                    result[:] = spams.trainDL(X, *args, **kwargs)
            else:
                result[:] = spams.trainDL(X, *args, **kwargs)
Ejemplo n.º 43
0
    X = X[0]

    # extract 2d patches from the image
    X_dl = extract_patches_2d(X, atom_support, max_patches=max_patches)
    X_dl = X_dl.reshape(X_dl.shape[0], -1)
    norm = np.linalg.norm(X_dl, axis=1)
    mask = norm != 0
    X_dl = X_dl[mask]
    X_dl /= norm[mask][:, None]

    meta = dict(lambda1=reg, iter=10_000, mode=2, posAlpha=True, posD=False)

    # Learn the dictionary with spams
    D_dl = spams.trainDL(np.asfortranarray(X_dl.T, dtype=np.float),
                         numThreads=n_jobs,
                         batchsize=512,
                         K=n_atoms,
                         **meta,
                         verbose=False).T

    return D_dl.reshape(n_atoms, 1, *atom_support), meta


@memory.cache
def compute_cdl(X,
                n_atoms,
                atom_support,
                D_init,
                reg=.2,
                window=False,
                n_jobs=10):
    """Compute dictionary using Dicodile.
Ejemplo n.º 44
0
import numpy as np
import spams

param = {'K': 100, 'lambda1': 0.15, 'numThreads': 4, 'batchsize': 400, 'iter': 10}
X = np.zeros((5, 5), dtype=float)
X = np.asfortranarray(X)
D = spams.trainDL(X, **param)
import cPickle
import spams

parser = argparse.ArgumentParser(description='dictionary learning')
parser.add_argument('--feature_file', dest='feature_file',
                    help='feature file',
                    default='cache/train_deep_feat.pkl', type=str)
args = parser.parse_args()

if not os.path.exists(args.feature_file):
    print 'feature file does not exist!'
    sys.exit(-1)

with open(args.feature_file, 'r') as fd:
    t1 = time.time()
    feat = cPickle.load(fd)
    t2 = time.time()
    feat = feat[1].T
    print "feature file loaded from " + args.feature_file + ' in %f seconds' % (t2-t1)

dl_params = {'K': 100,
             'lambda1' : 0.15,
             'numThreads' : 4,
             'batchsize' : 400,
             'iter': 1000}

t1 = time.time()
D = spams.trainDL(feat, **dl_params)
t2 = time.time()
print 'Dictionary Learning: finish %d iterations in %f seconds' % (dl_params['iter'], t2-t1)
Ejemplo n.º 46
0
def run_multiprocessing_array_spams_trainDL(result_array_type, result_array, X_array_type, X_array, *args, **kwargs):
    """
        Designed to start spams.trainDL in a separate process and handle the result in an unnoticeably different way.

        It is necessary to run SPAMS in a separate process as segmentation faults
        have been discovered in later parts of the Python code dependent on whether
        SPAMS has run or not. It is suspected that spams may interfere with the
        interpreter. Thus, it should be sandboxed (run in a different Python interpreter)
        so that it doesn't damage what happens in this one.

        This particular version uses a multiprocessing.Array to share memory to return the resulting dictionary.


        Args:
            result_array_type(numpy.ctypeslib.ndpointer):   a pointer type with properties needed by result_array.
            result_array(multiprocessing.RawArray):         shared memory array to store results in.
            X_array_type(numpy.ctypeslib.ndpointer):        a pointer type with properties needed by X_array.
            X_array(numpy.ndarray):                         currently uses numpy ndarray as input.
            *args(list):                                    a list of position arguments to pass to spams.trainDL.
            *kwargs(dict):                                  a dictionary of keyword arguments to pass to spams.trainDL.

        Note:
            This is somewhat faster than using multiprocessing.Queue.
    """

    # Just to make sure this exists in the new process. Shouldn't be necessary.
    import numpy
    # Just to make sure this exists in the new process. Shouldn't be necessary.
    # Also, it is not needed outside of calling this function.
    import spams

    as_ordered_array_dict = {
        "F_CONTIGUOUS" : numpy.asfortranarray,
        "C_CONTIGUOUS" : numpy.ascontiguousarray
    }


    # Construct X from shared array.
    X_dtype = X_array_type._dtype_
    X_shape = X_array_type._shape_
    X_flags = numpy.core.multiarray.flagsobj(X_array_type._flags_)

    X = numpy.frombuffer(X_array, dtype = X_dtype).reshape(X_shape)
    X.setflags(X_flags)

    for order_name, as_ordered_array in as_ordered_array_dict.items():
        if order_name in X_array_type.__name__:
            X = as_ordered_array(X)

    # Construct the result to use the shared buffer.
    result_dtype = result_array_type._dtype_
    result_shape = result_array_type._shape_
    result_flags = numpy.core.multiarray.flagsobj(result_array_type._flags_)

    result = numpy.frombuffer(result_array, dtype = result_dtype).reshape(result_shape)
    result.setflags(result_flags)

    for order_name, as_ordered_array in as_ordered_array_dict.items():
        if order_name in result_array_type.__name__:
            result = as_ordered_array(result)


    result[:] = spams.trainDL(X, *args, **kwargs)
Ejemplo n.º 47
0
import scipy.io
import matplotlib.pyplot as plt
import mpl_toolkits.mplot3d as p3d
import spams
#import cvxpy as cvx

from datasets.datasetMat import CMUDataSet

# The coordinate system of poses3Ds:
# origin: (x_0,y_0,z_0) which present the mean value of  all joint's coordinate (x,y,z)
# x Axis: vector point from left shoulder to right shoulder
# y Axis: vector point from midpoint of left and right shoulder to waist
# z Axis: the cross product of x Axis and y Axis
poses3Ds, scale3 = CMUDataSet.getPose3DNormalized()
poses2Ds, scale2 = CMUDataSet.getPose2DNormalized()
#

param = {'K': 200, 'lambda1': 0.01, 'iter': 300}

#not the same result compared to authors original source code
B = spams.trainDL(np.asfortranarray(poses3Ds), **param)
init_pose = np.expand_dims(np.mean(poses3Ds, 1), axis=1)
print(init_pose.shape)

#
scipy.io.savemat('./datasets/BaseMatrix.mat',
                 mdict={
                     'init_pose': init_pose,
                     'B': B
                 })
Ejemplo n.º 48
0
def extract_components(mov_tot,
                       n_components=6,
                       normalize_std=True,
                       max_iter_DL=-30,
                       method_factorization='nmf',
                       **kwargs):
    """
    From optical flow images can extract spatial and temporal components

    Args:
        mov_tot: ndarray (can be 3 or 4D)
            contains the optical flow values, either in cartesian or polar, either one (3D) or both (4D coordinates)
            the input is generated by the compute_optical_flow function

        n_components: int
            number of components to look for

        normalize_std: bool
            whether to normalize each oof the optical flow components

        normalize_output_traces: boolean
            whether to normalize the behavioral traces so that they match the units in the movie

    Returns:
        spatial_filter: ndarray
            set of spatial inferred filters

        time_trace:ndarray
            set of time components

        norm_fact: ndarray
            used notmalization factors
    """

    if mov_tot.ndim == 4:
        if normalize_std:
            norm_fact = np.nanstd(mov_tot, axis=(1, 2, 3))
            mov_tot = old_div(mov_tot, norm_fact[:, np.newaxis, np.newaxis,
                                                 np.newaxis])
        else:
            norm_fact = np.array([1., 1.])
        c, T, d1, d2 = np.shape(mov_tot)

    else:
        norm_fact = 1
        T, d1, d2 = np.shape(mov_tot)
        c = 1

    tt = time.time()
    newm = np.reshape(mov_tot, (c * T, d1 * d2))

    if method_factorization == 'nmf':
        nmf = NMF(n_components=n_components, **kwargs)

        time_trace = nmf.fit_transform(newm)
        spatial_filter = nmf.components_
        spatial_filter = np.concatenate([
            np.reshape(sp, (d1, d2))[np.newaxis, :, :] for sp in spatial_filter
        ],
                                        axis=0)

    elif method_factorization == 'dict_learn':
        import spams
        newm = np.asfortranarray(newm, dtype=np.float32)
        time_trace = spams.trainDL(newm,
                                   K=n_components,
                                   mode=0,
                                   lambda1=1,
                                   posAlpha=True,
                                   iter=max_iter_DL)

        spatial_filter = spams.lasso(newm,
                                     D=time_trace,
                                     return_reg_path=False,
                                     lambda1=0.01,
                                     mode=spams.spams_wrap.PENALTY,
                                     pos=True)

        spatial_filter = np.concatenate([
            np.reshape(sp, (d1, d2))[np.newaxis, :, :]
            for sp in spatial_filter.toarray()
        ],
                                        axis=0)

    time_trace = [np.reshape(ttr, (c, T)).T for ttr in time_trace.T]

    el_t = time.time() - tt
    print(el_t)
    return spatial_filter, time_trace, norm_fact
Ejemplo n.º 49
0
def local_denoise(data,
                  block_size,
                  overlap,
                  variance,
                  n_iter=10,
                  mask=None,
                  dtype=np.float64,
                  n_cores=None,
                  use_threading=False,
                  verbose=False,
                  mp_method=None):
    if verbose:
        logger.setLevel(logging.INFO)

    if mask is None:
        mask = np.ones(data.shape[:-1], dtype=np.bool)

    # no overlapping blocks for training
    no_over = (0, 0, 0, 0)
    X = im2col_nd(data, block_size, no_over)

    # Solving for D
    param_alpha = {}
    param_alpha['pos'] = True
    param_alpha['mode'] = 1

    param_D = {}
    param_D['verbose'] = False
    param_D['posAlpha'] = True
    param_D['posD'] = True
    param_D['mode'] = 2
    param_D['lambda1'] = 1.2 / np.sqrt(np.prod(block_size))
    param_D['K'] = int(2 * np.prod(block_size))
    param_D['iter'] = 150
    param_D['batchsize'] = 500
    param_D['numThreads'] = n_cores

    if 'D' in param_alpha:
        param_D['D'] = param_alpha['D']

    mask_col = im2col_nd(np.broadcast_to(mask[..., None], data.shape),
                         block_size, no_over)
    train_idx = np.sum(mask_col, axis=0) > (mask_col.shape[0] / 2.)

    train_data = X[:, train_idx]
    train_data = np.asfortranarray(train_data[:,
                                              np.any(train_data != 0, axis=0)],
                                   dtype=dtype)
    train_data /= np.sqrt(np.sum(train_data**2, axis=0, keepdims=True),
                          dtype=dtype)

    param_alpha['D'] = spams.trainDL(train_data, **param_D)
    param_alpha['D'] /= np.sqrt(
        np.sum(param_alpha['D']**2, axis=0, keepdims=True, dtype=dtype))
    param_D['D'] = param_alpha['D']

    del train_data, X, mask_col

    if use_threading or (n_cores == 1):
        param_alpha['numThreads'] = n_cores
        param_D['numThreads'] = n_cores
    else:
        param_alpha['numThreads'] = 1
        param_D['numThreads'] = 1

    arglist = ((data[:, :, k:k + block_size[2]], mask[:, :,
                                                      k:k + block_size[2]],
                variance[:, :, k:k + block_size[2]], block_size, overlap,
                param_alpha, param_D, dtype, n_iter)
               for k in range(data.shape[2] - block_size[2] + 1))

    if use_threading:
        data_denoised = starmap(processer, arglist)
    else:
        time_multi = time()
        parallel_processer = multiprocesser(processer,
                                            n_cores=n_cores,
                                            mp_method=mp_method)
        data_denoised = parallel_processer(arglist)
        logger.info('Multiprocessing done in {0:.2f} mins.'.format(
            (time() - time_multi) / 60.))

    # Put together the multiprocessed results
    data_subset = np.zeros_like(data, dtype=np.float32)
    divider = np.zeros_like(data, dtype=np.int16)

    for k, content in enumerate(data_denoised):
        data_subset[:, :, k:k + block_size[2]] += content
        divider[:, :, k:k + block_size[2]] += 1

    data_subset /= divider
    return data_subset
Ejemplo n.º 50
0
def test_trainDL():
    img_file = "boat.png"
    try:
        img = Image.open(img_file)
    except:
        print "Cannot load image %s : skipping test" % img_file
        return None
    I = np.array(img) / 255.0
    if I.ndim == 3:
        A = np.asfortranarray(I.reshape((I.shape[0], I.shape[1] * I.shape[2])))
        rgb = True
    else:
        A = np.asfortranarray(I)
        rgb = False

    m = 8
    n = 8
    X = spams.im2col_sliding(A, m, n, rgb)

    X = X - np.tile(np.mean(X, 0), (X.shape[0], 1))
    X = np.asfortranarray(X / np.tile(np.sqrt((X * X).sum(axis=0)), (X.shape[0], 1)), dtype=myfloat)
    param = {
        "K": 100,  # learns a dictionary with 100 elements
        "lambda1": 0.15,
        "numThreads": 4,
        "batchsize": 400,
        "iter": 1000,
    }

    ########## FIRST EXPERIMENT ###########
    tic = time.time()
    D = spams.trainDL(X, **param)
    tac = time.time()
    t = tac - tic
    print "time of computation for Dictionary Learning: %f" % t

    ##param['approx'] = 0
    # save dictionnary as dict.png
    _objective(X, D, param, "dict")

    #### SECOND EXPERIMENT ####
    print "*********** SECOND EXPERIMENT ***********"

    X1 = X[:, 0 : X.shape[1] / 2]
    X2 = X[:, X.shape[1] / 2 - 1 :]
    param["iter"] = 500
    tic = time.time()
    (D, model) = spams.trainDL(X1, return_model=True, **param)
    tac = time.time()
    t = tac - tic
    print "time of computation for Dictionary Learning: %f\n" % t

    _objective(X, D, param, "dict1")

    # Then reuse the learned model to retrain a few iterations more.
    param2 = param.copy()
    param2["D"] = D
    tic = time.time()
    (D, model) = spams.trainDL(X2, return_model=True, model=model, **param2)
    tac = time.time()
    t = tac - tic
    print "time of computation for Dictionary Learning: %f" % t
    _objective(X, D, param, "dict2")

    #################### THIRD & FOURTH EXPERIMENT ######################
    # let us add sparsity to the dictionary itself

    print "*********** THIRD EXPERIMENT ***********"
    param["modeParam"] = 0
    param["iter"] = 1000
    param["gamma1"] = 0.3
    param["modeD"] = 1

    tic = time.time()
    D = spams.trainDL(X, **param)
    tac = time.time()
    t = tac - tic
    print "time of computation for Dictionary Learning: %f" % t
    _objective(X, D, param)

    # * DISPLAY
    print "*********** FOURTH EXPERIMENT ***********"
    param["modeParam"] = 0
    param["iter"] = 1000
    param["gamma1"] = 0.3
    param["modeD"] = 3

    tic = time.time()
    D = spams.trainDL(X, **param)
    tac = time.time()
    t = tac - tic
    print "time of computation for Dictionary Learning: %f" % t
    _objective(X, D, param)

    return None