import masking
    import numpy as np
    import scipy.io as sio

    niter = 100
    D = 100
    if len(sys.argv) > 1:
        niter = int(sys.argv[1])
        print 'niter =',niter
    if len(sys.argv) > 2:
        D = int(sys.argv[2])
        print 'D =',D
    
    btchroma = sio.loadmat('/home/thierry/Columbia/covers80/coversongs/covers32k/Caroline_No/beach_boys+Pet_Sounds+13-Caroline_No.mp3.mat')['btchroma']

    mask,p1,p2 = masking.random_patch_mask(btchroma,ncols=10)

    # init data
    np.random.seed(8484)
    A = np.eye(D) * np.random.rand(1,D) / np.sqrt(D)
    C = np.random.rand(12,D)
    Q = map(lambda x: np.dot(x.T,x),[np.random.rand(D,D)])[0] / np.sqrt(D)
    R = map(lambda x: np.dot(x.T,x),[np.random.rand(12,12)])[0] / np.sqrt(12)
    initx = np.random.rand(D,1) / np.sqrt(D)
    initV = np.eye(D)

    # go
    res = learn_kalman_missing_patch(btchroma*mask,p1,p2,A,C,Q,R,initx,initV,
                                     niter,diagQ=1,diagR=1)
    recon = res[0]
    recon[:,:p1] = btchroma[:,:p1]
Exemple #2
0
def test_maskedpatch_on_dataset(dataset,
                                method='random',
                                ncols=2,
                                win=1,
                                rank=4,
                                codebook=None,
                                nstates=1,
                                verbose=1,
                                **kwargs):
    """
    Dataset is either dir, or list of files
    General method to test a method on a whole dataset for one masked column
    Methods are:
      - random
      - randomfromsong
      - average
      - averageall
      - codebook
      - knn_eucl
      - knn_kl
      - knn_eucl_delta
      - lintrans
      - kalman
      - hmm
      - siplca
      - siplca2
    Used arguments vary based on the method. For SIPLCA, we can use **kwargs
    to set priors.
    RETURN
      - all errors, as a list of dict
    """
    # get all matfiles
    if type(dataset).__name__ == 'str':
        matfiles = get_all_matfiles(dataset)
    else:
        matfiles = dataset
    # init
    total_cnt = 0
    all_errs = []
    # some specific inits
    if codebook != None and not type(codebook) == type([]):
        codebook = [p.reshape(12, codebook.shape[1] / 12) for p in codebook]
        print 'codebook in ndarray format transformed to list'
    # iterate
    for matfile in matfiles:
        btchroma = sio.loadmat(matfile)['btchroma']
        if len(btchroma.shape) < 2:
            continue
        if btchroma.shape[1] < MINLENGTH or np.isnan(btchroma).any():
            continue
        mask, p1, p2 = MASKING.random_patch_mask(btchroma, ncols=ncols, win=25)
        ########## ALGORITHM DEPENDENT
        if method == 'random':
            recon = IMPUTATION.random_patch(btchroma, mask, p1, p2)
        elif method == 'randomfromsong':
            recon = IMPUTATION.random_patch_from_song(btchroma, mask, p1, p2)
        elif method == 'average':
            recon = IMPUTATION.average_patch(btchroma, mask, p1, p2, win=win)
        elif method == 'averageall':
            recon = IMPUTATION.average_patch_all(btchroma, mask, p1, p2)
        elif method == 'codebook':
            recon, used_codes = IMPUTATION.codebook_patch(
                btchroma, mask, p1, p2, codebook)
        elif method == 'knn_eucl':
            recon, used_cols = IMPUTATION.knn_patch(btchroma,
                                                    mask,
                                                    p1,
                                                    p2,
                                                    win=win,
                                                    measure='eucl')
        elif method == 'knn_kl':
            recon, used_cols = IMPUTATION.knn_patch(btchroma,
                                                    mask,
                                                    p1,
                                                    p2,
                                                    win=win,
                                                    measure='kl')
        elif method == 'knn_eucl_delta':
            recon, used_cols = IMPUTATION.knn_patch_delta(btchroma,
                                                          mask,
                                                          p1,
                                                          p2,
                                                          win=win,
                                                          measure='eucl')
        elif method == 'lintrans':
            recon, proj = IMPUTATION.lintransform_patch(btchroma,
                                                        mask,
                                                        p1,
                                                        p2,
                                                        win=win)
        elif method == 'kalman':
            import kalman_toolbox as KALMAN
            recon = KALMAN.imputation(btchroma,
                                      p1,
                                      p2,
                                      dimstates=nstates,
                                      **kwargs)
        elif method == 'hmm':
            recon, recon2, hmm = IMPUTATION_HMM.imputation(btchroma * mask,
                                                           p1,
                                                           p2,
                                                           nstates=nstates,
                                                           **kwargs)
        elif method == 'siplca':
            res = IMPUTATION_PLCA.SIPLCA_mask.analyze((btchroma * mask).copy(),
                                                      rank,
                                                      mask,
                                                      win=win,
                                                      convergence_thresh=1e-15,
                                                      **kwargs)
            W, Z, H, norm, recon, logprob = res
        elif method == 'siplca2':
            res = IMPUTATION_PLCA.SIPLCA2_mask.analyze(
                (btchroma * mask).copy(),
                rank,
                mask,
                win=win,
                convergence_thresh=1e-15,
                **kwargs)
            W, Z, H, norm, recon, logprob = res
        else:
            print 'unknown method:', method
            return
        ########## ALGORITHM DEPENDENT END
        # measure recon
        errs = recon_error(btchroma, mask, recon, measure='all')
        all_errs.append(errs)
        total_cnt += 1
    # done
    if verbose > 0:
        print 'number of songs tested:', total_cnt
        for meas in np.sort(all_errs[0].keys()):
            errs = map(lambda d: d[meas], all_errs)
            print 'average', meas, '=', np.mean(errs), '(', np.std(errs), ')'
    return all_errs
import imputation as IMPUTATION
import imputation_plca

originals = []
recons = []
texts = []



def nice_nums(val):
    return evaluation.nice_nums(val)


# EASY RECONSTRUCTION
bt = sio.loadmat('/home/thierry/Columbia/covers80/coversongs/covers32kENmats/beatles+Revolver+14-Tomorrow_Never_Knows.mp3.mat')['btchroma']
mask,p1,p2 = masking.random_patch_mask(bt,ncols=15,win=25)
recon,tmp = IMPUTATION.lintransform_patch(bt,mask,p1,p2,win=1)
errs = evaluation.recon_error(bt,mask,recon)
s = 'method: linear transform\n'
s += 'eucl = ' + nice_nums(errs['eucl']) + '\n'
s += 'd1/2 = ' + nice_nums(errs['lhalf']) + '\n'
s += 'cond. ent. = ' + nice_nums(errs['condent']) + '\n'
s += 'delta diff. = ' + nice_nums(errs['ddiff']) + '\n'
s += 'D-ENT = ' + nice_nums(errs['dent']) + '\n'
s += 'Jensen diff. = ' + nice_nums(errs['jdiff']) + '\n'
originals.append( bt[:,p1:p2].copy() )
recons.append( recon[:,p1:p2].copy() )
texts.append(s)


# SMOOTH RECONSTRUCTION
def test_maskedpatch_on_dataset(dataset,method='random',ncols=2,win=1,rank=4,codebook=None,nstates=1,verbose=1,**kwargs):
    """
    Dataset is either dir, or list of files
    General method to test a method on a whole dataset for one masked column
    Methods are:
      - random
      - randomfromsong
      - average
      - averageall
      - codebook
      - knn_eucl
      - knn_kl
      - knn_eucl_delta
      - lintrans
      - kalman
      - hmm
      - siplca
      - siplca2
    Used arguments vary based on the method. For SIPLCA, we can use **kwargs
    to set priors.
    RETURN
      - all errors, as a list of dict
    """
    # get all matfiles
    if type(dataset).__name__ == 'str':
        matfiles = get_all_matfiles(dataset)
    else:
        matfiles = dataset
    # init
    total_cnt = 0
    all_errs = []
    # some specific inits
    if codebook != None and not type(codebook) == type([]):
        codebook = [p.reshape(12,codebook.shape[1]/12) for p in codebook]
        print 'codebook in ndarray format transformed to list'
    # iterate
    for matfile in matfiles:
        btchroma = sio.loadmat(matfile)['btchroma']
        if len(btchroma.shape) < 2:
            continue
        if btchroma.shape[1] < MINLENGTH or np.isnan(btchroma).any():
            continue
        mask,p1,p2 = MASKING.random_patch_mask(btchroma,ncols=ncols,win=25)
        ########## ALGORITHM DEPENDENT
        if method == 'random':
            recon = IMPUTATION.random_patch(btchroma,mask,p1,p2)
        elif method == 'randomfromsong':
            recon = IMPUTATION.random_patch_from_song(btchroma,mask,p1,p2)
        elif method == 'average':
            recon = IMPUTATION.average_patch(btchroma,mask,p1,p2,win=win)
        elif method == 'averageall':
            recon = IMPUTATION.average_patch_all(btchroma,mask,p1,p2)
        elif method == 'codebook':
            recon,used_codes = IMPUTATION.codebook_patch(btchroma,mask,p1,p2,codebook)
        elif method == 'knn_eucl':
            recon,used_cols = IMPUTATION.knn_patch(btchroma,mask,p1,p2,win=win,measure='eucl')
        elif method == 'knn_kl':
            recon,used_cols = IMPUTATION.knn_patch(btchroma,mask,p1,p2,win=win,measure='kl')
        elif method == 'knn_eucl_delta':
            recon,used_cols = IMPUTATION.knn_patch_delta(btchroma,mask,p1,p2,win=win,measure='eucl')
        elif method == 'lintrans':
            recon,proj = IMPUTATION.lintransform_patch(btchroma,mask,p1,p2,win=win)
        elif method == 'kalman':
            import kalman_toolbox as KALMAN
            recon = KALMAN.imputation(btchroma,p1,p2,
                                      dimstates=nstates,**kwargs)
        elif method == 'hmm':
            recon,recon2,hmm = IMPUTATION_HMM.imputation(btchroma*mask,p1,p2,
                                                         nstates=nstates,
                                                         **kwargs)
        elif method == 'siplca':
            res = IMPUTATION_PLCA.SIPLCA_mask.analyze((btchroma*mask).copy(),
                                                      rank,mask,win=win,
                                                      convergence_thresh=1e-15,
                                                      **kwargs)
            W, Z, H, norm, recon, logprob = res
        elif method == 'siplca2':
            res = IMPUTATION_PLCA.SIPLCA2_mask.analyze((btchroma*mask).copy(),
                                                       rank,mask,win=win,
                                                       convergence_thresh=1e-15,
                                                       **kwargs)
            W, Z, H, norm, recon, logprob = res
        else:
            print 'unknown method:',method
            return
        ########## ALGORITHM DEPENDENT END
        # measure recon
        errs = recon_error(btchroma,mask,recon,measure='all')
        all_errs.append( errs )
        total_cnt += 1
    # done
    if verbose > 0:
        print 'number of songs tested:',total_cnt
        for meas in np.sort(all_errs[0].keys()):
            errs = map(lambda d: d[meas], all_errs)
            print 'average',meas,'=',np.mean(errs),'(',np.std(errs),')'
    return all_errs
    import scipy.io as sio

    niter = 100
    D = 100
    if len(sys.argv) > 1:
        niter = int(sys.argv[1])
        print 'niter =', niter
    if len(sys.argv) > 2:
        D = int(sys.argv[2])
        print 'D =', D

    btchroma = sio.loadmat(
        '/home/thierry/Columbia/covers80/coversongs/covers32k/Caroline_No/beach_boys+Pet_Sounds+13-Caroline_No.mp3.mat'
    )['btchroma']

    mask, p1, p2 = masking.random_patch_mask(btchroma, ncols=10)

    # init data
    np.random.seed(8484)
    A = np.eye(D) * np.random.rand(1, D) / np.sqrt(D)
    C = np.random.rand(12, D)
    Q = map(lambda x: np.dot(x.T, x), [np.random.rand(D, D)])[0] / np.sqrt(D)
    R = map(lambda x: np.dot(x.T, x),
            [np.random.rand(12, 12)])[0] / np.sqrt(12)
    initx = np.random.rand(D, 1) / np.sqrt(D)
    initV = np.eye(D)

    # go
    res = learn_kalman_missing_patch(btchroma * mask,
                                     p1,
                                     p2,