import masking
import numpy as np
import scipy.io as sio
niter = 100
D = 100
if len(sys.argv) > 1:
niter = int(sys.argv[1])
print 'niter =',niter
if len(sys.argv) > 2:
D = int(sys.argv[2])
print 'D =',D
btchroma = sio.loadmat('/home/thierry/Columbia/covers80/coversongs/covers32k/Caroline_No/beach_boys+Pet_Sounds+13-Caroline_No.mp3.mat')['btchroma']
mask,p1,p2 = masking.random_patch_mask(btchroma,ncols=10)
# init data
np.random.seed(8484)
A = np.eye(D) * np.random.rand(1,D) / np.sqrt(D)
C = np.random.rand(12,D)
Q = map(lambda x: np.dot(x.T,x),[np.random.rand(D,D)])[0] / np.sqrt(D)
R = map(lambda x: np.dot(x.T,x),[np.random.rand(12,12)])[0] / np.sqrt(12)
initx = np.random.rand(D,1) / np.sqrt(D)
initV = np.eye(D)
# go
res = learn_kalman_missing_patch(btchroma*mask,p1,p2,A,C,Q,R,initx,initV,
niter,diagQ=1,diagR=1)
recon = res[0]
recon[:,:p1] = btchroma[:,:p1]
def test_maskedpatch_on_dataset(dataset,
method='random',
ncols=2,
win=1,
rank=4,
codebook=None,
nstates=1,
verbose=1,
**kwargs):
"""
Dataset is either dir, or list of files
General method to test a method on a whole dataset for one masked column
Methods are:
- random
- randomfromsong
- average
- averageall
- codebook
- knn_eucl
- knn_kl
- knn_eucl_delta
- lintrans
- kalman
- hmm
- siplca
- siplca2
Used arguments vary based on the method. For SIPLCA, we can use **kwargs
to set priors.
RETURN
- all errors, as a list of dict
"""
# get all matfiles
if type(dataset).__name__ == 'str':
matfiles = get_all_matfiles(dataset)
else:
matfiles = dataset
# init
total_cnt = 0
all_errs = []
# some specific inits
if codebook != None and not type(codebook) == type([]):
codebook = [p.reshape(12, codebook.shape[1] / 12) for p in codebook]
print 'codebook in ndarray format transformed to list'
# iterate
for matfile in matfiles:
btchroma = sio.loadmat(matfile)['btchroma']
if len(btchroma.shape) < 2:
continue
if btchroma.shape[1] < MINLENGTH or np.isnan(btchroma).any():
continue
mask, p1, p2 = MASKING.random_patch_mask(btchroma, ncols=ncols, win=25)
########## ALGORITHM DEPENDENT
if method == 'random':
recon = IMPUTATION.random_patch(btchroma, mask, p1, p2)
elif method == 'randomfromsong':
recon = IMPUTATION.random_patch_from_song(btchroma, mask, p1, p2)
elif method == 'average':
recon = IMPUTATION.average_patch(btchroma, mask, p1, p2, win=win)
elif method == 'averageall':
recon = IMPUTATION.average_patch_all(btchroma, mask, p1, p2)
elif method == 'codebook':
recon, used_codes = IMPUTATION.codebook_patch(
btchroma, mask, p1, p2, codebook)
elif method == 'knn_eucl':
recon, used_cols = IMPUTATION.knn_patch(btchroma,
mask,
p1,
p2,
win=win,
measure='eucl')
elif method == 'knn_kl':
recon, used_cols = IMPUTATION.knn_patch(btchroma,
mask,
p1,
p2,
win=win,
measure='kl')
elif method == 'knn_eucl_delta':
recon, used_cols = IMPUTATION.knn_patch_delta(btchroma,
mask,
p1,
p2,
win=win,
measure='eucl')
elif method == 'lintrans':
recon, proj = IMPUTATION.lintransform_patch(btchroma,
mask,
p1,
p2,
win=win)
elif method == 'kalman':
import kalman_toolbox as KALMAN
recon = KALMAN.imputation(btchroma,
p1,
p2,
dimstates=nstates,
**kwargs)
elif method == 'hmm':
recon, recon2, hmm = IMPUTATION_HMM.imputation(btchroma * mask,
p1,
p2,
nstates=nstates,
**kwargs)
elif method == 'siplca':
res = IMPUTATION_PLCA.SIPLCA_mask.analyze((btchroma * mask).copy(),
rank,
mask,
win=win,
convergence_thresh=1e-15,
**kwargs)
W, Z, H, norm, recon, logprob = res
elif method == 'siplca2':
res = IMPUTATION_PLCA.SIPLCA2_mask.analyze(
(btchroma * mask).copy(),
rank,
mask,
win=win,
convergence_thresh=1e-15,
**kwargs)
W, Z, H, norm, recon, logprob = res
else:
print 'unknown method:', method
return
########## ALGORITHM DEPENDENT END
# measure recon
errs = recon_error(btchroma, mask, recon, measure='all')
all_errs.append(errs)
total_cnt += 1
# done
if verbose > 0:
print 'number of songs tested:', total_cnt
for meas in np.sort(all_errs[0].keys()):
errs = map(lambda d: d[meas], all_errs)
print 'average', meas, '=', np.mean(errs), '(', np.std(errs), ')'
return all_errs
import imputation as IMPUTATION
import imputation_plca
originals = []
recons = []
texts = []
def nice_nums(val):
return evaluation.nice_nums(val)
# EASY RECONSTRUCTION
bt = sio.loadmat('/home/thierry/Columbia/covers80/coversongs/covers32kENmats/beatles+Revolver+14-Tomorrow_Never_Knows.mp3.mat')['btchroma']
mask,p1,p2 = masking.random_patch_mask(bt,ncols=15,win=25)
recon,tmp = IMPUTATION.lintransform_patch(bt,mask,p1,p2,win=1)
errs = evaluation.recon_error(bt,mask,recon)
s = 'method: linear transform\n'
s += 'eucl = ' + nice_nums(errs['eucl']) + '\n'
s += 'd1/2 = ' + nice_nums(errs['lhalf']) + '\n'
s += 'cond. ent. = ' + nice_nums(errs['condent']) + '\n'
s += 'delta diff. = ' + nice_nums(errs['ddiff']) + '\n'
s += 'D-ENT = ' + nice_nums(errs['dent']) + '\n'
s += 'Jensen diff. = ' + nice_nums(errs['jdiff']) + '\n'
originals.append( bt[:,p1:p2].copy() )
recons.append( recon[:,p1:p2].copy() )
texts.append(s)
# SMOOTH RECONSTRUCTION
def test_maskedpatch_on_dataset(dataset,method='random',ncols=2,win=1,rank=4,codebook=None,nstates=1,verbose=1,**kwargs):
"""
Dataset is either dir, or list of files
General method to test a method on a whole dataset for one masked column
Methods are:
- random
- randomfromsong
- average
- averageall
- codebook
- knn_eucl
- knn_kl
- knn_eucl_delta
- lintrans
- kalman
- hmm
- siplca
- siplca2
Used arguments vary based on the method. For SIPLCA, we can use **kwargs
to set priors.
RETURN
- all errors, as a list of dict
"""
# get all matfiles
if type(dataset).__name__ == 'str':
matfiles = get_all_matfiles(dataset)
else:
matfiles = dataset
# init
total_cnt = 0
all_errs = []
# some specific inits
if codebook != None and not type(codebook) == type([]):
codebook = [p.reshape(12,codebook.shape[1]/12) for p in codebook]
print 'codebook in ndarray format transformed to list'
# iterate
for matfile in matfiles:
btchroma = sio.loadmat(matfile)['btchroma']
if len(btchroma.shape) < 2:
continue
if btchroma.shape[1] < MINLENGTH or np.isnan(btchroma).any():
continue
mask,p1,p2 = MASKING.random_patch_mask(btchroma,ncols=ncols,win=25)
########## ALGORITHM DEPENDENT
if method == 'random':
recon = IMPUTATION.random_patch(btchroma,mask,p1,p2)
elif method == 'randomfromsong':
recon = IMPUTATION.random_patch_from_song(btchroma,mask,p1,p2)
elif method == 'average':
recon = IMPUTATION.average_patch(btchroma,mask,p1,p2,win=win)
elif method == 'averageall':
recon = IMPUTATION.average_patch_all(btchroma,mask,p1,p2)
elif method == 'codebook':
recon,used_codes = IMPUTATION.codebook_patch(btchroma,mask,p1,p2,codebook)
elif method == 'knn_eucl':
recon,used_cols = IMPUTATION.knn_patch(btchroma,mask,p1,p2,win=win,measure='eucl')
elif method == 'knn_kl':
recon,used_cols = IMPUTATION.knn_patch(btchroma,mask,p1,p2,win=win,measure='kl')
elif method == 'knn_eucl_delta':
recon,used_cols = IMPUTATION.knn_patch_delta(btchroma,mask,p1,p2,win=win,measure='eucl')
elif method == 'lintrans':
recon,proj = IMPUTATION.lintransform_patch(btchroma,mask,p1,p2,win=win)
elif method == 'kalman':
import kalman_toolbox as KALMAN
recon = KALMAN.imputation(btchroma,p1,p2,
dimstates=nstates,**kwargs)
elif method == 'hmm':
recon,recon2,hmm = IMPUTATION_HMM.imputation(btchroma*mask,p1,p2,
nstates=nstates,
**kwargs)
elif method == 'siplca':
res = IMPUTATION_PLCA.SIPLCA_mask.analyze((btchroma*mask).copy(),
rank,mask,win=win,
convergence_thresh=1e-15,
**kwargs)
W, Z, H, norm, recon, logprob = res
elif method == 'siplca2':
res = IMPUTATION_PLCA.SIPLCA2_mask.analyze((btchroma*mask).copy(),
rank,mask,win=win,
convergence_thresh=1e-15,
**kwargs)
W, Z, H, norm, recon, logprob = res
else:
print 'unknown method:',method
return
########## ALGORITHM DEPENDENT END
# measure recon
errs = recon_error(btchroma,mask,recon,measure='all')
all_errs.append( errs )
total_cnt += 1
# done
if verbose > 0:
print 'number of songs tested:',total_cnt
for meas in np.sort(all_errs[0].keys()):
errs = map(lambda d: d[meas], all_errs)
print 'average',meas,'=',np.mean(errs),'(',np.std(errs),')'
return all_errs
import scipy.io as sio
niter = 100
D = 100
if len(sys.argv) > 1:
niter = int(sys.argv[1])
print 'niter =', niter
if len(sys.argv) > 2:
D = int(sys.argv[2])
print 'D =', D
btchroma = sio.loadmat(
'/home/thierry/Columbia/covers80/coversongs/covers32k/Caroline_No/beach_boys+Pet_Sounds+13-Caroline_No.mp3.mat'
)['btchroma']
mask, p1, p2 = masking.random_patch_mask(btchroma, ncols=10)
# init data
np.random.seed(8484)
A = np.eye(D) * np.random.rand(1, D) / np.sqrt(D)
C = np.random.rand(12, D)
Q = map(lambda x: np.dot(x.T, x), [np.random.rand(D, D)])[0] / np.sqrt(D)
R = map(lambda x: np.dot(x.T, x),
[np.random.rand(12, 12)])[0] / np.sqrt(12)
initx = np.random.rand(D, 1) / np.sqrt(D)
initV = np.eye(D)
# go
res = learn_kalman_missing_patch(btchroma * mask,
p1,
p2,