def run_algorithm(audio_file, n_templates=[0, 0, 0], output_savename="extracted_loop"): """Complete pipeline of algorithm. Parameters ---------- audio_file : string Path to audio file to be loaded and analysed. n_templates : list of length 3 The number of sound, rhythm and loop templates. Default value (0,0,0) causes the script to estimate reasonable values. output_savename: : string Base string for saved output filenames. Returns ------- A set of files containing the extracted loops. Examples -------- >>> run_algorithm("example_song.mp3", [40,20,7], "extracted_loop") See also -------- tensorly.decomposition.non_negative_tucker """ assert os.path.exists(audio_file) assert len(n_templates) == 3 assert type(n_templates) is list # Load mono audio: signal_mono, fs = librosa.load(audio_file, sr=None, mono=True) # Use madmom to estimate the downbeat times: downbeat_times = get_downbeats(signal_mono) # Convert times to frames so we segment signal: downbeat_frames = librosa.time_to_samples(downbeat_times, sr=fs) # Create spectral cube out of signal: spectral_cube = make_spectral_cube(signal_mono, downbeat_frames) # Validate the input n_templates (inventing new ones if any is wrong): n_sounds, n_rhythms, n_loops = validate_template_sizes( spectral_cube, n_templates) # Use TensorLy to do the non-negative Tucker decomposition: core, factors = tld.non_negative_tucker(np.abs(spectral_cube), [n_sounds, n_rhythms, n_loops], n_iter_max=500, verbose=True) # Reconstruct each loop: for ith_loop in range(n_loops): # Multiply templates together to get real loop spectrum: loop_spectrum = create_loop_spectrum(factors[0], factors[1], core[:, :, ith_loop]) # Choose best bar to reconstruct from (we will use its phase): bar_ind = choose_bar_to_reconstruct(factors[2], ith_loop) # Reconstruct loop signal by masking original spectrum: ith_loop_signal = get_loop_signal(loop_spectrum, spectral_cube[:, :, bar_ind]) # Write signal to disk: librosa.output.write_wav( "{0}_{1}.wav".format(output_savename, ith_loop), ith_loop_signal, fs)
def decomposition_elbow(dat): pal = sns.color_palette('bright',10) palg = sns.color_palette('Greys',10) mat1 = np.zeros((5,15)) #finding the elbow point for i in range(2,15): for j in range(1,5): facs_overall = non_negative_tucker(dat,rank=[j,i,i],random_state = 2336) mat1[j,i] = np.mean((dat- tl.tucker_to_tensor(tucker_tensor = (facs_overall[0],facs_overall[1])))**2) figsize(10,5) plt.plot(2+np.arange(13),mat1[2][2:],c = 'red',label = 'rank = (2,x,x)') plt.plot(2+np.arange(13),mat1[1][2:],c = 'blue',label = 'rank = (1,x,x)') plt.xlabel('x') plt.ylabel('error') plt.show()
def tissue_module_plots(dat, person_rank, rank, nbs, cells_of_interest, random_state=0): facs_overall = non_negative_tucker(dat, rank=[person_rank, rank, rank], random_state=random_state) print(facs_overall[0].shape) sns.heatmap(pd.DataFrame(facs_overall[1][1], index=nbs)) plt.ylabel('CN') plt.xlabel('CN module') plt.title('CN modules') plt.show() sns.heatmap(pd.DataFrame(facs_overall[1][2], index=cells_of_interest)) plt.ylabel('CT') plt.xlabel('CT module') plt.title('CT modules') plt.show() print('--------Tissue modules ---------') for i in range(person_rank): sns.heatmap(pd.DataFrame(facs_overall[0][i])) plt.ylabel('CN module') plt.xlabel('CT module') plt.title('Tissue module {}'.format(i)) plt.show() return facs_overall
unfolding = tl.unfold(tensor, 1) original_shape = tensor.shape tl.fold(unfolding, mode=1, shape=original_shape) # In[82]: from tensorly.decomposition import non_negative_tucker random_state = 1234 # In[96]: core, factors = non_negative_tucker(tensor, rank=[3, 2, 1], n_iter_max=10, init='random', tol=0.0001, random_state=None, verbose=False, ranks=None) # In[97]: core.shape # In[98]: core # In[99]: len(factors)
def Tucker(datain, _x, _y, _z): # cell size and data range of coordinates xmax, xmin, ymax, ymin = 117.073, 115.668, 40.465, 39.465 xsize, ysize = round((xmax - xmin) * 200), round((ymax - ymin) * 200) # import data and decode geohashed location df = pd.read_csv(datain) #,date_parser='starttime') loc = [geohash.decode_exactly(i) for i in df['geohashed_start_loc']] df['start_x'], df['start_y'] = [i[1] for i in loc], [i[0] for i in loc] ## prepare a new dataset for group calculation df['datetime'] = pd.to_datetime(df['starttime']) dt = pd.DataFrame({ 'ts': df['datetime'], 'id': df['orderid'], 'x': df['start_x'], 'y': df['start_y'] }) #dt=dt.query("ts >= '{}' and ts <= '{}'".format('2018-03-08 00:00:00', '2018-04-10 00:00:00')) dt = dt.set_index(['ts']) dt['date'] = [(month - 5) * 31 + day - 10 for month, day in zip(dt.index.month, dt.index.day)] dt['hour'] = dt.index.hour dt = dt[dt['x'] <= xmax] dt = dt[dt['x'] >= xmin] dt = dt[dt['y'] <= ymax] dt = dt[dt['y'] >= ymin] x = ((np.array(dt['x']) - xmin) * ysize) // 1 y = ((np.array(dt['y']) - ymin) * ysize) // 1 dt['loc'] = y * xsize + x # transform x,y to cell id # join data idx = [(x, y) for x in range(24) for y in [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14]] dt_join = pd.DataFrame({'idx': idx}) dt_new = dt.groupby(['loc', 'hour', 'date']).count()['id'] group_count = dt_new.unstack(['loc']) group_count['idx'] = group_count.index group_count = pd.merge(dt_join, group_count, how='left', on='idx') group_count = group_count.set_index(['idx']) group_count = group_count.fillna(0) # generate tensor to decomposition X = np.array(group_count) #max_cell=np.max(b,axis=0) #m=np.tile(max_cell,(15*24,1)) #b=b/m X = X.reshape(24, 14, -1) #15327 X = X.transpose([2, 0, 1]) #张量转置 # tensor decomposition X = tl.tensor(X) core, factors = non_negative_tucker(X, rank=[_x, _y, _z]) #non_negative_ # plot for i in range(factors[1].shape[1]): plt.plot(factors[1][:, i]) plt.ylabel('Mode Value') plt.xlabel('Hour') plt.savefig('pic/M1.png', dpi=300) plt.show() for i in range(factors[2].shape[1]): plt.plot(factors[2][:, i]) plt.ylabel('Mode Value') plt.xlabel('Day') plt.savefig('pic/M2.png', dpi=300) plt.show() space_out = pd.DataFrame(factors[0], index=group_count.columns) space_out['id'] = space_out.index space_out.to_csv('space.csv', index=False) generateShp('space.csv') return factors[1], factors[2]
Created on Tue Jun 9 21:57:03 2020 @author: zlibn """ # In[] Import Neccessary Packages import tensorly as tl from tensorly.decomposition import non_negative_tucker import matplotlib.pyplot as plt import numpy as np user_tensor = user100_tensor # In[] # Experiment 1: Non-negative Tucker Decomposition core, factors = non_negative_tucker(user_tensor, rank=(20, 10, 10, 3), n_iter_max=3000) reconstruction = tl.tucker_to_tensor(core, factors) error = tl.norm(reconstruction - user_tensor) / tl.norm(user_tensor) print(error) user0 = user_tensor[0, :, :, :] user0_recon = reconstruction[0, :, :, :] user0_recon = np.round(user0_recon, 4) # In[] # Experiment 2: Robust PCA #### 2.1 Generate boolean mask mask = user_tensor > 0
import pandas as pd import numpy as np import pickle import tensorly as tl from tensorly.decomposition import tucker, parafac, non_negative_tucker, matrix_product_state tensor = np.random.random((10, 10, 10)) # mode-1 unfolding (i.e. zeroth mode) unfolded = tl.unfold(tensor, mode=0) # refold the unfolded tensor tl.fold(unfolded, mode=0, shape=tensor.shape) factors = parafac(tensor, rank=5) core, factors = non_negative_tucker(tensor, rank=[5, 5, 5]) file_to_use = open("icews_df.obj", 'rb') icews_df = pickle.load(file_to_use) file_to_use.close() dyad_event_counts = icews_df.sum( level=['Source Country Code', 'Target Country Code', 'Quad Count']) dyad_event_counts.reset_index(inplace=True) dyad_event_counts_melt = dyad_event_counts.melt( id_vars=list(dyad_event_counts)[0:3], value_vars=list(dyad_event_counts)[3:28], var_name='year', value_name='event_count') dyad_event_counts_melt_group = dyad_event_counts_melt.groupby( ['Source Country Code', 'Target Country Code', 'year', 'Quad Count'])['event_count'].mean() shape = tuple(map(len, dyad_event_counts_melt_group.index.levels)) events_array = np.full(shape, np.nan) events_array[dyad_event_counts_melt_group.index.
# Here we chose to generate a random tensor from the sequence of integers from # 1 to 1000. # tensor generation array = np.random.randint(1000, size=(10, 30, 40)) tensor = tl.tensor(array, dtype='float') ############################################################################## # Non-negative Tucker # ----------------------- # First, multiplicative update can be implemented as: tic = time.time() tensor_mu, error_mu = non_negative_tucker(tensor, rank=[5, 5, 5], tol=1e-12, n_iter_max=100, return_errors=True) tucker_reconstruction_mu = tl.tucker_to_tensor(tensor_mu) time_mu = time.time() - tic ############################################################################## # Here, we also compute the output tensor from the decomposed factors by using # the ``tucker_to_tensor`` function. The tensor ``tucker_reconstruction_mu`` is # therefore a low-rank non-negative approximation of the input tensor ``tensor``. ############################################################################## # Non-negative Tucker with HALS and FISTA # --------------------------------------- # HALS algorithm with FISTA can be calculated as:
for biz_id in biz_list: if in_biz_menu(biz_id, food_items): final_tensor[biz_dict[biz_id]][user_dict[ review[0]]][food_dict[ food_items]] = mean_user_business_rating_dict[ (k, review[0])] else: break final_tensor = tl.tensor(final_tensor, dtype='float64') #Training Tucker and PARAFAC tensor decomposition models core, factors = non_negative_tucker(final_tensor, ranks=[120, 20, 220], n_iter_max=100, init='random', tol=0.00001, random_state=None, verbose=True) factors = non_negative_parafac(final_tensor, rank=70, n_iter_max=100, init='random', tol=0.00001, random_state=None, verbose=True) reconstructed_tensor = tl.kruskal_to_tensor(factors) nonzero_mat = np.nonzero(final_tensor)
print('Image Shape:' + str(data.shape)) # In[67]: data_float = data.astype(np.float32) time0 = time.time() # In[68]: #1.compressing the data set using NTD tucker_ranks = [100, 50, 10] core, factors = non_negative_tucker(data_float, rank=tucker_ranks, n_iter_max=1000, init='random', tol=0.0001, random_state=random_state, verbose=True) tucker_tensor = (core, factors) #storing back the decomposed tensor data_reconstruction = tl.tucker_to_tensor(tucker_tensor, transpose_factors=False) # In[69]: print('Image Reconstruction Shape:' + str(data_reconstruction.shape)) # In[70]: im_reconstruction = (data_reconstruction.astype(int))
from tensorly.decomposition import tucker, non_negative_tucker, non_negative_parafac np.random.seed(1337) # 保证随机数的可复现性 DATAPATH = Config().DATAPATH #mapHeight = 24 #mapWidth = 24 mapHeight = 16 mapWidth = 16 filePath = "../datasets/JN_Fill_2017-2020_M{}x{}_Power.h5".format( mapHeight, mapWidth) data, timestamps = load_stdata(filePath) print() data_ = data.reshape(data.shape[0], data.shape[2], data.shape[3]) data__ = data_.copy() #使用tucker分解填充无数据的地方,这里填充的值只保存了原有数据的一些临近性、趋势性、周期性,对预测结果并不能作为考究点 X = tl.tensor(data__.reshape(data__.shape[0], data__.shape[1], data__.shape[2])) factors = non_negative_tucker(X, rank=3) full_tensor = tl.tucker_to_tensor(factors) #factors = non_negative_parafac(X, rank=3) #full_tensor=tl.kruskal_to_tensor(factors) data__ = full_tensor.reshape(full_tensor.shape[0], 1, full_tensor.shape[1], full_tensor.shape[2]) outPath = "../datasets/JN_Fill_2017-2020_M{}x{}_Power_Decomposition.h5".format( mapHeight, mapWidth) file = h5.File(outPath, "w") file.create_dataset("data", data=data__) file.create_dataset("date", data=timestamps)
def perform_tucker(tensor, rank_list): """ Perform Tucker decomposition. """ # index 0 is for core tensor, index 1 is for factors; out is a list of core and factors return non_negative_tucker(tensor, rank_list, tol=1.0e-10, n_iter_max=2000)
articles_count += 1 print("Year: {}, Articles: {}".format(year, articles_count)) clusters = [] for year in range(0, end_date - start_date, clustering_window): print(f"Clustering on year: {start_date + year}") # Recupero la matrice associata a quell'anno X = tensor[..., year] # Creo un array di punti su cui poi farò clustering X = np.array([[i, j] for i in range(len(X)) for j in range(len(X)) if X[i][j] > 0]) clusters.append(get_cluster_number(X, year)) print( f"Clusters={sorted(clusters, reverse=True)}\nMean={np.mean(clusters):.2f}" ) rank = int(np.mean(clusters)) print(f'Computing Tensor decomposition with rank={rank}') core, factors = non_negative_tucker(tensor, rank=[rank, rank, rank]) print('Writing results to disk') # write words dictionary to disk with open('out/dictionary.json', 'w') as handle: json.dump(word_map, handle) # write factors to disk for index, factor in enumerate(factors): write_to_file(factor, "out/{}".format(['A', 'B', 'C'][index])) write_to_file(core, "out/core")
# -*- coding: utf-8 -*- """ Created on Fri Apr 17 11:15:38 2020 @author: zlibn """ import tensorly as tl from tensorly.decomposition import non_negative_tucker import matplotlib.pyplot as plt import numpy as np core, factors = non_negative_tucker(od_tensor_Jan, rank=(10, 10, 7, 4), n_iter_max=1000) factorO = factors[0].T factorD = factors[1].T factorT = factors[3].T factorO = normalize(factorO, norm='l1') factorD = normalize(factorD, norm='l1') factorT = normalize(factorT, norm='l1') factorO_inKey = np.zeros((10, 98)) factorD_inKey = np.zeros((10, 98)) for i in range(98): factorO_inKey[:, int(b[i, 0])] = factorO[:, i] factorD_inKey[:, int(b[i, 0])] = factorD[:, i] reconstruction = tl.tucker_to_tensor(core, factors) error = tl.norm(reconstruction - od_tensor_Jan) / tl.norm(od_tensor_Jan)
from mpl_toolkits.mplot3d import axes3d, Axes3D fig = plt.figure() ax = Axes3D(fig) x = np.arange(8) y = np.arange(12) X, Y = np.meshgrid(y, x) ax.plot_wireframe(X, Y, data[10,:,:]) plt.title('mean removed') plt.xlabel('trials') plt.ylabel('time') plt.show() ##### test on single trial single = data[:,0,:] core, factors = non_negative_tucker(single, [12, 12]) reconstructed = np.dot(np.dot(factors[0], core), np.transpose(factors[1])) np.sum((reconstructed - single)**2)/single.size plt.imshow(factors[0]) ##### tuck decompostion on 3d data core, factors = tucker(data, [8,8,8]) plt.subplot(131) plt.imshow(factors[0]) plt.colorbar() plt.subplot(132) plt.imshow(factors[1]) plt.colorbar() plt.subplot(133) plt.imshow(factors[2])
from tensorly.decomposition import non_negative_tucker from tensorly.decomposition import tucker from math import ceil from tensorly.base import tensor_to_vec, partial_tensor_to_vec from tensorly.datasets.synthetic import gen_image from tensorly.random import check_random_state from tensorly.regression.kruskal_regression import KruskalRegressor import tensorly.backend as T import time tl.set_backend('numpy') rng = check_random_state(1) X = T.tensor(rng.normal(size=(1000, 1000, 1000), loc=0, scale=1)) start_time = time.time() core, tucker_factors = non_negative_tucker(X, rank=[10,10,10], init='svd', tol=10e-12, verbose=True, n_iter_max=2) print("--- %s seconds ---" % (time.time() - start_time)) tl.set_backend('mxnet') rng = check_random_state(1) X = T.tensor(rng.normal(size=(1000, 1000, 1000), loc=0, scale=1)) start_time = time.time() core, tucker_factors = non_negative_tucker(X, rank=[10,10,10], init='svd', tol=10e-12, verbose=True, n_iter_max=2) print("--- %s seconds ---" % (time.time() - start_time)) tl.set_backend('pytorch') rng = check_random_state(1) X = T.tensor(rng.normal(size=(1000, 1000, 1000), loc=0, scale=1)) start_time = time.time() core, tucker_factors = non_negative_tucker(X, rank=[10,10,10], init='svd', tol=10e-12, verbose=True, n_iter_max=2) print("--- %s seconds ---" % (time.time() - start_time))