def run_algorithm(audio_file,
n_templates=[0, 0, 0],
output_savename="extracted_loop"):
"""Complete pipeline of algorithm.
Parameters
----------
audio_file : string
Path to audio file to be loaded and analysed.
n_templates : list of length 3
The number of sound, rhythm and loop templates.
Default value (0,0,0) causes the script to estimate reasonable values.
output_savename: : string
Base string for saved output filenames.
Returns
-------
A set of files containing the extracted loops.
Examples
--------
>>> run_algorithm("example_song.mp3", [40,20,7], "extracted_loop")
See also
--------
tensorly.decomposition.non_negative_tucker
"""
assert os.path.exists(audio_file)
assert len(n_templates) == 3
assert type(n_templates) is list
# Load mono audio:
signal_mono, fs = librosa.load(audio_file, sr=None, mono=True)
# Use madmom to estimate the downbeat times:
downbeat_times = get_downbeats(signal_mono)
# Convert times to frames so we segment signal:
downbeat_frames = librosa.time_to_samples(downbeat_times, sr=fs)
# Create spectral cube out of signal:
spectral_cube = make_spectral_cube(signal_mono, downbeat_frames)
# Validate the input n_templates (inventing new ones if any is wrong):
n_sounds, n_rhythms, n_loops = validate_template_sizes(
spectral_cube, n_templates)
# Use TensorLy to do the non-negative Tucker decomposition:
core, factors = tld.non_negative_tucker(np.abs(spectral_cube),
[n_sounds, n_rhythms, n_loops],
n_iter_max=500,
verbose=True)
# Reconstruct each loop:
for ith_loop in range(n_loops):
# Multiply templates together to get real loop spectrum:
loop_spectrum = create_loop_spectrum(factors[0], factors[1],
core[:, :, ith_loop])
# Choose best bar to reconstruct from (we will use its phase):
bar_ind = choose_bar_to_reconstruct(factors[2], ith_loop)
# Reconstruct loop signal by masking original spectrum:
ith_loop_signal = get_loop_signal(loop_spectrum,
spectral_cube[:, :, bar_ind])
# Write signal to disk:
librosa.output.write_wav(
"{0}_{1}.wav".format(output_savename, ith_loop), ith_loop_signal,
fs)
def decomposition_elbow(dat):
pal = sns.color_palette('bright',10)
palg = sns.color_palette('Greys',10)
mat1 = np.zeros((5,15))
#finding the elbow point
for i in range(2,15):
for j in range(1,5):
facs_overall = non_negative_tucker(dat,rank=[j,i,i],random_state = 2336)
mat1[j,i] = np.mean((dat- tl.tucker_to_tensor(tucker_tensor = (facs_overall[0],facs_overall[1])))**2)
figsize(10,5)
plt.plot(2+np.arange(13),mat1[2][2:],c = 'red',label = 'rank = (2,x,x)')
plt.plot(2+np.arange(13),mat1[1][2:],c = 'blue',label = 'rank = (1,x,x)')
plt.xlabel('x')
plt.ylabel('error')
plt.show()
def tissue_module_plots(dat, person_rank, rank, nbs, cells_of_interest, random_state=0):
facs_overall = non_negative_tucker(dat, rank=[person_rank, rank, rank], random_state=random_state)
print(facs_overall[0].shape)
sns.heatmap(pd.DataFrame(facs_overall[1][1], index=nbs))
plt.ylabel('CN')
plt.xlabel('CN module')
plt.title('CN modules')
plt.show()
sns.heatmap(pd.DataFrame(facs_overall[1][2], index=cells_of_interest))
plt.ylabel('CT')
plt.xlabel('CT module')
plt.title('CT modules')
plt.show()
print('--------Tissue modules ---------')
for i in range(person_rank):
sns.heatmap(pd.DataFrame(facs_overall[0][i]))
plt.ylabel('CN module')
plt.xlabel('CT module')
plt.title('Tissue module {}'.format(i))
plt.show()
return facs_overall
unfolding = tl.unfold(tensor, 1)
original_shape = tensor.shape
tl.fold(unfolding, mode=1, shape=original_shape)
# In[82]:
from tensorly.decomposition import non_negative_tucker
random_state = 1234
# In[96]:
core, factors = non_negative_tucker(tensor,
rank=[3, 2, 1],
n_iter_max=10,
init='random',
tol=0.0001,
random_state=None,
verbose=False,
ranks=None)
# In[97]:
core.shape
# In[98]:
core
# In[99]:
len(factors)
def Tucker(datain, _x, _y, _z):
# cell size and data range of coordinates
xmax, xmin, ymax, ymin = 117.073, 115.668, 40.465, 39.465
xsize, ysize = round((xmax - xmin) * 200), round((ymax - ymin) * 200)
# import data and decode geohashed location
df = pd.read_csv(datain) #,date_parser='starttime')
loc = [geohash.decode_exactly(i) for i in df['geohashed_start_loc']]
df['start_x'], df['start_y'] = [i[1] for i in loc], [i[0] for i in loc]
## prepare a new dataset for group calculation
df['datetime'] = pd.to_datetime(df['starttime'])
dt = pd.DataFrame({
'ts': df['datetime'],
'id': df['orderid'],
'x': df['start_x'],
'y': df['start_y']
})
#dt=dt.query("ts >= '{}' and ts <= '{}'".format('2018-03-08 00:00:00', '2018-04-10 00:00:00'))
dt = dt.set_index(['ts'])
dt['date'] = [(month - 5) * 31 + day - 10
for month, day in zip(dt.index.month, dt.index.day)]
dt['hour'] = dt.index.hour
dt = dt[dt['x'] <= xmax]
dt = dt[dt['x'] >= xmin]
dt = dt[dt['y'] <= ymax]
dt = dt[dt['y'] >= ymin]
x = ((np.array(dt['x']) - xmin) * ysize) // 1
y = ((np.array(dt['y']) - ymin) * ysize) // 1
dt['loc'] = y * xsize + x # transform x,y to cell id
# join data
idx = [(x, y) for x in range(24)
for y in [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14]]
dt_join = pd.DataFrame({'idx': idx})
dt_new = dt.groupby(['loc', 'hour', 'date']).count()['id']
group_count = dt_new.unstack(['loc'])
group_count['idx'] = group_count.index
group_count = pd.merge(dt_join, group_count, how='left', on='idx')
group_count = group_count.set_index(['idx'])
group_count = group_count.fillna(0)
# generate tensor to decomposition
X = np.array(group_count)
#max_cell=np.max(b,axis=0)
#m=np.tile(max_cell,(15*24,1))
#b=b/m
X = X.reshape(24, 14, -1) #15327
X = X.transpose([2, 0, 1]) #张量转置
# tensor decomposition
X = tl.tensor(X)
core, factors = non_negative_tucker(X, rank=[_x, _y, _z]) #non_negative_
# plot
for i in range(factors[1].shape[1]):
plt.plot(factors[1][:, i])
plt.ylabel('Mode Value')
plt.xlabel('Hour')
plt.savefig('pic/M1.png', dpi=300)
plt.show()
for i in range(factors[2].shape[1]):
plt.plot(factors[2][:, i])
plt.ylabel('Mode Value')
plt.xlabel('Day')
plt.savefig('pic/M2.png', dpi=300)
plt.show()
space_out = pd.DataFrame(factors[0], index=group_count.columns)
space_out['id'] = space_out.index
space_out.to_csv('space.csv', index=False)
generateShp('space.csv')
return factors[1], factors[2]
Created on Tue Jun 9 21:57:03 2020
@author: zlibn
"""
# In[] Import Neccessary Packages
import tensorly as tl
from tensorly.decomposition import non_negative_tucker
import matplotlib.pyplot as plt
import numpy as np
user_tensor = user100_tensor
# In[]
# Experiment 1: Non-negative Tucker Decomposition
core, factors = non_negative_tucker(user_tensor,
rank=(20, 10, 10, 3),
n_iter_max=3000)
reconstruction = tl.tucker_to_tensor(core, factors)
error = tl.norm(reconstruction - user_tensor) / tl.norm(user_tensor)
print(error)
user0 = user_tensor[0, :, :, :]
user0_recon = reconstruction[0, :, :, :]
user0_recon = np.round(user0_recon, 4)
# In[]
# Experiment 2: Robust PCA
#### 2.1 Generate boolean mask
mask = user_tensor > 0
import pandas as pd
import numpy as np
import pickle
import tensorly as tl
from tensorly.decomposition import tucker, parafac, non_negative_tucker, matrix_product_state
tensor = np.random.random((10, 10, 10))
# mode-1 unfolding (i.e. zeroth mode)
unfolded = tl.unfold(tensor, mode=0)
# refold the unfolded tensor
tl.fold(unfolded, mode=0, shape=tensor.shape)
factors = parafac(tensor, rank=5)
core, factors = non_negative_tucker(tensor, rank=[5, 5, 5])
file_to_use = open("icews_df.obj", 'rb')
icews_df = pickle.load(file_to_use)
file_to_use.close()
dyad_event_counts = icews_df.sum(
level=['Source Country Code', 'Target Country Code', 'Quad Count'])
dyad_event_counts.reset_index(inplace=True)
dyad_event_counts_melt = dyad_event_counts.melt(
id_vars=list(dyad_event_counts)[0:3],
value_vars=list(dyad_event_counts)[3:28],
var_name='year',
value_name='event_count')
dyad_event_counts_melt_group = dyad_event_counts_melt.groupby(
['Source Country Code', 'Target Country Code', 'year',
'Quad Count'])['event_count'].mean()
shape = tuple(map(len, dyad_event_counts_melt_group.index.levels))
events_array = np.full(shape, np.nan)
events_array[dyad_event_counts_melt_group.index.
# Here we chose to generate a random tensor from the sequence of integers from
# 1 to 1000.
# tensor generation
array = np.random.randint(1000, size=(10, 30, 40))
tensor = tl.tensor(array, dtype='float')
##############################################################################
# Non-negative Tucker
# -----------------------
# First, multiplicative update can be implemented as:
tic = time.time()
tensor_mu, error_mu = non_negative_tucker(tensor,
rank=[5, 5, 5],
tol=1e-12,
n_iter_max=100,
return_errors=True)
tucker_reconstruction_mu = tl.tucker_to_tensor(tensor_mu)
time_mu = time.time() - tic
##############################################################################
# Here, we also compute the output tensor from the decomposed factors by using
# the ``tucker_to_tensor`` function. The tensor ``tucker_reconstruction_mu`` is
# therefore a low-rank non-negative approximation of the input tensor ``tensor``.
##############################################################################
# Non-negative Tucker with HALS and FISTA
# ---------------------------------------
# HALS algorithm with FISTA can be calculated as:
for biz_id in biz_list:
if in_biz_menu(biz_id, food_items):
final_tensor[biz_dict[biz_id]][user_dict[
review[0]]][food_dict[
food_items]] = mean_user_business_rating_dict[
(k, review[0])]
else:
break
final_tensor = tl.tensor(final_tensor, dtype='float64')
#Training Tucker and PARAFAC tensor decomposition models
core, factors = non_negative_tucker(final_tensor,
ranks=[120, 20, 220],
n_iter_max=100,
init='random',
tol=0.00001,
random_state=None,
verbose=True)
factors = non_negative_parafac(final_tensor,
rank=70,
n_iter_max=100,
init='random',
tol=0.00001,
random_state=None,
verbose=True)
reconstructed_tensor = tl.kruskal_to_tensor(factors)
nonzero_mat = np.nonzero(final_tensor)
print('Image Shape：' + str(data.shape))
# In[67]:
data_float = data.astype(np.float32)
time0 = time.time()
# In[68]:
#1.compressing the data set using NTD
tucker_ranks = [100, 50, 10]
core, factors = non_negative_tucker(data_float,
rank=tucker_ranks,
n_iter_max=1000,
init='random',
tol=0.0001,
random_state=random_state,
verbose=True)
tucker_tensor = (core, factors)
#storing back the decomposed tensor
data_reconstruction = tl.tucker_to_tensor(tucker_tensor,
transpose_factors=False)
# In[69]:
print('Image Reconstruction Shape：' + str(data_reconstruction.shape))
# In[70]:
im_reconstruction = (data_reconstruction.astype(int))
from tensorly.decomposition import tucker, non_negative_tucker, non_negative_parafac
np.random.seed(1337) # 保证随机数的可复现性
DATAPATH = Config().DATAPATH
#mapHeight = 24
#mapWidth = 24
mapHeight = 16
mapWidth = 16
filePath = "../datasets/JN_Fill_2017-2020_M{}x{}_Power.h5".format(
mapHeight, mapWidth)
data, timestamps = load_stdata(filePath)
print()
data_ = data.reshape(data.shape[0], data.shape[2], data.shape[3])
data__ = data_.copy()
#使用tucker分解填充无数据的地方，这里填充的值只保存了原有数据的一些临近性、趋势性、周期性，对预测结果并不能作为考究点
X = tl.tensor(data__.reshape(data__.shape[0], data__.shape[1],
data__.shape[2]))
factors = non_negative_tucker(X, rank=3)
full_tensor = tl.tucker_to_tensor(factors)
#factors = non_negative_parafac(X, rank=3)
#full_tensor=tl.kruskal_to_tensor(factors)
data__ = full_tensor.reshape(full_tensor.shape[0], 1, full_tensor.shape[1],
full_tensor.shape[2])
outPath = "../datasets/JN_Fill_2017-2020_M{}x{}_Power_Decomposition.h5".format(
mapHeight, mapWidth)
file = h5.File(outPath, "w")
file.create_dataset("data", data=data__)
file.create_dataset("date", data=timestamps)
def perform_tucker(tensor, rank_list):
""" Perform Tucker decomposition. """
# index 0 is for core tensor, index 1 is for factors; out is a list of core and factors
return non_negative_tucker(tensor, rank_list, tol=1.0e-10, n_iter_max=2000)
articles_count += 1
print("Year: {}, Articles: {}".format(year, articles_count))
clusters = []
for year in range(0, end_date - start_date, clustering_window):
print(f"Clustering on year: {start_date + year}")
# Recupero la matrice associata a quell'anno
X = tensor[..., year]
# Creo un array di punti su cui poi farò clustering
X = np.array([[i, j] for i in range(len(X)) for j in range(len(X))
if X[i][j] > 0])
clusters.append(get_cluster_number(X, year))
print(
f"Clusters={sorted(clusters, reverse=True)}\nMean={np.mean(clusters):.2f}"
)
rank = int(np.mean(clusters))
print(f'Computing Tensor decomposition with rank={rank}')
core, factors = non_negative_tucker(tensor, rank=[rank, rank, rank])
print('Writing results to disk')
# write words dictionary to disk
with open('out/dictionary.json', 'w') as handle:
json.dump(word_map, handle)
# write factors to disk
for index, factor in enumerate(factors):
write_to_file(factor, "out/{}".format(['A', 'B', 'C'][index]))
write_to_file(core, "out/core")
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 17 11:15:38 2020
@author: zlibn
"""
import tensorly as tl
from tensorly.decomposition import non_negative_tucker
import matplotlib.pyplot as plt
import numpy as np
core, factors = non_negative_tucker(od_tensor_Jan,
rank=(10, 10, 7, 4),
n_iter_max=1000)
factorO = factors[0].T
factorD = factors[1].T
factorT = factors[3].T
factorO = normalize(factorO, norm='l1')
factorD = normalize(factorD, norm='l1')
factorT = normalize(factorT, norm='l1')
factorO_inKey = np.zeros((10, 98))
factorD_inKey = np.zeros((10, 98))
for i in range(98):
factorO_inKey[:, int(b[i, 0])] = factorO[:, i]
factorD_inKey[:, int(b[i, 0])] = factorD[:, i]
reconstruction = tl.tucker_to_tensor(core, factors)
error = tl.norm(reconstruction - od_tensor_Jan) / tl.norm(od_tensor_Jan)
from mpl_toolkits.mplot3d import axes3d, Axes3D
fig = plt.figure()
ax = Axes3D(fig)
x = np.arange(8)
y = np.arange(12)
X, Y = np.meshgrid(y, x)
ax.plot_wireframe(X, Y, data[10,:,:])
plt.title('mean removed')
plt.xlabel('trials')
plt.ylabel('time')
plt.show()
##### test on single trial
single = data[:,0,:]
core, factors = non_negative_tucker(single, [12, 12])
reconstructed = np.dot(np.dot(factors[0], core), np.transpose(factors[1]))
np.sum((reconstructed - single)**2)/single.size
plt.imshow(factors[0])
##### tuck decompostion on 3d data
core, factors = tucker(data, [8,8,8])
plt.subplot(131)
plt.imshow(factors[0])
plt.colorbar()
plt.subplot(132)
plt.imshow(factors[1])
plt.colorbar()
plt.subplot(133)
plt.imshow(factors[2])
from tensorly.decomposition import non_negative_tucker
from tensorly.decomposition import tucker
from math import ceil
from tensorly.base import tensor_to_vec, partial_tensor_to_vec
from tensorly.datasets.synthetic import gen_image
from tensorly.random import check_random_state
from tensorly.regression.kruskal_regression import KruskalRegressor
import tensorly.backend as T
import time
tl.set_backend('numpy')
rng = check_random_state(1)
X = T.tensor(rng.normal(size=(1000, 1000, 1000), loc=0, scale=1))
start_time = time.time()
core, tucker_factors = non_negative_tucker(X, rank=[10,10,10], init='svd', tol=10e-12, verbose=True, n_iter_max=2)
print("--- %s seconds ---" % (time.time() - start_time))
tl.set_backend('mxnet')
rng = check_random_state(1)
X = T.tensor(rng.normal(size=(1000, 1000, 1000), loc=0, scale=1))
start_time = time.time()
core, tucker_factors = non_negative_tucker(X, rank=[10,10,10], init='svd', tol=10e-12, verbose=True, n_iter_max=2)
print("--- %s seconds ---" % (time.time() - start_time))
tl.set_backend('pytorch')
rng = check_random_state(1)
X = T.tensor(rng.normal(size=(1000, 1000, 1000), loc=0, scale=1))
start_time = time.time()
core, tucker_factors = non_negative_tucker(X, rank=[10,10,10], init='svd', tol=10e-12, verbose=True, n_iter_max=2)
print("--- %s seconds ---" % (time.time() - start_time))
