def test_cp_decomposition(max_d_size, num_dims, d_interval, max_rank,
rank_interval, num_samples):
"""
Purpose:
benchmark the cp decomposition of a randomly generated CP decomposable tensor
run tests using hypercube tensors for consistency
:param max_d_size: maximum dimension size that each mode will reach
:param num_dims: number of dimensions to test along
:param d_interval: size of interval to jump by for each data point
:param max_rank: maximum rank to test against
:param rank_interval: size of interval for rank to jump by for each data point
:param num_samples: number of items to sample over for each data point
"""
rand_state = 5
for r in range(1, max_rank, rank_interval):
dims = []
times = []
for d in range(2, max_d_size, d_interval):
time_sum = 0
print(d)
for n in range(0, num_samples):
shp = tuple([d] * num_dims)
t = rnd.cp_tensor(shp, r, full=True, random_state=rand_state)
start = time.time()
parafac(t, rank=r, tol=10e-6, random_state=rand_state)
end = time.time()
time_sum += end - start
dims.append(d)
times.append(time_sum / num_samples)
plt.plot(dims, times, label='r = ' + str(r))
plt.xlabel("Matrix dimension (square matrix)")
plt.ylabel("Time elapsed (sec)")
plt.legend(loc='best')
plt.savefig('test_cp_decomposition.eps', format='eps', dpi=1000)
def get_cp_factors(self):
if self.pretrained is not None:
mat_dict = scipy.io.loadmat(self.pretrained)
if mat_dict['R'][0][0] != self.rank:
print('WRONG FACTORS, do not correspond to desired rank')
PU_z, PU_cout, PU_cin = [Ui[0] for Ui in mat_dict['P_bals_epc_U']]
Plmbda = mat_dict['P_bals_epc_lambda'].ravel()
f_cin = np.array(PU_cin)
f_cout = np.array(PU_cout)
f_z = (np.array(PU_z)*(Plmbda))
else:
bias = self.bias
if isinstance(self.layer, nn.Sequential):
# Tensorly case
_, (f_cout, f_cin, f_z) = parafac(kruskal_to_tensor((None, self.weight)),
self.rank,
n_iter_max=5000,
init='random',
tol=1e-8,
svd = None,
cvg_criterion = 'rec_error')
else:
# Tensorly case
_, (f_cout, f_cin, f_z) = parafac(self.weight,
self.rank,
n_iter_max=5000,
init='random',
tol=1e-8,
svd = None,
cvg_criterion = 'rec_error')
# # Reshape factor matrices to 4D weight tensors
# f_cin: (cin, rank) -> (rank, cin, 1, 1)
# f_z: (z, rank) -> (rank, 1, h, w)
# f_cout: (count, rank) -> (count, rank, 1, 1)
# Pytorh case
f_cin = f_cin.t().unsqueeze_(2).unsqueeze_(3).contiguous()
f_z = torch.einsum('hwr->rhw', f_z.resize_((*self.kernel_size, self.rank))\
).unsqueeze_(1).contiguous()
f_cout = f_cout.unsqueeze_(2).unsqueeze_(3).contiguous()
return [f_cin, f_z, f_cout], [None, None, bias]
def loss(self, *views):
m = views[0].size(0)
views = _demean(*views)
covs = [
(1 - self.r) * view.T @ view
+ self.r * torch.eye(view.size(1), device=view.device)
for view in views
]
whitened_z = [
view @ mat_pow(cov, -0.5, self.eps) for view, cov in zip(views, covs)
]
# The idea here is to form a matrix with M dimensions one for each view where at index
# M[p_i,p_j,p_k...] we have the sum over n samples of the product of the pth feature of the
# ith, jth, kth view etc.
for i, el in enumerate(whitened_z):
# To achieve this we start with the first view so M is nxp.
if i == 0:
M = el
# For the remaining views we expand their dimensions to match M i.e. nx1x...x1xp
else:
for _ in range(len(M.size()) - 1):
el = torch.unsqueeze(el, 1)
# Then we perform an outer product by expanding the dimensionality of M and
# outer product with the expanded el
M = torch.unsqueeze(M, -1) @ el
M = torch.mean(M, 0)
tl.set_backend("pytorch")
M_parafac = parafac(
M.detach(), self.latent_dims, verbose=False, normalize_factors=True
)
M_parafac.weights = 1
M_hat = cp_to_tensor(M_parafac)
return torch.linalg.norm(M - M_hat)
def decompose_moment3(self):
"""
Performs CP decomposition on third moment.
:return:
"""
return np.sort(np.array(parafac(self.whitened_moment3,
self.n_topics)))[::-1]
def test(G, verbose=False, compare_parafac=False):
print "rank of G:", G.rank
print "G:"
print G.get_full_tensor()
print ""
c = Compressor(accuracy=0.0001,
n_iter_max=1000,
min_error_dec=1e-2,
display_progress=verbose)
F, info = c.compress(G)
print "rank of F:", F.rank
print "number of iter:", info.n_iter
print "where the rank is increased:", info.iter_with_rank_inc
print "F:"
print F.get_full_tensor()
print ""
if verbose:
print "lambdas in F:"
print F.lambdas
print "factors in F:"
for f in F.factors:
print f
print ""
if compare_parafac:
G_tl = tl.tensor(G.get_full_tensor())
factors_tl = parafac(G_tl, rank=2)
for f in factors_tl:
print f
print ""
print tl.kruskal_to_tensor(factors_tl)
print ""
def cp_filter(imgarray, cp_rank):
"""
imgarray is 3-d numpy array of png image
"""
imgtensor = tl.tensor(imgarray)
factors = parafac(imgtensor, rank=cp_rank, init='random', tol=10e-6)
pass
def torch_cp_decomp(W, rank):
last, first, vertical, horizontal = parafac(W, rank=rank, init='random')
sr = first.t_()
rt = last
rr = torch.stack([vertical.narrow(1, i, 1) @ torch.t(horizontal).narrow(0, i, 1) for i in range(rank)]).unsqueeze_(
1)
return [sr, rr, rt]
def fit(self, *views: Tuple[np.ndarray, ...], ):
if self.c is None:
self.c = [0] * len(views)
assert (len(self.c) == len(views)), 'c requires as many values as #views'
z = self.demean_data(*views)
n = z[0].shape[0]
covs = [(1 - self.c[i]) * view.T @ view / (1 - n) + self.c[i] * np.eye(view.shape[1]) for i, view in
enumerate(z)]
covs_invsqrt = [np.linalg.inv(sqrtm(cov)) for cov in covs]
z = [z_ @ cov_invsqrt for z_, cov_invsqrt in zip(z, covs_invsqrt)]
for i, el in enumerate(z):
if i == 0:
M = el
else:
for _ in range(len(M.shape) - 1):
el = np.expand_dims(el, 1)
M = np.expand_dims(M, -1) @ el
M = np.mean(M, 0)
# for i, cov_invsqrt in enumerate(covs_invsqrt):
# M = np.tensordot(M, cov_invsqrt, axes=[[0], [0]])
tl.set_backend('numpy')
M_parafac = parafac(M, self.latent_dims, verbose=True)
self.weights_list = [cov_invsqrt @ fac for i, (view, cov_invsqrt, fac) in
enumerate(zip(z, covs_invsqrt, M_parafac.factors))]
self.score_list = [view @ self.weights_list[i] for i, view in enumerate(z)]
self.weights_list = [weights / np.linalg.norm(score) for weights, score in
zip(self.weights_list, self.score_list)]
self.score_list = [view @ self.weights_list[i] for i, view in enumerate(z)]
self.train_correlations = self.predict_corr(*views)
return self
def fit(
self,
*views: np.ndarray,
):
self.n_views = len(views)
self.check_params()
assert (len(
self.c) == len(views)), 'c requires as many values as #views'
train_views, covs_invsqrt = self.setup_tensor(*views)
for i, el in enumerate(train_views):
if i == 0:
M = el
else:
for _ in range(len(M.shape) - 1):
el = np.expand_dims(el, 1)
M = np.expand_dims(M, -1) @ el
M = np.mean(M, 0)
tl.set_backend('numpy')
M_parafac = parafac(M, self.latent_dims, verbose=True)
self.alphas = [
cov_invsqrt @ fac for i, (view, cov_invsqrt, fac) in enumerate(
zip(train_views, covs_invsqrt, M_parafac.factors))
]
self.score_list = [
view @ self.alphas[i] for i, view in enumerate(train_views)
]
self.weights_list = [
weights / np.linalg.norm(score)
for weights, score in zip(self.alphas, self.score_list)
]
self.score_list = [
view @ self.weights_list[i] for i, view in enumerate(train_views)
]
self.train_correlations = self.predict_corr(*views)
return self
def factorize_dense_cp(tensor, params):
input_shape, output_shape, rank = params["input_shape"], params[
"output_shape"], params["rank"]
shape = tensor.shape
assert len(shape) == 2, "The tensor should be 2-order."
order = len(input_shape)
assert len(output_shape) == order, \
"The lengths of input and output shape should match."
assert shape[0] == np.prod(input_shape), \
"The product of input_shape should match 1st-dimension of the tensor."
assert shape[1] == np.prod(output_shape), \
"The product of output_shape should match 2nd-dimension of the tensor."
tensor = np.reshape(tensor, input_shape + output_shape)
tensor = np.transpose(
tensor,
axes=[
val for pair in zip(range(order), range(order, 2 * order))
for val in pair
])
tensor = np.reshape(tensor, np.multiply(input_shape, output_shape))
factors = parafac(tensor, rank)
for l in range(order):
factors[l] = np.reshape(np.transpose(factors[l]),
[rank, input_shape[l], output_shape[l]])
return factors
def decomposition_interact_re_button(self, foo):
dataset = self.EEMstack_cw[self.datlist_cw.index(
self.range1.value):self.datlist_cw.index(self.range2.value) + 1]
if self.decomposition_method_list.value == 'parafac':
factors = parafac(dataset, rank=self.rank_display.value)
elif self.decomposition_method_list.value == 'non_negative_parafac':
factors = non_negative_parafac(dataset,
rank=self.rank_display.value)
elif self.decomposition_method_list.value == 'test_function':
factors = non_negative_parafac(dataset,
rank=self.rank_display.value,
fixed_modes=[0, 1],
init="random")
I_0 = factors[1][0]
J_0 = factors[1][1]
K_0 = factors[1][2]
decomposition_reconstruction_interact(
I_0,
J_0,
K_0,
self.EEMstack_cw[self.datlist_cw.index(self.data_to_view.value)],
self.Em_range_cw,
self.Ex_range_cw,
self.datlist_cw[self.datlist_cw.index(self.range1.value):self.
datlist_cw.index(self.range2.value) + 1],
self.data_to_view.value,
crange=self.crange_cw.value)
def do_tensor(input_movie_ids):
# read the pickle file that contains tensor
actor_movie_year_3d_matrix = cPickle.load(
open("actor_movie_genre_tensor.pkl", "rb"))
actor_movie_year_array = np.array(actor_movie_year_3d_matrix)
# perform cp decomposition
decomposed = parafac(actor_movie_year_array,
no_of_components,
init='random')
mlmovies = util.read_mlmovies()
mlmovies = mlmovies.loc[mlmovies['year'] >= util.movie_year_for_tensor]
movies_list = mlmovies.movieid.unique()
# data frame for movie factor matrix from cp decomposition
decomposed_movies_df = pd.DataFrame(decomposed[1], index=movies_list)
# dataframe containing only input movies
input_movie_df = decomposed_movies_df.loc[input_movie_ids]
output_movies = {}
# finding cosine similarity of each movie vector with the input movie vector and fetching the top 5 values
for index, movie in decomposed_movies_df.iterrows():
cosine_sum = 0
order = 1
for j, input_movie in input_movie_df.iterrows():
cosine_sum += (1 - cosine(movie, input_movie)) * order
order -= order_factor
output_movies[index] = cosine_sum
return output_movies, decomposed_movies_df, movies_list
def __init__(self, num_filters, filter_h, filter_w, image_channels, rank):
self.image_channels = image_channels
self.num_filters = num_filters
self.filter_h = filter_h
self.filter_w = filter_w
self.rank = rank
self.filters = np.random.randn(num_filters, filter_h, filter_w,
image_channels) / (filter_h * filter_w)
tensor = tl.tensor(self.filters)
##initialize the cp-decomposed convolutional factors
self.factors = parafac(tensor, rank)
self.filters_recon = tl.kruskal_to_tensor((self.factors))
##initialize moments and parameters for adam
self.v0 = np.zeros(self.factors[0].shape)
self.v1 = np.zeros(self.factors[1].shape)
self.v2 = np.zeros(self.factors[2].shape)
self.v3 = np.zeros(self.factors[3].shape)
self.v = [self.v0, self.v1, self.v2, self.v3]
self.s0 = np.zeros(self.factors[0].shape)
self.s1 = np.zeros(self.factors[1].shape)
self.s2 = np.zeros(self.factors[2].shape)
self.s3 = np.zeros(self.factors[3].shape)
self.s = [self.s0, self.s1, self.s2, self.s3]
self.beta1 = 0.99
self.beta2 = 0.999
def linear_decomp(self, layer):
W = layer.weight.data
ranks = self.estimate_rank(W)
print('Ranks:', ranks)
if ranks == 0:
return layer
last, first = [None for _ in range(2)]
if self.decomp_type == 'tucker':
first, [last] = partial_tucker(W,
modes=[0],
ranks=ranks,
init='svd')
first_layer = nn.Linear(first.shape[1], first.shape[0], bias=False)
first_layer.weight.data = first
last_layer = nn.Linear(last.shape[1], last.shape[0], bias=True)
last_layer.weight.data = last
elif self.decomp_type == 'cp':
weights, [last, first] = parafac(W, rank=ranks, init='random')
first_layer = nn.Linear(first.shape[0], first.shape[1], bias=False)
first_layer.weight.data = first.t()
last_layer = nn.Linear(last.shape[1], last.shape[0], bias=True)
last_layer.weight.data = last
if layer.bias is not None:
last_layer.bias.data = layer.bias.data
new_layer = nn.Sequential(first_layer, last_layer)
return new_layer
def grouping_use_cp_on_actor_tensor(actor_movie_year_tensor, amy_info):
# print(actor_movie_year_tensor)
U = actor_movie_year_tensor
T = tensor(U.reshape((U.shape[0], U.shape[1], U.shape[2])))
# Compute rank-5 CP decomposition of Tensor with ALS
P = decomposition.parafac(T, 5, init="random")
# Result is a decomposed tensor stored as a Kruskal operator in P
#fit: float
# Fit of the factorization compared to Tensor
#itr : int
# Number of iterations that were needed until convergence
#exectimes : ndarray of floats
# Time needed for each single iteration
X = P[0].asnumpy()
Y = P[1].asnumpy()
Z = P[2].asnumpy()
print("Top 5 latent sementics for Actor :")
print(X)
print("Top 5 latent sementics for Movie :")
print(Y)
print("Top 5 latent sementics for Year :")
print(Z)
print("Actor Groupings:")
create_n_groupings(X, amy_info[0], 5)
print("Movie Groupings:")
create_n_groupings(Y, list(amy_info[2].keys()), 5)
print("Year Groupings:")
create_n_groupings(Z, amy_info[4], 5)
def grouping_use_cp_on_tag_tensor(tag_movie_rating, tmr_info):
U = tag_movie_rating
# T = vec_to_tensor(U, (U.shape[0], U.shape[1], U.shape[2]))
T = tensor(U.reshape((U.shape[0], U.shape[1], U.shape[2])))
# Compute rank-5 CP decomposition of Tensor with ALS and for Hosvd method replace init as nvecs
P = decomposition.parafac(T, 5, init="random")
# Result is a decomposed tensor stored as a Kruskal operator in P
# fit: float
# Fit of the factorization compared to Tensor
# itr : int
# Number of iterations that were needed until convergence
# exectimes : ndarray of floats
# Time needed for each single iteration
X = P[0].asnumpy()
Y = P[1].asnumpy()
Z = P[2].asnumpy()
print("Top 5 latent sementics for Tag :")
print(X)
print("Top 5 latent sementics for Movie :")
print(Y)
print("Top 5 latent sementics for Rating :")
print(Z)
print("Tag Groups")
create_n_groupings(X, tmr_info[0], 5)
print("Movie Groups")
create_n_groupings(Y, tmr_info[2], 5)
print(" Rating Groups:")
create_n_groupings(Z, tmr_info[4], 5)
def factorize_conv2d_rcp(tensor, params):
input_shape, output_shape, rank = params["input_shape"], params["output_shape"], params["rank"]
shape = tensor.shape
assert len(shape) == 4, "The tensor should be 4-order."
order = len(input_shape)
assert len(output_shape) == order, \
"The lengths of input shape and output shape should match."
assert shape[2] == np.prod(input_shape), \
"The product of input shape should match the 3rd-dimension of the tensor."
assert shape[3] == np.prod(output_shape), \
"The product of output shape should match the 4th-dimension of the tensor."
if shape[0] == 1:
tensor = np.reshape(tensor, input_shape + output_shape)
tensor = np.transpose(tensor, axes = [val for pair in zip(range(order), range(order, 2*order)) for val in pair])
tensor = np.reshape(tensor, np.multiply(input_shape, output_shape))
else:
tensor = np.reshape(tensor, [shape[0] * shape[1]] + input_shape + output_shape)
tensor = np.transpose(tensor, axes = [0] + [val for pair in zip(range(1, 1+order), range(1+order, 1+2*order)) for val in pair])
tensor = np.reshape(tensor, [shape[0] * shape[1]] + list(np.multiply(input_shape, output_shape)))
dense_factors = parafac(tensor, rank)
if shape[0] == 1:
conv_factor = []
else:
dense_factors, conv_factor = dense_factors[1:], np.reshape(dense_factors[0], [shape[0], shape[1], rank])
for l in range(order):
dense_factors[l] = np.reshape(np.transpose(dense_factors[l]), [rank, input_shape[l], output_shape[l]])
return dense_factors, conv_factor
def tensorDecom(lst):
lst2 = []
for i in lst:
factors, weights, errors = decomposition.parafac(i, 1)
factors = factors.reshape(factors.shape[0])
lst2.append(factors)
return np.asarray(lst2)
def test_randomized_svd():
""" Imports the tensor of union of all genes among 6 cell lines and performs parafac. """
tensor, _, _ = form_tensor()
tInit = initialize_cp(tensor, 7)
tfac = parafac(tensor, rank=7, init=tInit, linesearch=True, n_iter_max=2)
r2x = 1 - tl.norm(
(tl.cp_to_tensor(tfac) - tensor))**2 / (tl.norm(tensor))**2
assert r2x > 0
def cp_decomposition_conv_layer(layer, rank):
""" Gets a conv layer and a target rank,
returns a nn.Sequential object with the decomposition """
# Perform CP decomposition on the layer weight tensorly.
_, (last, first, vertical, horizontal) = parafac(layer.weight.data,
rank=rank,
init='svd')
pointwise_s_to_r_layer = torch.nn.Conv2d(in_channels=first.shape[0],
out_channels=first.shape[1],
kernel_size=1,
stride=1,
padding=0,
dilation=layer.dilation,
bias=False)
depthwise_vertical_layer = torch.nn.Conv2d(in_channels=vertical.shape[1],
out_channels=vertical.shape[1],
kernel_size=(vertical.shape[0],
1),
stride=1,
padding=(layer.padding[0], 0),
dilation=layer.dilation,
groups=vertical.shape[1],
bias=False)
depthwise_horizontal_layer = \
torch.nn.Conv2d(in_channels=horizontal.shape[1],
out_channels=horizontal.shape[1],
kernel_size=(1, horizontal.shape[0]), stride=layer.stride,
padding=(0, layer.padding[0]),
dilation=layer.dilation, groups=horizontal.shape[1], bias=False)
pointwise_r_to_t_layer = torch.nn.Conv2d(in_channels=last.shape[1],
out_channels=last.shape[0],
kernel_size=1,
stride=1,
padding=0,
dilation=layer.dilation,
bias=True)
pointwise_r_to_t_layer.bias.data = layer.bias.data
depthwise_horizontal_layer.weight.data = \
torch.transpose(horizontal, 1, 0).unsqueeze(1).unsqueeze(1)
depthwise_vertical_layer.weight.data = \
torch.transpose(vertical, 1, 0).unsqueeze(1).unsqueeze(-1)
pointwise_s_to_r_layer.weight.data = \
torch.transpose(first, 1, 0).unsqueeze(-1).unsqueeze(-1)
pointwise_r_to_t_layer.weight.data = last.unsqueeze(-1).unsqueeze(-1)
new_layers = [
pointwise_s_to_r_layer, depthwise_vertical_layer,
depthwise_horizontal_layer, pointwise_r_to_t_layer
]
return nn.Sequential(*new_layers)
def cp_decompose_conv(layer, rank=None, criterion=cp_rank):
""" Gets a conv layer and a target rank,
returns a nn.Sequential object with the decomposition """
if rank is None or rank == -1:
rank = criterion(layer)
'''
# Sanity Check
if (rank**2 >= conv_layer.in_channels * conv_layer.out_channels):
print("(rank**2 >= conv_layer.in_channels * conv_layer.out_channels")
continue
'''
# Perform CP decomposition on the layer weight tensorly.
last, first, vertical, horizontal = \
parafac(layer.weight.data, rank=rank, init='svd')[1]
pointwise_s_to_r_layer = torch.nn.Conv2d(in_channels=first.shape[0], \
out_channels=first.shape[1], kernel_size=1, stride=1, padding=0,
dilation=layer.dilation, bias=False)
depthwise_vertical_layer = torch.nn.Conv2d(in_channels=vertical.shape[1],
out_channels=vertical.shape[1],
kernel_size=(vertical.shape[0],
1),
stride=1,
padding=(layer.padding[0], 0),
dilation=layer.dilation,
groups=vertical.shape[1],
bias=False)
depthwise_horizontal_layer = \
torch.nn.Conv2d(in_channels=horizontal.shape[1], \
out_channels=horizontal.shape[1],
kernel_size=(1, horizontal.shape[0]), stride=layer.stride,
padding=(0, layer.padding[0]),
dilation=layer.dilation, groups=horizontal.shape[1], bias=False)
pointwise_r_to_t_layer = torch.nn.Conv2d(in_channels=last.shape[1], \
out_channels=last.shape[0], kernel_size=1, stride=1,
padding=0, dilation=layer.dilation, bias=True)
if layer.bias is not None:
pointwise_r_to_t_layer.bias.data = layer.bias.data
depthwise_horizontal_layer.weight.data = \
torch.transpose(horizontal, 1, 0).unsqueeze(1).unsqueeze(1)
depthwise_vertical_layer.weight.data = \
torch.transpose(vertical, 1, 0).unsqueeze(1).unsqueeze(-1)
pointwise_s_to_r_layer.weight.data = \
torch.transpose(first, 1, 0).unsqueeze(-1).unsqueeze(-1)
pointwise_r_to_t_layer.weight.data = last.unsqueeze(-1).unsqueeze(-1)
new_layers = [pointwise_s_to_r_layer, depthwise_vertical_layer, \
depthwise_horizontal_layer, pointwise_r_to_t_layer]
return nn.Sequential(*new_layers)
def cp_decomposition(layer, rank, device):
if isinstance(rank, int):
ranks = rank
elif rank == 'VBMF':
ranks = estimate_ranks(layer)
last, first, vertical, horizontal = parafac(
layer.weight.data.cpu().numpy(), rank=ranks, init='random')
s_to_r_layer = nn.Conv2d(in_channels=first.shape[0],
out_channels=first.shape[1],
kernel_size=1,
padding=0,
bias=False)
r_to_r_layer = nn.Conv2d(in_channels=ranks,
out_channels=ranks,
kernel_size=vertical.shape[0],
stride=layer.stride,
padding=layer.padding,
dilation=layer.dilation,
groups=ranks,
bias=False)
if layer.bias is not None:
r_to_t_layer = torch.nn.Conv2d(in_channels=last.shape[1],
out_channels=last.shape[0],
kernel_size=1,
stride=1,
padding=0,
dilation=layer.dilation,
bias=True)
r_to_t_layer.bias.data = layer.bias.data
else:
r_to_t_layer = torch.nn.Conv2d(in_channels=last.shape[1],
out_channels=last.shape[0],
kernel_size=1,
stride=1,
padding=0,
dilation=layer.dilation,
bias=False)
sr = first.t_().unsqueeze_(-1).unsqueeze_(-1)
rt = last.unsqueeze_(-1).unsqueeze_(-1)
rr = torch.stack([
vertical.narrow(1, i, 1) @ torch.t(horizontal).narrow(0, i, 1)
for i in range(rank)
]).unsqueeze_(1)
s_to_r_layer.weight.data = sr
r_to_t_layer.weight.data = rt
r_to_r_layer.weight.data = rr
new_layers = [s_to_r_layer, r_to_r_layer, r_to_t_layer]
return new_layers
def cp_decomposition_conv_layer(layer, rank):
l, f, v, h = parafac(layer.weight.data, rank=rank)[1]
factors = [l, f, v, h]
#print([f.shape for f in factors])
pointwise_s_to_r_layer = torch.nn.Conv2d(in_channels=f.shape[0],
out_channels=f.shape[1],
kernel_size=1,
stride=1,
padding=0,
dilation=layer.dilation,
bias=False)
depthwise_vertical_layer = torch.nn.Conv2d(in_channels=v.shape[1],
out_channels=v.shape[1],
kernel_size=(v.shape[0], 1),
stride=1,
padding=(layer.padding[0], 0),
dilation=layer.dilation,
groups=v.shape[1],
bias=False)
depthwise_horizontal_layer = torch.nn.Conv2d(in_channels=h.shape[1],
out_channels=h.shape[1],
kernel_size=(1, h.shape[0]),
stride=layer.stride,
padding=(0, layer.padding[0]),
dilation=layer.dilation,
groups=h.shape[1],
bias=False)
pointwise_r_to_t_layer = torch.nn.Conv2d(in_channels=l.shape[1],
out_channels=l.shape[0],
kernel_size=1,
stride=1,
padding=0,
dilation=layer.dilation,
bias=True)
pointwise_r_to_t_layer.bias.data = layer.bias.data
depthwise_horizontal_layer.weight.data = torch.transpose(
h, 1, 0).unsqueeze(1).unsqueeze(1)
depthwise_vertical_layer.weight.data = torch.transpose(
v, 1, 0).unsqueeze(1).unsqueeze(-1)
pointwise_s_to_r_layer.weight.data = torch.transpose(
f, 1, 0).unsqueeze(-1).unsqueeze(-1)
pointwise_r_to_t_layer.weight.data = l.unsqueeze(-1).unsqueeze(-1)
new_layers = [
pointwise_s_to_r_layer, depthwise_vertical_layer,
depthwise_horizontal_layer, pointwise_r_to_t_layer
]
#for l in new_layers:
# print(l.weight.data.shape)
return nn.Sequential(*new_layers)
def learn_embedding(self,
graph=None,
edge_f=None,
is_weighted=False,
no_python=False):
if not graph and not edge_f:
raise Exception('graph/edge_f needed')
if not graph:
graph = graph_util.loadGraphFromEdgeListTxt(edge_f)
t1 = time()
nNodes = graph.number_of_nodes()
nEdges = graph.number_of_edges()
print('num nodes: ', nNodes)
print('num edges: ', nEdges)
S = nx.to_numpy_matrix(graph, nodelist=sorted(graph.nodes()))
A = Normalizer(norm='l1').fit_transform(S)
if self._d == None:
self._d = 2 * self._K
else:
assert self._d == 2 * self._K
# Tensorization
md_array = np.zeros((nNodes, nNodes, self._K))
int_res = A
for i in range(self._K):
md_array[:, :, i] = int_res
int_res = int_res.dot(A)
emb = np.zeros((nNodes, self._d))
for i in range(self._K):
print('Slab id: ', i)
slab = np.reshape(md_array[:, :, i], (nNodes, nNodes, 1))
XX = tl.tensor(slab)
print('Tensor shape: ', XX.shape)
factors = parafac(XX,
rank=self._R,
n_iter_max=self._n_iter,
init='random') # random_state=123,
source_emb = factors[0]
target_emb = factors[1]
proximity_emb = factors[2]
print('Source emb shape: ', source_emb.shape)
print('Target emb shape: ', target_emb.shape)
print('Proximity emb shape: ', proximity_emb.shape)
source_proximity_emb = np.dot(source_emb, proximity_emb.T)
target_proximity_emb = np.dot(target_emb, proximity_emb.T)
emb[:, [i, i + self._K]] = np.concatenate(
(source_proximity_emb, target_proximity_emb), axis=1)
self._X = emb
print("Embedding shape: ", self._X.shape)
t2 = time()
return self._X, (t2 - t1)
def rank_search_parafac(tensor, rank_range):
AIC = []
for rank in range(1, rank_range + 1):
factors = parafac(tensor, rank=rank)
recon = tl.kruskal_to_tensor(factors)
err = tensor - recon
rank_AIC = 2 * tl.tenalg.inner(err, err) + 2 * rank
AIC.append(rank_AIC)
return AIC
def from_tensor(cls, tensor, tensorized_row_shape, tensorized_column_shape, rank, n_matrices=(), init='random', **kwargs):
n_matrices = _ensure_tuple(n_matrices)
rank = tl.cp_tensor.validate_cp_rank(n_matrices + tensorized_row_shape + tensorized_column_shape, rank)
with torch.no_grad():
weights, factors = parafac(tensor, rank, **kwargs)
weights = nn.Parameter(weights)
factors = [nn.Parameter(f) for f in factors]
return cls(weights, factors, tensorized_row_shape, tensorized_column_shape, rank, n_matrices)
def init_from_tensor(self, tensor, l2_reg=1e-5, **kwargs):
with torch.no_grad():
weights, factors = parafac(tensor,
self.rank,
l2_reg=l2_reg,
**kwargs)
self.weights = nn.Parameter(weights)
self.factors = FactorList([nn.Parameter(f) for f in factors])
return self
def _get_gaussians_from_kernel(self):
try:
x, y, z = parafac(self.psf, 1)
except ValueError:
__, (x, y, z) = parafac(self.psf, 1)
gauss_params = np.abs(self._fit_gaussian(np.array(np.squeeze(x))))
gauss_params[1] = gauss_params[2] * 12
x = self._get_gaussian(gauss_params, self.scaling[0])
gauss_params = np.abs(self._fit_gaussian(np.array(np.squeeze(y))))
gauss_params[1] = gauss_params[2] * 12
y = self._get_gaussian(gauss_params, self.scaling[1])
gauss_params = np.abs(self._fit_gaussian(np.array(np.squeeze(z))))
gauss_params[1] = gauss_params[2] * 12
z = self._get_gaussian(gauss_params, self.scaling[2])
return x, y, z
def forward(self, x):
y=None
b, c, _, _ = x.size()
for i in range(0, x.size()[0]):
x_item = x[i, :, :, :]
factors = parafac(x_item, rank=self.rank)
y = torch.cat((y,factors[0].unsqueeze(0)),dim=0) if not y is None else factors[0].unsqueeze(0)
y = self.rank_fc(y).view(b, c)
y = self.fc(y).view(b, c, 1, 1)
return x * y
def from_tensor(cls, tensor, rank='same', **kwargs):
shape = tensor.shape
rank = tl.cp_tensor.validate_cp_rank(shape, rank)
dtype = tensor.dtype
with torch.no_grad():
weights, factors = parafac(tensor.to(torch.float64), rank,
**kwargs)
return cls(nn.Parameter(weights.to(dtype)),
[nn.Parameter(f.to(dtype)) for f in factors])
def _cp_decomposition(self, layer, rank, offline=False, filename=''):
""" Gets a conv layer and a target rank, di
returns a nn.Sequential object with the decomposition
Args:
layer: the conv layer to decompose
rank: the rank of the CP-decomposition
offline: bool, if true the weights will be loaded from the file
specified in file name
filename: string, file from which we have to load the weights.
Returns:
The compressed 4 layers that substitutes the original one.
"""
# Perform CP decomposition on the layer weight tensor.
print('[Decomposer]: computing CP-decomposition of the layer {} with rank {}'.format(layer, rank))
X = layer.weight.data.numpy()
size = max(X.shape)
# THIS SHOULD BE ENHANCED BY USING A SUBPROCESS CALL
# WHICH CALLS THE MATLAB SCRIPT AND RETRIEVE THE RESULT
if offline:
last, first, vertical, horizontal = load_cpd_weights(filename)
else:
# SVD init leads to generally better overall compression.
# However, it can stall for large matrices.
if size >= 256:
print("Init random")
last, first, vertical, horizontal = parafac(
X, rank=rank, init='random')
else:
last, first, vertical, horizontal = parafac(X, rank=rank, init='svd')
first_pointwise = torch.nn.Conv2d(in_channels=first.shape[0],
out_channels=first.shape[1],
kernel_size=1,
stride=layer.stride,
padding=0,
dilation=layer.dilation,
bias=False)
separable_vertical = torch.nn.Conv2d(in_channels=vertical.shape[1],
out_channels=vertical.shape[1],
kernel_size=(
vertical.shape[0], 1),
stride=layer.stride,
padding=(layer.padding[0], 0),
dilation=layer.dilation,
groups=vertical.shape[1],
bias=False)
separable_horizontal = torch.nn.Conv2d(in_channels=horizontal.shape[1],
out_channels=horizontal.shape[1],
kernel_size=(
1, horizontal.shape[0]),
stride=layer.stride,
padding=(0, layer.padding[0]),
dilation=layer.dilation,
groups=horizontal.shape[1],
bias=False)
last_pointwise = torch.nn.Conv2d(in_channels=last.shape[1],
out_channels=last.shape[0],
kernel_size=1,
stride=layer.stride,
padding=0,
dilation=layer.dilation,
bias=True)
last_pointwise.bias.data = layer.bias.data
# Transpose dimensions back to what PyTorch expects
separable_vertical_weights = np.expand_dims(np.expand_dims(
vertical.transpose(1, 0), axis=1), axis=-1)
separable_horizontal_weights = np.expand_dims(np.expand_dims(
horizontal.transpose(1, 0), axis=1), axis=1)
first_pointwise_weights = np.expand_dims(
np.expand_dims(first.transpose(1, 0), axis=-1), axis=-1)
last_pointwise_weights = np.expand_dims(np.expand_dims(
last, axis=-1), axis=-1)
set_layer_weights(separable_horizontal,
separable_horizontal_weights)
set_layer_weights(separable_vertical,
separable_vertical_weights)
set_layer_weights(first_pointwise,
first_pointwise_weights)
set_layer_weights(last_pointwise,
last_pointwise_weights)
return [first_pointwise, separable_vertical, separable_horizontal, last_pointwise]
def cp_decomposition_conv_layer_BN(layer, rank, matlab=False):
""" Gets a conv layer and a target rank,
returns a nn.Sequential object with the decomposition """
# Perform CP decomposition on the layer weight tensor.
print(layer, rank)
X = layer.weight.data.numpy()
size = max(X.shape)
if matlab:
last, first, vertical, horizontal = load_cpd_weights(
'dumps/TODO.mat')
else:
# using a random initializaer is better for very large matrices
# SVD is a bit quicker on smaller ones
if size >= 256:
print("Init random")
last, first, vertical, horizontal = parafac(
X, rank=rank, init='random')
else:
last, first, vertical, horizontal = parafac(
X, rank=rank, init='svd')
pointwise_s_to_r_layer = torch.nn.Conv2d(in_channels=first.shape[0],
out_channels=first.shape[1],
kernel_size=1,
stride=layer.stride,
padding=0,
dilation=layer.dilation,
bias=False)
depthwise_vertical_layer = torch.nn.Conv2d(in_channels=vertical.shape[1],
out_channels=vertical.shape[1],
kernel_size=(
vertical.shape[0], 1),
stride=layer.stride,
padding=(layer.padding[0], 0),
dilation=layer.dilation,
groups=vertical.shape[1],
bias=False)
depthwise_horizontal_layer = torch.nn.Conv2d(in_channels=horizontal.shape[1],
out_channels=horizontal.shape[1],
kernel_size=(
1, horizontal.shape[0]),
stride=layer.stride,
padding=(0, layer.padding[0]),
dilation=layer.dilation,
groups=horizontal.shape[1],
bias=False)
add_bias = True and layer.bias is not None or False and not layer.bias
pointwise_r_to_t_layer = torch.nn.Conv2d(in_channels=last.shape[1],
out_channels=last.shape[0],
kernel_size=1,
stride=layer.stride,
padding=0,
dilation=layer.dilation,
bias=add_bias)
if add_bias:
pointwise_r_to_t_layer.bias.data = layer.bias.data
# Transpose dimensions back to what PyTorch expects
depthwise_vertical_layer_weights = np.expand_dims(np.expand_dims(
vertical.transpose(1, 0), axis=1), axis=-1)
depthwise_horizontal_layer_weights = np.expand_dims(np.expand_dims(
horizontal.transpose(1, 0), axis=1), axis=1)
pointwise_s_to_r_layer_weights = np.expand_dims(
np.expand_dims(first.transpose(1, 0), axis=-1), axis=-1)
pointwise_r_to_t_layer_weights = np.expand_dims(np.expand_dims(
last, axis=-1), axis=-1)
# Fill in the weights of the new layers
depthwise_horizontal_layer.weight.data = \
torch.from_numpy(np.float32(depthwise_horizontal_layer_weights))
depthwise_vertical_layer.weight.data = \
torch.from_numpy(np.float32(depthwise_vertical_layer_weights))
pointwise_s_to_r_layer.weight.data = \
torch.from_numpy(np.float32(pointwise_s_to_r_layer_weights))
pointwise_r_to_t_layer.weight.data = \
torch.from_numpy(np.float32(pointwise_r_to_t_layer_weights))
# create BatchNorm layers wrt to decomposed layers weights
bn_first = nn.BatchNorm2d(first.shape[1])
bn_vertical = nn.BatchNorm2d(vertical.shape[1])
bn_horizontal = nn.BatchNorm2d(horizontal.shape[1])
bn_last = nn.BatchNorm2d(last.shape[0])
new_layers = [pointwise_s_to_r_layer, bn_first, depthwise_vertical_layer, bn_vertical,
depthwise_horizontal_layer, bn_horizontal, pointwise_r_to_t_layer,
bn_last]
return nn.Sequential(*new_layers)
def cp_decomposition_conv_layer(layer, rank):
""" Gets a conv layer and a target rank,
returns a nn.Sequential object with the decomposition """
# Perform CP decomposition on the layer weight tensor.
print(layer, rank)
X = layer.weight.data.numpy()
size = max(X.shape)
# Using the SVD init gives better results, but stalls for large matrices.
if size >= 256:
print("Init random")
last, first, vertical, horizontal = parafac(X, rank=rank, init = 'random')
else:
last, first, vertical, horizontal = parafac(X, rank=rank, init = 'svd')
pointwise_s_to_r_layer = torch.nn.Conv2d(in_channels = first.shape[0], \
out_channels = first.shape[1],
kernel_size = 1, \
stride = layer.stride,
padding = 0,
dilation = layer.dilation,
bias = False)
depthwise_vertical_layer = torch.nn.Conv2d(in_channels = vertical.shape[1], \
out_channels = vertical.shape[1],
kernel_size = (vertical.shape[0], 1),
stride = layer.stride,
padding = (layer.padding[0], 0),
dilation = layer.dilation,
groups = vertical.shape[1],
bias = False)
depthwise_horizontal_layer = torch.nn.Conv2d(in_channels = horizontal.shape[1], \
out_channels = horizontal.shape[1],
kernel_size = (1, horizontal.shape[0]),
stride = layer.stride,
padding = (0, layer.padding[0]),
dilation = layer.dilation,
groups = horizontal.shape[1],
bias = False)
pointwise_r_to_t_layer = torch.nn.Conv2d(in_channels = last.shape[1], \
out_channels = last.shape[0],
kernel_size = 1, \
stride = layer.stride,
padding = 0,
dilation = layer.dilation,
bias = True)
pointwise_r_to_t_layer.bias.data = layer.bias.data
# Transpose dimensions back to what PyTorch expects
depthwise_vertical_layer_weights = np.expand_dims(np.expand_dims(\
vertical.transpose(1, 0), axis = 1), axis = -1)
depthwise_horizontal_layer_weights = np.expand_dims(np.expand_dims(\
horizontal.transpose(1, 0), axis = 1), axis = 1)
pointwise_s_to_r_layer_weights = np.expand_dims(\
np.expand_dims(first.transpose(1, 0), axis = -1), axis = -1)
pointwise_r_to_t_layer_weights = np.expand_dims(np.expand_dims(\
last, axis = -1), axis = -1)
# Fill in the weights of the new layers
depthwise_horizontal_layer.weight.data = \
torch.from_numpy(np.float32(depthwise_horizontal_layer_weights))
depthwise_vertical_layer.weight.data = \
torch.from_numpy(np.float32(depthwise_vertical_layer_weights))
pointwise_s_to_r_layer.weight.data = \
torch.from_numpy(np.float32(pointwise_s_to_r_layer_weights))
pointwise_r_to_t_layer.weight.data = \
torch.from_numpy(np.float32(pointwise_r_to_t_layer_weights))
new_layers = [pointwise_s_to_r_layer, depthwise_vertical_layer, \
depthwise_horizontal_layer, pointwise_r_to_t_layer]
return nn.Sequential(*new_layers)