def test_tensor_product():
"""Test tensor_dot"""
rng = random.check_random_state(1234)
X = tl.tensor(rng.random_sample((4, 5, 6)))
Y = tl.tensor(rng.random_sample((3, 4, 7)))
tdot = tl.tensor_to_vec(tensor_dot(X, Y))
true_dot = tl.tensor_to_vec(
tenalg.outer([tl.tensor_to_vec(X),
tl.tensor_to_vec(Y)]))
testing.assert_array_almost_equal(tdot, true_dot)
def _e_step(Xt, Wt, T, L, N, U, B, z0, psi0, sgm0, sgmO, sgmR, sgmV):
# matricize U and B
matU = kronecker(U, reverse=True)
matB = kronecker(B, reverse=True)
# workspace
Lp = np.prod(L)
Np = np.prod(N)
P = np.zeros((T, Lp, Lp))
J = np.zeros((T, Lp, Lp))
mu_ = np.zeros((T, Lp))
psi = np.zeros((T, Lp, Lp))
mu_h = np.zeros((T, Lp))
psih = np.zeros((T, Lp, Lp))
# forward algorithm
for t in trange(T, desc='forward'):
ot = tensor_to_vec(
Wt[t]) # indices of the observed entries of a tensor X
xt = tensor_to_vec(Xt[t])[ot]
lt = sum(ot) # of observed samples
Ht = matU[ot, :]
if t == 0:
K = sgm0 * Ht.T @ pinv(sgm0 * Ht @ Ht.T + sgmR * np.eye(lt))
# K = psi0 @ Ht.T @ pinv(Ht @ psi0 @ Ht.T + sgmR * np.eye(lt))
# psi[0] = sgm0 * np.eye(Lp) - K[0] @ Ht
psi[0] = sgm0 * (np.eye(Lp) - K[0] @ Ht)
# psi[0] = (np.eye(Lp) - K @ Ht) @ psi0
mu_[0] = z0 + K @ (xt - Ht @ z0)
else:
P[t - 1] = matB @ psi[t - 1] @ matB.T + sgmO * np.eye(Lp)
K = P[t - 1] @ Ht.T @ pinv(Ht @ P[t - 1] @ Ht.T +
sgmR * np.eye(lt))
mu_[t] = matB @ mu_[t - 1] + K @ (xt - Ht @ matB @ mu_[t - 1])
psi[t] = (np.eye(Lp) - K @ Ht) @ P[t - 1]
# backward
mu_h[-1] = mu_[-1]
psih[-1] = psi[-1]
for t in tqdm(list(reversed(range(T - 1))), desc='backward'):
J[t] = psi[t] @ matB.T @ pinv(P[t])
mu_h[t] = mu_[t] + J[t] @ (mu_h[t + 1] - matB @ mu_[t])
psih[t] = psi[t] + J[t] @ (psih[t + 1] - P[t]) @ J[t].T
# compute expectations
ztt = np.zeros((T, Lp, Lp))
zt_ = np.zeros((T, Lp, Lp))
cov_zt_ = np.zeros((T, Lp, Lp))
for t in trange(T, desc='compute expectations'):
if t > 0:
cov_zt_[t] = psih[t] @ J[t - 1].T
zt_[t] = cov_zt_[t] + np.outer(mu_h[t], mu_h[t - 1])
ztt[t] = psih[t] + np.outer(mu_h[t], mu_h[t])
zt = mu_h
cov_ztt = psih
return zt, cov_ztt, cov_zt_, ztt, zt_
def set_tensor_vectorization(x):
vectorization = []
for sample in x:
vectorization.append(tl.tensor_to_vec(sample))
return np.asarray(vectorization)
def test_active_set_nnls():
"""Test for active_set_nnls operator"""
a = T.tensor(np.random.rand(20, 10))
true_res = T.tensor(np.random.rand(10, 1))
b = T.dot(a, true_res)
atb = T.dot(T.transpose(a), b)
ata = T.dot(T.transpose(a), a)
x_as = active_set_nnls(tensor_to_vec(atb), ata)
x_as = T.reshape(x_as, T.shape(atb))
assert_array_almost_equal(true_res, x_as, decimal=2)
def test_batched_tensor_product():
"""Test batched-tensor_dot
Notes
-----
At the time of writing, MXNet doesn't support transpose
for tensors of order higher than 6
"""
rng = random.check_random_state(1234)
batch_size = 3
X = tl.tensor(rng.random_sample((batch_size, 4, 5, 6)))
Y = tl.tensor(rng.random_sample((batch_size, 3, 7)))
tdot = tl.unfold(batched_tensor_dot(X, Y), 0)
for i in range(batch_size):
true_dot = tl.tensor_to_vec(
tenalg.outer([tl.tensor_to_vec(X[i]),
tl.tensor_to_vec(Y[i])]))
testing.assert_array_almost_equal(tdot[i], true_dot)
def factors2vec(factors):
"""Wrapper function detailed in Appendix C [1]
Stacks the column vectors of a set of matrices into a single vecto
Parameters
---------
factors : list of ndarrays
Factor matrices or Gradient wrt factor gradient
Returns
-------
vec : ndarry
column-wise vectorization of a list of matrices
"""
vec = None
for factor in factors:
if vec is None:
vec = tl.tensor_to_vec(tl.transpose(factor))
else:
vec = tl.concatenate([vec, tl.tensor_to_vec(tl.transpose(factor))])
return vec
def tr_to_vec(factors):
"""Returns the tensor defined by its TR format ('factors') into
its vectorized format
Parameters
----------
factors: list of 3D-arrays
TR factors
Returns
-------
1-D array
vectorized format of tensor defined by 'factors'
"""
return tl.tensor_to_vec(tr_to_tensor(factors))
def tt_matrix_to_vec(tt_matrix):
"""Returns the tensor defined by its TT-Matrix format ('factors') into
its vectorized format
Parameters
----------
factors : list of 3D-arrays
TT factors
Returns
-------
1-D array
format of tensor defined by 'factors'
"""
return tl.tensor_to_vec(tt_matrix_to_tensor(tt_matrix))
def plot_coef(coef_img, img_name, thre_rate=0.01):
coef = coef_img.get_data()
coef_vec = tl.tensor_to_vec(coef)
# selection = SelectPercentile(f_classif, percentile=thre_rate)
n_voxel_th = int(coef_vec.shape[0] * thre_rate)
top_voxel_idx = (abs(coef_vec)).argsort()[::-1][:n_voxel_th]
thre = coef_vec[top_voxel_idx[-1]]
# coef_to_plot = np.zeros(coef.shape[0])
# coef_to_plot[top_voxel_idx] = coef[top_voxel_idx]
# thre = np.amax(abs(coef)) * thre_rate # high absulte value times threshold rate
# coef_img = nib.Nifti1Image(coef, maskimg.affine)
# plotting.plot_stat_map(coef_img, threshold=thre, output_file='%s.png'%img_name, cut_coords=(0, 15, 55))
plotting.plot_stat_map(coef_img,
threshold=thre,
output_file='%s_.pdf' % img_name,
display_mode='x',
vmax=0.0004,
cut_coords=range(0, 1, 1),
colorbar=False)
def hard_thresholding(tensor, number_of_non_zero):
"""
Proximal operator of the l0 ``norm''
Keeps greater "number_of_non_zero" elements untouched and sets other elements to zero.
Parameters
----------
tensor : ndarray
number_of_non_zero : int
Returns
-------
ndarray
Thresholded tensor on which the operator has been applied
"""
tensor_vec = tl.copy(tl.tensor_to_vec(tensor))
sorted_indices = tl.argsort(tl.argsort(tl.abs(tensor_vec),
axis=0,
descending=True),
axis=0)
return tl.reshape(
tl.where(sorted_indices < number_of_non_zero, tensor_vec,
tl.tensor(0, **tl.context(tensor_vec))), tensor.shape)
def one_ntf_step(unfolded_tensors,
rank,
in_factors,
norm_tensor,
update_rule,
beta,
sparsity_coefficients,
fixed_modes,
normalize,
alpha=0.5,
delta=0.01):
"""
One pass of Hierarchical Alternating Least Squares update along all modes
Update the factors by solving a least squares problem per mode (in hals), as described in [1],
or using the Multiplicative Update for the entire factors [2].
Note that the unfolding order is the one described in [3], which is different from [1].
Parameters
----------
unfolded_tensors: list of array
The spectrogram tensor, unfolded according to all its modes.
in_factors: list of array
Current estimates for the PARAFAC decomposition of
tensor. The value of factor[update_mode]
will be updated using a least squares update.
The values in in_factors are not modified.
rank: int
Rank of the decomposition.
norm_tensor : float
The Frobenius norm of the input tensor
update_rule: string "hals" | "mu"
The chosen update rule.
HALS performs optimization with the euclidean norm,
MU performs the optimization using the $\beta$-divergence loss,
which generalizes the Euclidean norm, and the Kullback-Leibler and
Itakura-Saito divergences.
The chosen beta-divergence is specified with the parameter `beta`.
beta: float
The beta parameter for the beta-divergence.
2 - Euclidean norm
1 - Kullback-Leibler divergence
0 - Itakura-Saito divergence
sparsity_coefficients : List of floats
sparsity coefficients for every mode.
fixed_modes : List of integers
Indexes of modes that are not updated
normalize: List of boolean (as much as the number of modes)
A boolean where the factors need to be normalized.
The normalization is a l_2 normalization on each of the rank components
(columnwise)
alpha : positive float
Ratio between outer computations and inner loops. Typically set to
0.5 or 1.
Default: 0.5
delta : float in [0,1]
Early stop criterion, while err_k > delta*err_0. Set small for
almost exact nnls solution, or larger (e.g. 1e-2) for inner loops
of a PARAFAC computation.
Default: 0.01
Returns
-------
np.array(factors): numpy array
An array containing all the factors computed with PARAFAC decomposition
cost_fct_val:
The value of the cost function at this step,
normalized by the squared norm of the original tensor.
References
----------
[1] Tamara G Kolda and Brett W Bader. "Tensor decompositions and applications",
SIAM review 51.3 (2009), pp. 455{500.
[2] Févotte, C., & Idier, J. (2011).
Algorithms for nonnegative matrix factorization with the β-divergence.
Neural computation, 23(9), 2421-2456.
[3] Jeremy E Cohen. "About notations in multiway array processing",
arXiv preprint arXiv:1511.01306, (2015).
"""
if update_rule not in ["hals", "mu"]:
raise err.InvalidArgumentValue(
f"Invalid update rule: {update_rule}") from None
if update_rule == "hals" and beta != 2:
raise err.InvalidArgumentValue(
f"The hals is only valid for the frobenius norm, corresponding to the beta divergence with beta = 2. Here, beta was set to {beta}. To compute NMF with this value of beta, please use the mu update_rule."
) from None
# Avoiding errors
for fixed_value in fixed_modes:
sparsity_coefficients[fixed_value] = None
# Copy
factors = in_factors.copy()
# Generating the mode update sequence
gen = [
mode for mode in range(len(unfolded_tensors))
if mode not in fixed_modes
]
for mode in gen:
if update_rule == "hals":
tic = time.time()
# Computing Hadamard of cross-products
cross = tl.tensor(tl.ones((rank, rank))) #, **tl.context(tensor))
for i, factor in enumerate(factors):
if i != mode:
cross *= tl.dot(tl.transpose(factor), factor)
# Computing the Khatri Rao product
krao = tl.tenalg.khatri_rao(factors, skip_matrix=mode)
rhs = tl.dot(unfolded_tensors[mode], krao)
timer = time.time() - tic
# Call the hals resolution with nnls, optimizing the current mode
factors[mode] = tl.transpose(
nnls.hals_nnls_acc(
tl.transpose(rhs),
cross,
tl.transpose(factors[mode]),
maxiter=100,
atime=timer,
alpha=alpha,
delta=delta,
sparsity_coefficient=sparsity_coefficients[mode],
normalize=normalize[mode])[0])
elif update_rule == "mu":
krao = tl.tenalg.khatri_rao(factors, skip_matrix=mode)
factors[mode] = mu.mu_betadivmin(factors[mode], krao.T,
unfolded_tensors[mode], beta)
# Adding the l1 norm value to the reconstruction error
sparsity_error = 0
for index, sparse in enumerate(sparsity_coefficients):
if sparse:
sparsity_error += 2 * (sparse *
np.linalg.norm(factors[index], ord=1))
if update_rule == "hals":
# error computation (improved using precomputed quantities)
rec_error = norm_tensor**2 - 2 * tl.dot(
tl.tensor_to_vec(factors[mode]), tl.tensor_to_vec(rhs)) + tl.norm(
tl.dot(factors[mode], tl.transpose(krao)), 2)**2
elif update_rule == "mu":
rec_error = beta_div.beta_divergence(unfolded_tensors[mode],
factors[mode] @ krao.T, beta)
cost_fct_val = (rec_error + sparsity_error) / (norm_tensor**2)
return factors, cost_fct_val
def tl_gcp_fg(M, X, f, g, W=None, computeF=True, computeG=True, vectorG=True):
"""Loss function and gradient for generalized CP decomposition.
(Analogous to tt_gcp_fg.m)
Parameters
----------
M : CPTensor
X : ndarray
Dense tensor
f : Function handle
elementwise loss of the form f(x,m)
g : Function handle
Elementwise intermediate gradient of the form g(x,m)
W : ndarray
Weight tensor, 1's for known values, 0's for missing values. Function/gradient is only computed w.r.t
know values. Setting W =[] indicates no missing data.
computeF : boolean
Include computation of the loss function. Default is true.
computeG : boolean
Include computation of the gradient.
vectorG : boolean
Reshape gradient matrices into a single vector.
Returns
-------
F : scalar
Loss function value
G : ndarray(s)
If vectorG = False, G is a list of matrices where G[k] is the same size as the k-th factor matrix
Otherwise, G is the gradient in vector form
"""
# setup
Mfull = M.to_tensor()
Mv = tl.tensor_to_vec(Mfull)
Xv = tl.tensor_to_vec(X)
F = None
G = []
# calculate loss
if computeF:
Fvec = f(Xv, Mv)
if W is not None:
# TODO handle applying weight tensor, probably need to vec it then elementwise product
pass
F = Fvec.sum()
# calculate gradient
if computeG:
Y = g(Xv, Mv)
Y = tl.tensor(tl.reshape(Y, tl.shape(X)))
if W is not None:
# TODO handle applying weight tensor, probably need to vec it then elementwise product
pass
for i in range(len(M[1])):
mGrad = tl.unfolding_dot_khatri_rao(Y, M, i)
G.append(mGrad)
if vectorG:
G = factors2vec(G)
return F, G
def gcp(X, R, type='normal', func=None, grad=None, lower=None,\
opt='lbfgsb', mask=None, maxiters=1000, \
init='random', printitn=10, state=None, factr=1e7, pgtol=1e-4, \
fsamp=None, gsamp=None, oversample=1.1, sampler='uniform', \
fsampler=None, rate=1e-3, decay=0.1, maxfails=1, epciters=1000, \
festtol=-math.inf, beta1=0.9, beta2=0.999, epsilon=1e-8):
"""Generalized CANDECOMP/PARAFAC (GCP) decomposition via all-at-once optimization (OPT) [1]
Computes a rank-'R' decomposition of 'tensor' such that::
tensor = [|weights; factors[0], ..., factors[-1] |].
GCP-OPT allows the use of a variety of statistically motivated loss functions
suited to the data held in a tensor (i.e. continuous, discrete, binary, etc)
Parameters
----------
X : ndarray
Tensor to factorize
**COMING SOON**
Sparse tensor support
R : int
Rank of decomposition (Number of components).
type : str,
Type of objective function used
Options include:
'normal' or 'gaussian' - Gaussian for real-valued data (DEFAULT)
'binary' or 'bernoulli-odds' - Bernoulli w/ odds link for binary data
'bernoulli-logit' - Bernoulli w/ logit link for binary data
'count' or 'poisson' - Poisson for count data
'poisson-log' - Poisson w/ log link for count data
'rayleigh' - Rayleigh distribution for real-valued data
'gamma' - Gamma distribution for non-negative real-valued data
**COMING SOON**:
'huber (DELTA) - Similar to Gaussian, for real-valued data
'negative-binomial (r)' - Negative binomial for count data
'beta (BETA)' - Beta divergence for non-negative real-valued data
'user-specified' - Customized objective function provided by user
func: lambda function
User specified custom objective function, eg. lambda x, m: (m-x)**2
grad: lambda function
User specified custom gradient function, eg. lambda x, m: 2*(m-x)
lower: 0 or -inf
Lower bound for custom objective/gradient
opt : str
Optimization method
Options include:
'lbfgsb' - Bound-constrained limited-memory BFGS
'sgd' - Stochastic gradient descent (SGD)
**COMING SOON**
'adam' - Momentum-based SGD method
'adagrad' - Adaptive gradient algorithm, well suited for sparse data
If 'tensor' is dense, all 4 options can be used, 'lbfgsb' by default.
**COMING SOON** - Sparse format support
If 'tensor' is sparse, only 'sgd', 'adam' and 'adagrad' can be used, 'adam' by
default.
Each method has specific parameters, see documentation
mask : ndarray
Specifies a mask, 0's for missing/incomplete entries, 1's elsewhere, with
the same shape as 'tensor'.
**COMING SOON** - Missing/incomplete data simulation.
maxiters : int
Maximum number of outer iterations, 1000 by default.
init : {'random', 'svd', cptensor}
Initialization for factor matrices, 'random' by default.
Options include:
'random' - random initialization from a uniform distribution on [0,1)
'svd' - initialize the `m`th factor matrix using the `rank` left
singular vectors of the `m`th unfolding of the input tensor.
cptensor - initialization provided by user. NOTE: weights are pulled
in the last factor and then the weights are set to "1" for
the output tensor.
Initializations all result in a cptensor where the weights are one.
printitn : int
Print every n iterations; 0 for no printing, 10 by default.
state : {None, int, np.random.RandomState}
Seed for reproducable random number generation
factr : float
(L-BFGS-B parameter)
Tolerance on the change of objective values. Defaults to 1e7.
pgtol : float
(L-BFGS-B parameter)
Projected gradient tolerance. Defaults to 1e-5
sampler : {uniform, stratified, semi-stratified}
Type of sampling to use for stochastic gradient (SGD/ADAM/ADAGRAD).
Defaults to 'uniform' for dense tensors.
Defaults to 'stratified' for sparse tensors.
Options include:
'uniform' - Uniform random sampling
**COMING SOON**
'stratified' - Stratified sampling, targets sparse data. Zero and
nonzero values sampled separately.
'semi-stratified' - Similar to stratified sampling, but is more
computationally efficient (See papers referenced).
gsamp : int
Number of samples for stochastic gradient (SGD/ADAM/ADAGRAD parameter).
Generally set to be O(sum(shape)*R).
**COMING SOON**
For stratified or semi-stratified, this may be two numbers:
- the number of nnz samples
- the number of zero samples.
If only one number is specified, then this value is used for both nnzs and
zeros (total number of samples is 2x specified value in this case).
fsampler : {'uniform', 'stratified', custom}
Type of sampling for estimating objective function (SGD/ADAM/ADAGRAD parameter).
Options include:
'uniform' - Uniform random sampling
**COMING SOON**
'stratified' - Stratified sampling, targets sparse data. Zero and
nonzero values sampled separately.
custom - User-defined sampler (lambda function). Custom option
is primarily useful in reusing sampled elements across
multiple tests.
fsamp : int
(SGD/ADAM/ADAGRAD parameter)
Number of samples to estimate objective function.
This should generally be somewhat large since we want this sample to generate a
reliable estimate of the true function value.
oversample : float
(Stratified sampling parameter)
Factor to oversample when implicitly sampling zeros in the sparse case.
Defaults to 1.1. Only adjust for very small tensors.
rate : float
(SGD/ADAM parameter)
Initial learning rate. Defaults to 1e-3.
decay : float
(SGD/ADAM parameter)
Amount to decrease learning rate when progress stagnates, i.e. no change in
objective function between epochs. Defaults to 0.1.
maxfails : int
(SGD/ADAM parameter)
Number of times to decrease the learning rate.
Defaults to 1, may be set to zero.
epciters : int
(SGD/ADAM parameter)
Iterations per epoch. Defaults to 1000.
festtol : float
(SGD/ADAM parameter)
Quit estimation of function if it goes below this level.
Defaults to -inf.
beta1 : float
(ADAM parameter) - generally doesn't need to be changed
Defaults to 0.9
beta2 : float
(ADAM parameter) - generally doesn't need to be changed
Defaults to 0.999
epsilon : float
(ADAM parameter) - generally doesn't need to be changed
Defaults to 1e-8
Returns
-------
Mfin : CPTensor
Canonical polyadic decomposition of input tensor X
Reference
---------
[1] D. Hong, T. G. Kolda, J. A. Duersch, Generalized Canonical
Polyadic Tensor Decomposition, SIAM Review, 62:133-163, 2020,
https://doi.org/10.1137/18M1203626
[2] T. G. Kolda, D. Hong, Stochastic Gradients for Large-Scale Tensor
Decomposition. SIAM J. Mathematics of Data Science, 2:1066-1095,
2020, https://doi.org/10.1137/19m1266265
"""
# Timer - Setup (outside optimization)
start_setup0 = time.perf_counter()
# Initial setup
nd = tl.ndim(X)
sz = tl.shape(X)
tsz = X.size
X_context = tl.context(X)
vecsz = 0
for i in range(nd):
# tsz *= sz[i]
vecsz += sz[i]
vecsz *= R
W = mask
# Random set-up
if state is not None:
state = tl.check_random_state(state)
# capture stats(nnzs, zeros, missing)
nnonnzeros = 0
X = tl.tensor_to_vec(X)
for i in X:
if i > 0:
nnonnzeros += 1
X = tl.reshape(X, sz)
nzeros = tsz - nnonnzeros
nmissing = 0
if W is not None:
W = tl.tensor_to_vec(W)
for i in range(tl.shape(W)[0]):
if W[i] > 0: nmissing += 1 # TODO: is this right??
W = tl.reshape(W, sz)
# Dictionary for storing important information regarding the decomposition problem
info = {}
info['tsz'] = tsz
info[
'nmissing'] = 0 # TODO: revisit once missing value functionality incorporated
info['nnonnzeros'] = nnonnzeros
info[
'nzeros'] = nzeros # TODO: revisit once missing value functionality incorporated
# Set up function, gradient, and bounds
fh, gh, lb = validate_type(type, X)
info['type'] = type
info['fh'] = fh
info['gh'] = gh
info['lb'] = lb
# initialize CP-tensor and make a copy to work with so as to have the starting guess
M0 = initialize_cp(X, R, init=init, random_state=state)
wghts0 = tl.copy(M0[0])
fcts0 = []
for i in range(nd):
f = tl.copy(M0[1][i])
fcts0.append(f)
M = CPTensor((wghts0, fcts0))
# Lambda weights are assumed to be all ones throughout, check initial guess satisfies assumption
if not tl.all(M[0]):
print("Initialization of CP tensor has failed (lambda weight(s) != 1.")
sys.exit(1)
# check optimization method
if validate_opt(opt):
print("Choose optimization method from: {lbfgsb, sgd}")
sys.exit(1)
use_stoc = False
if opt != 'lbfgsb':
use_stoc = True
info['opt'] = opt
# set up for stochastic optimization (e.g. sgd, adam, adagrad)
if use_stoc:
# set up fsampler, gsampler ---> uniform sampling only for now
# TODO : add stratified, semi-stratified and user-specified sampling options
if not sampler == "uniform":
print(
"Only uniform sampling currently supported for stochastic optimization."
)
sys.exit(1)
fsampler_type = sampler
gsampler_type = sampler
# setup fsampler
f_samp = fsamp
if f_samp == None:
upper = np.maximum(math.ceil(tsz / 10), 10 ^ 6)
f_samp = np.minimum(upper, tsz)
# set up lambda function/function handle for uniform sampling
fsampler = lambda: tl_sample_uniform(X, f_samp)
fsampler_str = "{} with {} samples".format(fsampler_type, f_samp)
# setup gsampler
g_samp = gsamp
if g_samp == None:
upper = np.maximum(1000, math.ceil(10 * tsz / maxiters))
g_samp = np.minimum(upper, tsz)
# setup lambda function/function handle for uniform sampling
gsampler = lambda: tl_sample_uniform(X, g_samp)
gsampler_str = "{} with {} samples".format(gsampler_type, g_samp)
# capture the info
info['fsampler'] = fsampler_str
info['gsampler'] = gsampler_str
info['fsamp'] = f_samp
info['gsamp'] = g_samp
time_setup0 = time.perf_counter() - start_setup0
# Welcome message
if printitn > 0:
print("GCP-OPT-{} (Generalized CP Tensor Decomposition)".format(opt))
print("------------------------------------------------")
print("Tensor size:\t\t\t\t{} ({} total entries)".format(sz, tsz))
if nmissing > 0:
print("Missing entries: {} ({})".format(nmissing,
100 * nmissing / tsz))
print("Generalized function type:\t{}".format(type))
print("Objective function:\t\t\t{}".format(
inspect.getsource(fh).strip()))
print("Gradient function:\t\t\t{}".format(
inspect.getsource(gh).strip()))
print("Lower bound of factors:\t\t{}".format(lb))
print("Optimization method:\t\t{}".format(opt))
if use_stoc:
print("Max iterations (epochs): {}".format(maxiters))
print("Iterations per epoch: {}".format(epciters))
print("Learning rate / decay / maxfails: {} {} {}".format(
rate, decay, maxfails))
print("Function Sampler: {}".format(fsampler_str))
print("Gradient Sampler: {}".format(gsampler_str))
else:
print("Max iterations:\t\t\t\t{}".format(maxiters))
print("Projected gradient tol:\t\t{}\n".format(pgtol))
# Make like a zombie and start decomposing
Mfin = None
# L-BFGS-B optimization
if opt == 'lbfgsb':
# Timer - Setup (inside optimization)
start_setup1 = time.perf_counter()
# set up bounds for l-bfgs-b if lb = 0
bounds = None
if lb == 0:
lb = tl.zeros(tsz)
ub = math.inf * tl.ones(tsz)
fcn = lambda x: tl_gcp_fg(vec2factors(x, sz, R, X_context), X, fh, gh)
m = factors2vec(M[1])
# capture params for l-bfgs-b
lbfgsb_params = {}
lbfgsb_params['x0'] = factors2vec(M0.factors)
lbfgsb_params['printEvery'] = printitn
lbfgsb_params['maxIts'] = maxiters
lbfgsb_params['maxTotalIts'] = maxiters * 10
lbfgsb_params['factr'] = factr
lbfgsb_params['pgtol'] = pgtol
time_setup1 = time.perf_counter() - start_setup1
if printitn > 0:
print("Begin main loop")
# Timer - Main operation
start_main = time.perf_counter()
x, f, info_dict = fmin_l_bfgs_b(fcn, m, approx_grad=False, bounds=None, \
pgtol=pgtol, factr=factr, maxiter=maxiters)
time_main = time.perf_counter() - start_main
# capture info
info['fcn'] = fcn
info['lbfgsbopts'] = lbfgsb_params
info['lbfgsbout'] = info_dict
info['finalf'] = f
if printitn > 0:
print("\nFinal objective: {}".format(f))
print("Setup time: {}".format(time_setup0 + time_setup1))
print("Main loop time: {}".format(time_main))
print("Outer iterations:"
) # TODO: access this value (see manpage for fmin_l_bfgs_b)
print("Total iterations: {}".format(info_dict['nit']))
print("L-BFGS-B exit message: {} ({})".format(
info_dict['task'], info_dict['warnflag']))
Mfin = vec2factors(x, sz, R, X_context)
# Stochastic optimization
else:
# Timer - Setup (inside optimization)
start_setup1 = time.perf_counter()
if opt == "adam" or opt == "adagrad":
print("{} not currently supported".format(opt))
sys.exit(1)
# prepare for sgd
# initialize moments
m = []
v = []
# Extract samples for estimating function value (i.e. call fsampler), these never change
fsubs, fvals, fwgts = fsampler()
# Compute initial estimated function value
fest = tl_gcp_fg_est(M, fh, gh, fsubs, fvals, fwgts, True, False,
False, False)
# Set up loop variables
nfails = 0
titers = 0
M_weights = tl.copy(M[0])
M_factors = []
for k in range(nd):
M_factors.append(tl.copy(M[1][k]))
Msave = CPTensor(
(M_weights, M_factors)) # save a copy of the initial model
msave = m
vsave = v
fest_prev = fest[0]
# Tracing the progress in function value by epoch
fest_trace = tl.zeros(maxiters + 1)
step_trace = tl.zeros(maxiters + 1)
time_trace = tl.zeros(maxiters + 1)
fest_trace[0] = fest[0]
# Print status
if printitn > 0:
print("Begin main loop")
print("Initial f-est: {}".format(fest[0]))
time_setup1 = time.perf_counter() - start_setup1
start_main = time.perf_counter()
time_trace[0] = time.perf_counter() - start_setup0
# Main loop - outer iteration
for nepoch in range(maxiters):
step = (decay**nfails) * rate
# Main loop - inner iteration
for iter in range(epciters):
# Tracking iterations
titers = titers + 1
# Select subset for stochastic gradient (i.e. call gsampler)
gsubs, gvals, gwts = gsampler()
# Compute gradients for each mode
Gest = tl_gcp_fg_est(M, fh, gh, gsubs, gvals, gwts, False,
True, False, False)
# Check for inf gradient
for g in Gest[0]:
g_max = tl.max(g)
g_min = tl.min(g)
if math.isinf(g_max) or math.isinf(g_min):
print(
"Infinite gradient encountered! (epoch = {}, iter = {})"
.format(nepoch, iter))
# TODO : add functionality for ADAM and ADAGRAD optimization
# Take gradient step
for k in range(nd):
M.factors[k] = M.factors[k] - step * Gest[0][k]
# Estimate objective (i.e. call tl_gcp_fg_est)
fest = tl_gcp_fg_est(M, fh, gh, fsubs, fvals, fwgts, True, False,
False, False)
# Save trace (fest & step)
fest_trace[nepoch + 1] = fest[0]
step_trace[nepoch + 1] = step
# Check convergence condition
failed_epoch = False
if fest[0] > fest_prev:
failed_epoch = True
if failed_epoch:
nfails += 1
festtol_test = False
if fest[0] < festtol:
festtol_test = True
# Reporting
if printitn > 0 and (nepoch % printitn == 0 or failed_epoch
or festtol_test):
print("Epoch {}: f-est = {}, step = {}".format(
nepoch, fest[0], step),
end='')
if failed_epoch:
print(
", nfails = {} (resetting to solution from last epoch)"
.format(nfails))
print("")
# Rectify failed epoch or save current solution
if failed_epoch:
M = Msave
m = msave
v = vsave
fest[0] = fest_prev
titers = titers - epciters
else:
Msave = CPTensor((tl.copy(M.weights), tl.copy(M.factors)))
msave = m
vsave = v
fest_prev = fest[0]
time_trace[nepoch] = time.perf_counter() - start_setup0
if (nfails > maxfails) or festtol_test:
break
Mfin = M
time_main = time.perf_counter() - start_main
# capture info
info['fest_trace'] = fest_trace
info['step_trace'] = step_trace
info['time_trace'] = time_trace
info['nepoch'] = nepoch
# Report end of main loop
if printitn > 0:
print("End Main Loop")
print("")
print("Final f-east: {}".format(fest[0]))
print("Setup time: {0:0.6f}".format(time_setup0 + time_setup1))
print("Main loop time: {0:0.6f}".format(time_main))
print("Total iterations: {}".format(nepoch * epciters))
# Wrap up / capture remaining info
info['mainTime'] = time_main
info['setupTime0'] = time_setup0
info['setupTime1'] = time_setup1
info['setupTime'] = time_setup0 + time_setup1
return Mfin
