def DefineLoss(x, y, g_list, g_list_omega, params, trainable_var): # Minimize the mean squared errors. # subtraction and squaring element-wise, then average over both dimensions # n columns # average of each row (across columns), then average the rows den_nonzero = 10**(-5) # autoencoder loss if params['relative_loss']: loss1den = tf.reduce_mean(tf.reduce_mean(tf.square(tf.squeeze(x[0,:,:])),1))+den_nonzero else: loss1den = tf.to_double(1.0) loss1 = params['recon_lam']*tf.truediv(tf.reduce_mean(tf.reduce_mean(tf.square(y[0] - tf.squeeze(x[0,:,:])),1)), loss1den) # gets dynamics loss2 = tf.zeros([1,], dtype=tf.float64) if params['num_shifts'] > 0: for j in np.arange(params['num_shifts']): # xk+1, xk+2, xk+3 shift = params['shifts'][j] if params['relative_loss']: loss2den = tf.reduce_mean(tf.reduce_mean(tf.square(tf.squeeze(x[shift,:,:])),1))+den_nonzero else: loss2den = tf.to_double(1.0) loss2 = loss2 + params['recon_lam']*tf.truediv(tf.reduce_mean(tf.reduce_mean(tf.square(y[j+1] - tf.squeeze(x[shift,:,:])),1)), loss2den) loss2 = loss2/params['num_shifts'] # K linear loss3 = tf.zeros([1,], dtype=tf.float64) countSM = 0 if params['num_shifts_middle'] > 0: next_step = net.varying_multiply(g_list[0], g_list_omega[0], params['deltat']) for j in np.arange(max(params['shifts_middle'])): if ((j+1) in params['shifts_middle']): # muliply g_list[0] by L (j+1) times # next_step = tf.matmul(g_list[0], L_pow) if params['relative_loss']: loss3den = tf.reduce_mean(tf.reduce_mean(tf.square(tf.squeeze(g_list[countSM+1])),1))+den_nonzero else: loss3den = tf.to_double(1.0) loss3 = loss3 + params['mid_shift_lam']*tf.truediv(tf.reduce_mean(tf.reduce_mean(tf.square(next_step - g_list[countSM+1]),1)),loss3den) countSM +=1 # hopefully still on correct traj, so same omegas as before next_step = net.varying_multiply(next_step, g_list_omega[j+1], params['deltat']) loss3 = loss3/params['num_shifts_middle'] if params['reg_lam']: l1_regularizer = tf.contrib.layers.l1_regularizer(scale=params['reg_lam'],scope=None) # TODO: don't include biases? use weights dict instead? loss_L1 = tf.contrib.layers.apply_regularization(l1_regularizer, weights_list=trainable_var) else: loss_L1 = tf.zeros([1,], dtype=tf.float64) # tf.nn.l2_loss returns number l2_regularizer = tf.add_n([tf.nn.l2_loss(v) for v in trainable_var if 'b' not in v.name]) loss_L2 = params['l2_lam'] * l2_regularizer # inf norm on autoencoder error linf1_den = tf.norm(tf.norm(tf.squeeze(x[0,:,:]),axis=1,ord=np.inf),ord=np.inf)+den_nonzero linf2_den = tf.norm(tf.norm(tf.squeeze(x[1,:,:]),axis=1,ord=np.inf),ord=np.inf)+den_nonzero regularization_penalty_Linf_1 = tf.truediv(tf.norm(tf.norm(y[0] - tf.squeeze(x[0,:,:]),axis=1,ord=np.inf),ord=np.inf), linf1_den) regularization_penalty_Linf_2 = tf.truediv(tf.norm(tf.norm(y[1] - tf.squeeze(x[1,:,:]),axis=1,ord=np.inf),ord=np.inf), linf2_den) loss_Linf = params['reg_inf_lam'] * (regularization_penalty_Linf_1 + regularization_penalty_Linf_2) loss = loss1 + loss2 + loss3 + loss_Linf regularized_loss = loss + loss_L1 + loss_L2 return loss1, loss2, loss3, loss_Linf, loss_L1, loss_L2, loss, regularized_loss
def define_loss(x, y, g_list, weights, biases, params):
"""Define the (unregularized) loss functions for the training.
Arguments:
x -- placeholder for input
y -- list of outputs of network for each shift (each prediction step)
g_list -- list of output of encoder for each shift (encoding each step in x)
weights -- dictionary of weights for all networks
biases -- dictionary of biases for all networks
params -- dictionary of parameters for experiment
Returns:
loss1 -- autoencoder loss function
loss2 -- dynamics/prediction loss function
loss3 -- linearity loss function
loss_Linf -- inf norm on autoencoder loss and one-step prediction loss
loss -- sum of above four losses
Side effects:
None
"""
# Minimize the mean squared errors.
# subtraction and squaring element-wise, then average over both dimensions
# n columns
# average of each row (across columns), then average the rows
denominator_nonzero = 10**(-5)
# autoencoder loss
if params['relative_loss']:
loss1_denominator = tf.reduce_mean(
tf.reduce_mean(tf.square(tf.squeeze(x[0, :, :])),
1)) + denominator_nonzero
else:
loss1_denominator = tf.to_double(1.0)
mean_squared_error = tf.reduce_mean(
tf.reduce_mean(tf.square(y[0] - tf.squeeze(x[0, :, :])), 1))
loss1 = params['recon_lam'] * tf.truediv(mean_squared_error,
loss1_denominator)
# gets dynamics/prediction
loss2 = tf.zeros([
1,
], dtype=tf.float64)
if params['num_shifts'] > 0:
for j in np.arange(params['num_shifts']):
# xk+1, xk+2, xk+3
shift = params['shifts'][j]
if params['relative_loss']:
loss2_denominator = tf.reduce_mean(
tf.reduce_mean(tf.square(tf.squeeze(x[shift, :, :])),
1)) + denominator_nonzero
else:
loss2_denominator = tf.to_double(1.0)
loss2 = loss2 + params['recon_lam'] * tf.truediv(
tf.reduce_mean(
tf.reduce_mean(
tf.square(y[j + 1] - tf.squeeze(x[shift, :, :])), 1)),
loss2_denominator)
loss2 = loss2 / params['num_shifts']
# K linear
loss3 = tf.zeros([
1,
], dtype=tf.float64)
count_shifts_middle = 0
if params['num_shifts_middle'] > 0:
# generalization of: next_step = tf.matmul(g_list[0], L_pow)
omegas = net.omega_net_apply(params, g_list[0], weights, biases)
next_step = net.varying_multiply(g_list[0], omegas, params['delta_t'],
params['num_real'],
params['num_complex_pairs'])
# multiply g_list[0] by L (j+1) times
for j in np.arange(max(params['shifts_middle'])):
if (j + 1) in params['shifts_middle']:
if params['relative_loss']:
loss3_denominator = tf.reduce_mean(
tf.reduce_mean(
tf.square(
tf.squeeze(g_list[count_shifts_middle + 1])),
1)) + denominator_nonzero
else:
loss3_denominator = tf.to_double(1.0)
loss3 = loss3 + params['mid_shift_lam'] * tf.truediv(
tf.reduce_mean(
tf.reduce_mean(
tf.square(next_step -
g_list[count_shifts_middle + 1]), 1)),
loss3_denominator)
count_shifts_middle += 1
omegas = net.omega_net_apply(params, next_step, weights, biases)
next_step = net.varying_multiply(next_step, omegas,
params['delta_t'],
params['num_real'],
params['num_complex_pairs'])
loss3 = loss3 / params['num_shifts_middle']
# inf norm on autoencoder error and one prediction step
if params['relative_loss']:
Linf1_den = tf.norm(tf.norm(tf.squeeze(x[0, :, :]), axis=1,
ord=np.inf),
ord=np.inf) + denominator_nonzero
Linf2_den = tf.norm(tf.norm(tf.squeeze(x[1, :, :]), axis=1,
ord=np.inf),
ord=np.inf) + denominator_nonzero
else:
Linf1_den = tf.to_double(1.0)
Linf2_den = tf.to_double(1.0)
Linf1_penalty = tf.truediv(
tf.norm(tf.norm(y[0] - tf.squeeze(x[0, :, :]), axis=1, ord=np.inf),
ord=np.inf), Linf1_den)
Linf2_penalty = tf.truediv(
tf.norm(tf.norm(y[1] - tf.squeeze(x[1, :, :]), axis=1, ord=np.inf),
ord=np.inf), Linf2_den)
loss_Linf = params['Linf_lam'] * (Linf1_penalty + Linf2_penalty)
loss = loss1 + loss2 + loss3 + loss_Linf
return loss1, loss2, loss3, loss_Linf, loss
def define_loss(x, y, g_list, g_list_omega, params):
# Minimize the mean squared errors.
# subtraction and squaring element-wise, then average over both dimensions
# n columns
# average of each row (across columns), then average the rows
denominator_nonzero = 10**(-5)
# autoencoder loss
if params['relative_loss']:
loss1_denominator = tf.reduce_mean(
tf.reduce_mean(tf.square(tf.squeeze(x[0, :, :])),
1)) + denominator_nonzero
else:
loss1_denominator = tf.to_double(1.0)
mean_squared_error = tf.reduce_mean(
tf.reduce_mean(tf.square(y[0] - tf.squeeze(x[0, :, :])), 1))
loss1 = params['recon_lam'] * tf.truediv(mean_squared_error,
loss1_denominator)
# gets dynamics
loss2 = tf.zeros([
1,
], dtype=tf.float64)
if params['num_shifts'] > 0:
for j in np.arange(params['num_shifts']):
# xk+1, xk+2, xk+3
shift = params['shifts'][j]
if params['relative_loss']:
loss2_denominator = tf.reduce_mean(
tf.reduce_mean(tf.square(tf.squeeze(x[shift, :, :])),
1)) + denominator_nonzero
else:
loss2_denominator = tf.to_double(1.0)
loss2 = loss2 + params['recon_lam'] * tf.truediv(
tf.reduce_mean(
tf.reduce_mean(
tf.square(y[j + 1] - tf.squeeze(x[shift, :, :])), 1)),
loss2_denominator)
loss2 = loss2 / params['num_shifts']
# K linear
loss3 = tf.zeros([
1,
], dtype=tf.float64)
count_shifts_middle = 0
if params['num_shifts_middle'] > 0:
next_step = net.varying_multiply(g_list[0], g_list_omega[0],
params['delta_t'])
for j in np.arange(max(params['shifts_middle'])):
if (j + 1) in params['shifts_middle']:
# multiply g_list[0] by L (j+1) times
# next_step = tf.matmul(g_list[0], L_pow)
if params['relative_loss']:
loss3_denominator = tf.reduce_mean(
tf.reduce_mean(
tf.square(
tf.squeeze(g_list[count_shifts_middle + 1])),
1)) + denominator_nonzero
else:
loss3_denominator = tf.to_double(1.0)
loss3 = loss3 + params['mid_shift_lam'] * tf.truediv(
tf.reduce_mean(
tf.reduce_mean(
tf.square(next_step -
g_list[count_shifts_middle + 1]), 1)),
loss3_denominator)
count_shifts_middle += 1
# hopefully still on correct traj, so same omegas as before
next_step = net.varying_multiply(next_step, g_list_omega[j + 1],
params['delta_t'])
loss3 = loss3 / params['num_shifts_middle']
# inf norm on autoencoder error
Linf1_den = tf.norm(tf.norm(tf.squeeze(x[0, :, :]), axis=1, ord=np.inf),
ord=np.inf) + denominator_nonzero
Linf2_den = tf.norm(tf.norm(tf.squeeze(x[1, :, :]), axis=1, ord=np.inf),
ord=np.inf) + denominator_nonzero
Linf1_penalty = tf.truediv(
tf.norm(tf.norm(y[0] - tf.squeeze(x[0, :, :]), axis=1, ord=np.inf),
ord=np.inf), Linf1_den)
Linf2_penalty = tf.truediv(
tf.norm(tf.norm(y[1] - tf.squeeze(x[1, :, :]), axis=1, ord=np.inf),
ord=np.inf), Linf2_den)
loss_Linf = params['Linf_lam'] * (Linf1_penalty + Linf2_penalty)
loss = loss1 + loss2 + loss3 + loss_Linf
return loss1, loss2, loss3, loss_Linf, loss