def nn_pdf_model(features, labels, mode, params):
positive_transform = params['positive_transform']
learning_rate = params["learning_rate"]
cov_type = params["cov_type"]
x_size, y_size, x, y = extract_xy(features, params)
if y is None:
raise NotImplementedError
x_transform = transform_x(params, x)
cdfs, pdfs = density_estimator(params, positive_transform, x_transform, y, y_size)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = marginals_and_joint(cov_type, cdfs, pdfs,x, params)
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
predictions = marginals_and_joint(cov_type, cdfs, pdfs, x, params)
loss = get_loss(predictions)
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(predictions["log_likelihood"])}
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def nn_pdf_ar_model(features, labels, mode, params):
positive_transform = params['positive_transform']
learning_rate = params["learning_rate"]
x_size, y_size, x, y = extract_xy(features, params)
if y is None:
raise NotImplementedError
predictions_list = []
all_cdfs = []
for y_i in range(y_size):
with tf.variable_scope("y_%d" % y_i):
if y_i > 0:
y_slice = tf.slice(y, [0, y_i - 1], [-1, 1])
if x is not None:
x = tf.concat([x, y_slice], axis=1)
else:
x = y_slice
y_marginal = tf.slice(y, [0, y_i], [-1, 1])
x_transform = transform_x(params, x)
cdfs, pdfs = density_estimator(params,
positive_transform,
x_transform,
y_marginal,
y_size=1)
all_cdfs.extend(cdfs)
predictions_list.append(
marginals_and_joint(None, cdfs, pdfs, x, params))
ll = tf.add_n(
[predictions["log_likelihood"] for predictions in predictions_list])
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
"log_likelihood_%d" % i: pred["log_likelihood"]
for i, pred in enumerate(predictions_list)
}
for i, cdf in enumerate(all_cdfs):
predictions['cdf_%d' % i] = cdf
predictions["log_likelihood"] = ll
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = tf.negative(tf.reduce_mean(ll), name="loss")
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(ll)}
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op(loss, learning_rate))
def rnade_deep_model(features, labels, mode, params):
x_size, y_size, x, y = extract_xy(features, params)
learning_rate = params['learning_rate']
if mode == tf.estimator.ModeKeys.PREDICT:
ll = compute_ensamble(params, x,y, y_size)
predictions = {"log_likelihood": ll, "pdf": tf.exp(ll)}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
if mode == tf.estimator.ModeKeys.EVAL:
ll = compute_ensamble(params, x, y, y_size)
metrics = {"log_likelihood": metric_loglikelihood(ll)}
return tf.estimator.EstimatorSpec(mode, loss=tf.negative(tf.reduce_mean(ll)), eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
idx = sample_idx(y_size)
ordering = sample_ordering(y_size)
ll = ll_for_random_ordering(x, idx, ordering, y, y_size, params)
loss = tf.cast(y_size, dtype=tf.float32) / tf.cast(y_size - idx, tf.float32) * tf.negative(tf.reduce_mean(ll))
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def nn_pdf_m_model(features, labels, mode, params):
positive_transform = params['positive_transform']
learning_rate = params["learning_rate"]
x_size, y_size, x, y = extract_xy(features, params)
y = print_tensor(y, name="y")
if y is None or y.shape[-1].value != 2:
raise NotImplementedError
y_components = [tf.slice(y, [0,i],[-1,1]) for i in range(y_size)]
y_components_combined = tf.concat(y_components, axis=1)
non_normalized_cdf, cdf_normalization = cdf_transforms(x, y_components_combined, params, positive_transform, mode)
non_normalized_cdf = tf.check_numerics(non_normalized_cdf, message="non_normalized_cdf: ")
cdf_normalization = tf.check_numerics(cdf_normalization, message="cdf_normalization")
grads = tf.maximum(create_pdf_layer_mv(non_normalized_cdf, y_components), 1e-24)
log_likelihood = tf.log(grads) - tf.log(tf.maximum(cdf_normalization, 1e-24))
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {"log_likelihood": log_likelihood, "cdf":tf.div(non_normalized_cdf,cdf_normalization)}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = print_tensor(tf.negative(tf.reduce_mean(log_likelihood), name="loss"))
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(log_likelihood)}
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def rnade_model(features, labels, mode, params):
k_mix = params["k_mix"]
hidden_units = params['hidden_units']
learning_rate = params["learning_rate"]
components_distribution_param = params["components_distribution"]
x_size, y_size, x, y = extract_xy(features, params)
c = print_tensor(
add_all_summary_stats(
tf.get_variable(name="c", shape=(hidden_units, ),
dtype=tf.float32)))
W = print_tensor(
add_all_summary_stats(
tf.get_variable(name="W",
shape=(x_size + y_size - 1, hidden_units),
dtype=tf.float32)), "W")
rho = add_all_summary_stats(
tf.get_variable(name="rho", shape=(y_size, ), dtype=tf.float32))
if x is not None:
a = print_tensor(
add_all_summary_stats(
tf.add(c,
tf.matmul(
x,
print_tensor(tf.slice(W, [0, 0],
size=[x.shape[1].value, -1]),
name="x_times_W")),
name='a_first')))
else:
a = tf.fill((tf.shape(y)[0], hidden_units), 0.0)
ll = tf.constant([0.], dtype=tf.float32)
lls = []
for d in range(y_size):
psi = add_all_summary_stats(tf.multiply(rho[d], a, name="psi_%d" %
d)) # Rescaling factors
h = add_all_summary_stats(tf.nn.relu(psi, name="h_%d" %
d)) # Rectified linear unit
V_alpha = add_all_summary_stats(
tf.get_variable(name="V_alpha_%d" % d,
shape=(hidden_units, k_mix),
dtype=tf.float32))
b_alpha = add_all_summary_stats(
tf.get_variable(name="b_alpha_%d" % d,
shape=(k_mix, ),
dtype=tf.float32))
z_alpha = print_tensor(
add_all_summary_stats(
tf.add(tf.matmul(h, V_alpha), b_alpha, name="z_alpha_%d" % d)))
V_mu = add_all_summary_stats(
tf.get_variable(name="V_mu_%d" % d,
shape=(hidden_units, k_mix),
dtype=tf.float32))
b_mu = add_all_summary_stats(
tf.get_variable(name="b_mu_%d" % d,
shape=(k_mix, ),
dtype=tf.float32))
z_mu = tf.add(tf.matmul(h, V_mu), b_mu, name="mu_%d" % d)
V_sigma = add_all_summary_stats(
tf.get_variable(name="V_sigma_%d" % d,
shape=(hidden_units, k_mix),
dtype=tf.float32))
b_sigma = add_all_summary_stats(
tf.get_variable(name="b_sigma_%d" % d,
shape=(k_mix, ),
dtype=tf.float32))
z_sigma = print_tensor(
add_all_summary_stats(
tf.add(tf.matmul(h, V_sigma), b_sigma, name="z_sigma_%d" % d)))
mu = print_tensor(add_all_summary_stats(z_mu))
sigma = print_tensor(
add_all_summary_stats(
tf.maximum(1e-24, tf.square(z_sigma), name="sigma_%d" % d)))
if components_distribution_param == "normal":
components_distribution = tp.distributions.Normal(loc=mu,
scale=sigma)
elif components_distribution_param == "laplace":
components_distribution = tp.distributions.Laplace(loc=mu,
scale=sigma)
else:
raise ValueError(components_distribution_param)
mix = tp.distributions.MixtureSameFamily(
mixture_distribution=tp.distributions.Categorical(
logits=z_alpha, allow_nan_stats=False),
components_distribution=components_distribution,
allow_nan_stats=False)
if y is not None:
y_d = tf.slice(y, [0, d], size=[-1, 1], name="y_%d" % d)
ll_component = mix.log_prob(tf.reshape(y_d, [-1]))
ll = ll + ll_component
lls.append(ll_component)
if d < (y_size - 1):
a = add_all_summary_stats(
tf.add(a,
tf.matmul(
y_d, tf.slice(W, [x_size + d, 0], size=[1,
-1])),
name="a_%d" % d))
if y is not None:
ll = print_tensor(
add_all_summary_stats(tf.reshape(ll, [-1, 1], name="ll")))
if mode == tf.estimator.ModeKeys.PREDICT:
if y is not None:
predictions = {
"log_likelihood_%d" % i: ll
for i, ll in enumerate(lls)
}
predictions["log_likelihood"] = ll
else:
predictions = {
'samples0': tf.reshape(mix.sample(1, name="sample"), [-1, 1])
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = print_tensor(-tf.reduce_mean(ll), "loss")
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(ll)}
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op(loss, learning_rate))
def mdn_model(features, labels, mode, params):
k_mix = params["k_mix"]
num_layers = params["num_layers"]
hidden_units = params['hidden_units']
learning_rate = params["learning_rate"]
cov_type = params["cov_type"]
x_size, y_size, x, y = extract_xy(features, params)
layer = x
n_out = k_mix * 3 * y_size # pi, mu, stdev
if x is not None:
for i in range(num_layers):
layer = tf.layers.dense(layer,
units=hidden_units,
activation=tf.nn.tanh,
name="h_layer_%d" % i)
Wo = tf.get_variable("Wo",
shape=[params['hidden_units'], n_out],
dtype=tf.float32)
bo = tf.get_variable("bo", shape=[1, n_out], dtype=tf.float32)
output = tf.matmul(layer, Wo) + bo
else:
output = tf.get_variable("output", shape=(1, n_out))
mixtures = get_mixtures(output, y_size)
if mode == tf.estimator.ModeKeys.PREDICT:
if 'y' in "".join(features.keys()):
marginal_lls = compute_lls(mixtures, y)
predictions = {
'pdf%d' % i: tf.exp(ll)
for i, ll in enumerate(marginal_lls)
}
predictions["log_likelihood"] = log_likelihood(
cov_type, mixtures, y, marginal_lls, x, params)
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
else:
predictions = {
'samples%d' % i: generate_ensemble(mix)
for i, mix in enumerate(mixtures)
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
marginal_lls = compute_lls(mixtures, y)
ll = log_likelihood(cov_type, mixtures, y, marginal_lls, x, params)
loss = print_tensor(get_loss(ll))
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(ll)}
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op(loss, learning_rate))