def create_cdf_layer_partial_monotonic_MLP(x_batch, y_batch, arch, positive_transform, final_activation=tf.nn.sigmoid, activation=tf.nn.tanh, print_prefix=""):
"""
CDF(y|x)
:param x_batch:
:param y_batch:
:param arch:
:return:
"""
with tf.name_scope("cdf_layer"):
if x_batch is None:
layer = y_batch
else:
layer = tf.concat([x_batch, y_batch], axis=1)
layer = create_partially_monotone_dense_layer(layer, arch[0], 0 if x_batch is None else x_batch.shape[-1].value,
y_batch.shape[-1].value,
activation=activation, positive_transform=positive_transform)
for i, size in enumerate(arch[1:]):
with tf.variable_scope("l%d" % (i + 1)):
layer = print_tensor(create_monotone_dense_layer(layer, size, activation=activation,
positive_transform=positive_transform), name=print_prefix+"monotone_%d"%i)
with tf.variable_scope("cdf"):
cdf = print_tensor(create_monotone_dense_layer(layer, 1, activation=final_activation, positive_transform=positive_transform),
name=print_prefix + "monotone_final")
return cdf
def transform_x(params, x):
x_transform = x
activation = get_activation(params)
if x is not None:
for i, units in enumerate(params['arch1']):
x_transform = print_tensor(tf.layers.dense(x_transform, units=units, activation=activation, name="x_transform_%d" % i), name="x_transform_%d" % i)
return x_transform
def nn_pdf_m_model(features, labels, mode, params):
positive_transform = params['positive_transform']
learning_rate = params["learning_rate"]
x_size, y_size, x, y = extract_xy(features, params)
y = print_tensor(y, name="y")
if y is None or y.shape[-1].value != 2:
raise NotImplementedError
y_components = [tf.slice(y, [0,i],[-1,1]) for i in range(y_size)]
y_components_combined = tf.concat(y_components, axis=1)
non_normalized_cdf, cdf_normalization = cdf_transforms(x, y_components_combined, params, positive_transform, mode)
non_normalized_cdf = tf.check_numerics(non_normalized_cdf, message="non_normalized_cdf: ")
cdf_normalization = tf.check_numerics(cdf_normalization, message="cdf_normalization")
grads = tf.maximum(create_pdf_layer_mv(non_normalized_cdf, y_components), 1e-24)
log_likelihood = tf.log(grads) - tf.log(tf.maximum(cdf_normalization, 1e-24))
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {"log_likelihood": log_likelihood, "cdf":tf.div(non_normalized_cdf,cdf_normalization)}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = print_tensor(tf.negative(tf.reduce_mean(log_likelihood), name="loss"))
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(log_likelihood)}
return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def get_mixtures(output, num_marginals):
marginal_params = tf.split(output,
num_or_size_splits=num_marginals,
axis=1)
mixtures = []
for marginal_id in range(num_marginals):
with tf.variable_scope("mixture_%d" % marginal_id):
out_logits, out_sigma, out_mu = tf.split(
marginal_params[marginal_id], num_or_size_splits=3, axis=1)
out_sigma = print_tensor(
add_all_summary_stats(
tf.maximum(1e-24, tf.square(out_sigma), name="sigma")))
mix = tp.distributions.MixtureSameFamily(
mixture_distribution=tp.distributions.Categorical(
logits=out_logits),
components_distribution=tp.distributions.Normal(
loc=out_mu, scale=out_sigma))
mixtures.append(mix)
return mixtures
def rnade_model(features, labels, mode, params):
k_mix = params["k_mix"]
hidden_units = params['hidden_units']
learning_rate = params["learning_rate"]
components_distribution_param = params["components_distribution"]
x_size, y_size, x, y = extract_xy(features, params)
c = print_tensor(
add_all_summary_stats(
tf.get_variable(name="c", shape=(hidden_units, ),
dtype=tf.float32)))
W = print_tensor(
add_all_summary_stats(
tf.get_variable(name="W",
shape=(x_size + y_size - 1, hidden_units),
dtype=tf.float32)), "W")
rho = add_all_summary_stats(
tf.get_variable(name="rho", shape=(y_size, ), dtype=tf.float32))
if x is not None:
a = print_tensor(
add_all_summary_stats(
tf.add(c,
tf.matmul(
x,
print_tensor(tf.slice(W, [0, 0],
size=[x.shape[1].value, -1]),
name="x_times_W")),
name='a_first')))
else:
a = tf.fill((tf.shape(y)[0], hidden_units), 0.0)
ll = tf.constant([0.], dtype=tf.float32)
lls = []
for d in range(y_size):
psi = add_all_summary_stats(tf.multiply(rho[d], a, name="psi_%d" %
d)) # Rescaling factors
h = add_all_summary_stats(tf.nn.relu(psi, name="h_%d" %
d)) # Rectified linear unit
V_alpha = add_all_summary_stats(
tf.get_variable(name="V_alpha_%d" % d,
shape=(hidden_units, k_mix),
dtype=tf.float32))
b_alpha = add_all_summary_stats(
tf.get_variable(name="b_alpha_%d" % d,
shape=(k_mix, ),
dtype=tf.float32))
z_alpha = print_tensor(
add_all_summary_stats(
tf.add(tf.matmul(h, V_alpha), b_alpha, name="z_alpha_%d" % d)))
V_mu = add_all_summary_stats(
tf.get_variable(name="V_mu_%d" % d,
shape=(hidden_units, k_mix),
dtype=tf.float32))
b_mu = add_all_summary_stats(
tf.get_variable(name="b_mu_%d" % d,
shape=(k_mix, ),
dtype=tf.float32))
z_mu = tf.add(tf.matmul(h, V_mu), b_mu, name="mu_%d" % d)
V_sigma = add_all_summary_stats(
tf.get_variable(name="V_sigma_%d" % d,
shape=(hidden_units, k_mix),
dtype=tf.float32))
b_sigma = add_all_summary_stats(
tf.get_variable(name="b_sigma_%d" % d,
shape=(k_mix, ),
dtype=tf.float32))
z_sigma = print_tensor(
add_all_summary_stats(
tf.add(tf.matmul(h, V_sigma), b_sigma, name="z_sigma_%d" % d)))
mu = print_tensor(add_all_summary_stats(z_mu))
sigma = print_tensor(
add_all_summary_stats(
tf.maximum(1e-24, tf.square(z_sigma), name="sigma_%d" % d)))
if components_distribution_param == "normal":
components_distribution = tp.distributions.Normal(loc=mu,
scale=sigma)
elif components_distribution_param == "laplace":
components_distribution = tp.distributions.Laplace(loc=mu,
scale=sigma)
else:
raise ValueError(components_distribution_param)
mix = tp.distributions.MixtureSameFamily(
mixture_distribution=tp.distributions.Categorical(
logits=z_alpha, allow_nan_stats=False),
components_distribution=components_distribution,
allow_nan_stats=False)
if y is not None:
y_d = tf.slice(y, [0, d], size=[-1, 1], name="y_%d" % d)
ll_component = mix.log_prob(tf.reshape(y_d, [-1]))
ll = ll + ll_component
lls.append(ll_component)
if d < (y_size - 1):
a = add_all_summary_stats(
tf.add(a,
tf.matmul(
y_d, tf.slice(W, [x_size + d, 0], size=[1,
-1])),
name="a_%d" % d))
if y is not None:
ll = print_tensor(
add_all_summary_stats(tf.reshape(ll, [-1, 1], name="ll")))
if mode == tf.estimator.ModeKeys.PREDICT:
if y is not None:
predictions = {
"log_likelihood_%d" % i: ll
for i, ll in enumerate(lls)
}
predictions["log_likelihood"] = ll
else:
predictions = {
'samples0': tf.reshape(mix.sample(1, name="sample"), [-1, 1])
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
loss = print_tensor(-tf.reduce_mean(ll), "loss")
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(ll)}
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op(loss, learning_rate))
def mdn_model(features, labels, mode, params):
k_mix = params["k_mix"]
num_layers = params["num_layers"]
hidden_units = params['hidden_units']
learning_rate = params["learning_rate"]
cov_type = params["cov_type"]
x_size, y_size, x, y = extract_xy(features, params)
layer = x
n_out = k_mix * 3 * y_size # pi, mu, stdev
if x is not None:
for i in range(num_layers):
layer = tf.layers.dense(layer,
units=hidden_units,
activation=tf.nn.tanh,
name="h_layer_%d" % i)
Wo = tf.get_variable("Wo",
shape=[params['hidden_units'], n_out],
dtype=tf.float32)
bo = tf.get_variable("bo", shape=[1, n_out], dtype=tf.float32)
output = tf.matmul(layer, Wo) + bo
else:
output = tf.get_variable("output", shape=(1, n_out))
mixtures = get_mixtures(output, y_size)
if mode == tf.estimator.ModeKeys.PREDICT:
if 'y' in "".join(features.keys()):
marginal_lls = compute_lls(mixtures, y)
predictions = {
'pdf%d' % i: tf.exp(ll)
for i, ll in enumerate(marginal_lls)
}
predictions["log_likelihood"] = log_likelihood(
cov_type, mixtures, y, marginal_lls, x, params)
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
else:
predictions = {
'samples%d' % i: generate_ensemble(mix)
for i, mix in enumerate(mixtures)
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
marginal_lls = compute_lls(mixtures, y)
ll = log_likelihood(cov_type, mixtures, y, marginal_lls, x, params)
loss = print_tensor(get_loss(ll))
if mode == tf.estimator.ModeKeys.EVAL:
metrics = {"log_likelihood": metric_loglikelihood(ll)}
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op(loss, learning_rate))