def nn_pdf_model(features, labels, mode, params): positive_transform = params['positive_transform'] learning_rate = params["learning_rate"] cov_type = params["cov_type"] x_size, y_size, x, y = extract_xy(features, params) if y is None: raise NotImplementedError x_transform = transform_x(params, x) cdfs, pdfs = density_estimator(params, positive_transform, x_transform, y, y_size) if mode == tf.estimator.ModeKeys.PREDICT: predictions = marginals_and_joint(cov_type, cdfs, pdfs,x, params) return tf.estimator.EstimatorSpec(mode, predictions=predictions) predictions = marginals_and_joint(cov_type, cdfs, pdfs, x, params) loss = get_loss(predictions) if mode == tf.estimator.ModeKeys.EVAL: metrics = {"log_likelihood": metric_loglikelihood(predictions["log_likelihood"])} return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) assert mode == tf.estimator.ModeKeys.TRAIN return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def nn_pdf_ar_model(features, labels, mode, params): positive_transform = params['positive_transform'] learning_rate = params["learning_rate"] x_size, y_size, x, y = extract_xy(features, params) if y is None: raise NotImplementedError predictions_list = [] all_cdfs = [] for y_i in range(y_size): with tf.variable_scope("y_%d" % y_i): if y_i > 0: y_slice = tf.slice(y, [0, y_i - 1], [-1, 1]) if x is not None: x = tf.concat([x, y_slice], axis=1) else: x = y_slice y_marginal = tf.slice(y, [0, y_i], [-1, 1]) x_transform = transform_x(params, x) cdfs, pdfs = density_estimator(params, positive_transform, x_transform, y_marginal, y_size=1) all_cdfs.extend(cdfs) predictions_list.append( marginals_and_joint(None, cdfs, pdfs, x, params)) ll = tf.add_n( [predictions["log_likelihood"] for predictions in predictions_list]) if mode == tf.estimator.ModeKeys.PREDICT: predictions = { "log_likelihood_%d" % i: pred["log_likelihood"] for i, pred in enumerate(predictions_list) } for i, cdf in enumerate(all_cdfs): predictions['cdf_%d' % i] = cdf predictions["log_likelihood"] = ll return tf.estimator.EstimatorSpec(mode, predictions=predictions) loss = tf.negative(tf.reduce_mean(ll), name="loss") if mode == tf.estimator.ModeKeys.EVAL: metrics = {"log_likelihood": metric_loglikelihood(ll)} return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) assert mode == tf.estimator.ModeKeys.TRAIN return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def rnade_deep_model(features, labels, mode, params): x_size, y_size, x, y = extract_xy(features, params) learning_rate = params['learning_rate'] if mode == tf.estimator.ModeKeys.PREDICT: ll = compute_ensamble(params, x,y, y_size) predictions = {"log_likelihood": ll, "pdf": tf.exp(ll)} return tf.estimator.EstimatorSpec(mode, predictions=predictions) if mode == tf.estimator.ModeKeys.EVAL: ll = compute_ensamble(params, x, y, y_size) metrics = {"log_likelihood": metric_loglikelihood(ll)} return tf.estimator.EstimatorSpec(mode, loss=tf.negative(tf.reduce_mean(ll)), eval_metric_ops=metrics) assert mode == tf.estimator.ModeKeys.TRAIN idx = sample_idx(y_size) ordering = sample_ordering(y_size) ll = ll_for_random_ordering(x, idx, ordering, y, y_size, params) loss = tf.cast(y_size, dtype=tf.float32) / tf.cast(y_size - idx, tf.float32) * tf.negative(tf.reduce_mean(ll)) return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def nn_pdf_m_model(features, labels, mode, params): positive_transform = params['positive_transform'] learning_rate = params["learning_rate"] x_size, y_size, x, y = extract_xy(features, params) y = print_tensor(y, name="y") if y is None or y.shape[-1].value != 2: raise NotImplementedError y_components = [tf.slice(y, [0,i],[-1,1]) for i in range(y_size)] y_components_combined = tf.concat(y_components, axis=1) non_normalized_cdf, cdf_normalization = cdf_transforms(x, y_components_combined, params, positive_transform, mode) non_normalized_cdf = tf.check_numerics(non_normalized_cdf, message="non_normalized_cdf: ") cdf_normalization = tf.check_numerics(cdf_normalization, message="cdf_normalization") grads = tf.maximum(create_pdf_layer_mv(non_normalized_cdf, y_components), 1e-24) log_likelihood = tf.log(grads) - tf.log(tf.maximum(cdf_normalization, 1e-24)) if mode == tf.estimator.ModeKeys.PREDICT: predictions = {"log_likelihood": log_likelihood, "cdf":tf.div(non_normalized_cdf,cdf_normalization)} return tf.estimator.EstimatorSpec(mode, predictions=predictions) loss = print_tensor(tf.negative(tf.reduce_mean(log_likelihood), name="loss")) if mode == tf.estimator.ModeKeys.EVAL: metrics = {"log_likelihood": metric_loglikelihood(log_likelihood)} return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) assert mode == tf.estimator.ModeKeys.TRAIN return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def rnade_model(features, labels, mode, params): k_mix = params["k_mix"] hidden_units = params['hidden_units'] learning_rate = params["learning_rate"] components_distribution_param = params["components_distribution"] x_size, y_size, x, y = extract_xy(features, params) c = print_tensor( add_all_summary_stats( tf.get_variable(name="c", shape=(hidden_units, ), dtype=tf.float32))) W = print_tensor( add_all_summary_stats( tf.get_variable(name="W", shape=(x_size + y_size - 1, hidden_units), dtype=tf.float32)), "W") rho = add_all_summary_stats( tf.get_variable(name="rho", shape=(y_size, ), dtype=tf.float32)) if x is not None: a = print_tensor( add_all_summary_stats( tf.add(c, tf.matmul( x, print_tensor(tf.slice(W, [0, 0], size=[x.shape[1].value, -1]), name="x_times_W")), name='a_first'))) else: a = tf.fill((tf.shape(y)[0], hidden_units), 0.0) ll = tf.constant([0.], dtype=tf.float32) lls = [] for d in range(y_size): psi = add_all_summary_stats(tf.multiply(rho[d], a, name="psi_%d" % d)) # Rescaling factors h = add_all_summary_stats(tf.nn.relu(psi, name="h_%d" % d)) # Rectified linear unit V_alpha = add_all_summary_stats( tf.get_variable(name="V_alpha_%d" % d, shape=(hidden_units, k_mix), dtype=tf.float32)) b_alpha = add_all_summary_stats( tf.get_variable(name="b_alpha_%d" % d, shape=(k_mix, ), dtype=tf.float32)) z_alpha = print_tensor( add_all_summary_stats( tf.add(tf.matmul(h, V_alpha), b_alpha, name="z_alpha_%d" % d))) V_mu = add_all_summary_stats( tf.get_variable(name="V_mu_%d" % d, shape=(hidden_units, k_mix), dtype=tf.float32)) b_mu = add_all_summary_stats( tf.get_variable(name="b_mu_%d" % d, shape=(k_mix, ), dtype=tf.float32)) z_mu = tf.add(tf.matmul(h, V_mu), b_mu, name="mu_%d" % d) V_sigma = add_all_summary_stats( tf.get_variable(name="V_sigma_%d" % d, shape=(hidden_units, k_mix), dtype=tf.float32)) b_sigma = add_all_summary_stats( tf.get_variable(name="b_sigma_%d" % d, shape=(k_mix, ), dtype=tf.float32)) z_sigma = print_tensor( add_all_summary_stats( tf.add(tf.matmul(h, V_sigma), b_sigma, name="z_sigma_%d" % d))) mu = print_tensor(add_all_summary_stats(z_mu)) sigma = print_tensor( add_all_summary_stats( tf.maximum(1e-24, tf.square(z_sigma), name="sigma_%d" % d))) if components_distribution_param == "normal": components_distribution = tp.distributions.Normal(loc=mu, scale=sigma) elif components_distribution_param == "laplace": components_distribution = tp.distributions.Laplace(loc=mu, scale=sigma) else: raise ValueError(components_distribution_param) mix = tp.distributions.MixtureSameFamily( mixture_distribution=tp.distributions.Categorical( logits=z_alpha, allow_nan_stats=False), components_distribution=components_distribution, allow_nan_stats=False) if y is not None: y_d = tf.slice(y, [0, d], size=[-1, 1], name="y_%d" % d) ll_component = mix.log_prob(tf.reshape(y_d, [-1])) ll = ll + ll_component lls.append(ll_component) if d < (y_size - 1): a = add_all_summary_stats( tf.add(a, tf.matmul( y_d, tf.slice(W, [x_size + d, 0], size=[1, -1])), name="a_%d" % d)) if y is not None: ll = print_tensor( add_all_summary_stats(tf.reshape(ll, [-1, 1], name="ll"))) if mode == tf.estimator.ModeKeys.PREDICT: if y is not None: predictions = { "log_likelihood_%d" % i: ll for i, ll in enumerate(lls) } predictions["log_likelihood"] = ll else: predictions = { 'samples0': tf.reshape(mix.sample(1, name="sample"), [-1, 1]) } return tf.estimator.EstimatorSpec(mode, predictions=predictions) loss = print_tensor(-tf.reduce_mean(ll), "loss") if mode == tf.estimator.ModeKeys.EVAL: metrics = {"log_likelihood": metric_loglikelihood(ll)} return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) assert mode == tf.estimator.ModeKeys.TRAIN return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))
def mdn_model(features, labels, mode, params): k_mix = params["k_mix"] num_layers = params["num_layers"] hidden_units = params['hidden_units'] learning_rate = params["learning_rate"] cov_type = params["cov_type"] x_size, y_size, x, y = extract_xy(features, params) layer = x n_out = k_mix * 3 * y_size # pi, mu, stdev if x is not None: for i in range(num_layers): layer = tf.layers.dense(layer, units=hidden_units, activation=tf.nn.tanh, name="h_layer_%d" % i) Wo = tf.get_variable("Wo", shape=[params['hidden_units'], n_out], dtype=tf.float32) bo = tf.get_variable("bo", shape=[1, n_out], dtype=tf.float32) output = tf.matmul(layer, Wo) + bo else: output = tf.get_variable("output", shape=(1, n_out)) mixtures = get_mixtures(output, y_size) if mode == tf.estimator.ModeKeys.PREDICT: if 'y' in "".join(features.keys()): marginal_lls = compute_lls(mixtures, y) predictions = { 'pdf%d' % i: tf.exp(ll) for i, ll in enumerate(marginal_lls) } predictions["log_likelihood"] = log_likelihood( cov_type, mixtures, y, marginal_lls, x, params) return tf.estimator.EstimatorSpec(mode, predictions=predictions) else: predictions = { 'samples%d' % i: generate_ensemble(mix) for i, mix in enumerate(mixtures) } return tf.estimator.EstimatorSpec(mode, predictions=predictions) marginal_lls = compute_lls(mixtures, y) ll = log_likelihood(cov_type, mixtures, y, marginal_lls, x, params) loss = print_tensor(get_loss(ll)) if mode == tf.estimator.ModeKeys.EVAL: metrics = {"log_likelihood": metric_loglikelihood(ll)} return tf.estimator.EstimatorSpec(mode, loss=loss, eval_metric_ops=metrics) # Create training op. assert mode == tf.estimator.ModeKeys.TRAIN return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op(loss, learning_rate))