def __init__(self,num_classes, learning_rate, batch_size, decay_steps, decay_rate,sequence_length,
vocab_size,embed_size,is_training,initializer=tf.random_normal_initializer(stddev=0.1)):
"""init all hyperparameter here"""
# set hyperparamter
self.num_classes = num_classes
self.batch_size = batch_size
self.sequence_length=sequence_length
self.vocab_size=vocab_size
self.embed_size=embed_size
self.hidden_size=embed_size
self.is_training=is_training
self.learning_rate=learning_rate
self.initializer=initializer
self.num_sampled=20
# add placeholder (X,label)
self.input_x = tf.placeholder(tf.int32, [None, self.sequence_length], name="input_x") # X
self.input_y = tf.placeholder(tf.int32,[None], name="input_y") # y [None,num_classes]
self.dropout_keep_prob=tf.placeholder(tf.float32,name="dropout_keep_prob")
self.global_step = tf.Variable(0, trainable=False, name="Global_Step")
self.epoch_step=tf.Variable(0,trainable=False,name="Epoch_Step")
self.epoch_increment=tf.assign(self.epoch_step,tf.add(self.epoch_step,tf.constant(1)))
self.decay_steps, self.decay_rate = decay_steps, decay_rate
self.instantiate_weights()
self.logits = self.inference() #[None, self.label_size]. main computation graph is here.
if not is_training:
return
self.loss_val = self.loss() #-->self.loss_nce()
self.train_op = self.train()
self.predictions = tf.argmax(self.logits, axis=1, name="predictions") # shape:[None,]
correct_prediction = tf.equal(tf.cast(self.predictions,tf.int32), self.input_y) #tf.argmax(self.logits, 1)-->[batch_size]
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name="Accuracy") # shape=()
def _build_net(self):
def build_layers(s, c_names, n_l1, w_initializer, b_initializer):
with tf.variable_scope('l1'):
w1 = tf.get_variable('w1', [self.n_features, n_l1], initializer=w_initializer, collections=c_names)
b1 = tf.get_variable('b1', [1, n_l1], initializer=b_initializer, collections=c_names)
l1 = tf.nn.relu(tf.matmul(s, w1) + b1)
with tf.variable_scope('l2'):
w2 = tf.get_variable('w2', [n_l1, self.n_actions], initializer=w_initializer, collections=c_names)
b2 = tf.get_variable('b2', [1, self.n_actions], initializer=b_initializer, collections=c_names)
out = tf.matmul(l1, w2) + b2
return out
# ------------------ build evaluate_net ------------------
self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s') # input
self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target') # for calculating loss
with tf.variable_scope('eval_net'):
c_names, n_l1, w_initializer, b_initializer = \
['eval_net_params', tf.GraphKeys.GLOBAL_VARIABLES], 20, \
tf.random_normal_initializer(0., 0.3), tf.constant_initializer(0.1) # config of layers
self.q_eval = build_layers(self.s, c_names, n_l1, w_initializer, b_initializer)
with tf.variable_scope('loss'):
self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))
with tf.variable_scope('train'):
self._train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
# ------------------ build target_net ------------------
self.s_ = tf.placeholder(tf.float32, [None, self.n_features], name='s_') # input
with tf.variable_scope('target_net'):
c_names = ['target_net_params', tf.GraphKeys.GLOBAL_VARIABLES]
self.q_next = build_layers(self.s_, c_names, n_l1, w_initializer, b_initializer)
def __init__(self, is_training=True):
self.classes = cfg.CLASSES
self.num_class = len(self.classes)
self.image_size = cfg.IMAGE_SIZE
self.cell_size = cfg.CELL_SIZE
self.boxes_per_cell = cfg.BOXES_PER_CELL
self.output_size = (self.cell_size * self.cell_size) * (self.num_class + self.boxes_per_cell * 5)
self.scale = 1.0 * self.image_size / self.cell_size
self.boundary1 = self.cell_size * self.cell_size * self.num_class
self.boundary2 = self.boundary1 + self.cell_size * self.cell_size * self.boxes_per_cell
self.object_scale = cfg.OBJECT_SCALE
self.noobject_scale = cfg.NOOBJECT_SCALE
self.class_scale = cfg.CLASS_SCALE
self.coord_scale = cfg.COORD_SCALE
self.learning_rate = cfg.LEARNING_RATE
self.batch_size = cfg.BATCH_SIZE
self.alpha = cfg.ALPHA
self.offset = np.transpose(np.reshape(np.array(
[np.arange(self.cell_size)] * self.cell_size * self.boxes_per_cell),
(self.boxes_per_cell, self.cell_size, self.cell_size)), (1, 2, 0))
self.images = tf.placeholder(tf.float32, [None, self.image_size, self.image_size, 3], name='images')
self.logits = self.build_network(self.images, num_outputs=self.output_size, alpha=self.alpha, is_training=is_training)
if is_training:
self.labels = tf.placeholder(tf.float32, [None, self.cell_size, self.cell_size, 5 + self.num_class])
self.loss_layer(self.logits, self.labels)
self.total_loss = tf.losses.get_total_loss()
tf.summary.scalar('total_loss', self.total_loss)
def testGradient(self):
with tf.Graph().as_default() as g:
inputs = tf.placeholder(tf.float32, shape=[None, 100], name="input")
weights = tf.placeholder(tf.float32, shape=[100, 10], name="weights")
biases = tf.placeholder(tf.float32, shape=[10], name="biases")
activations = tf.nn.relu(tf.matmul(inputs, weights) + biases,
name="activations")
loss = tf.reduce_mean(activations, name="loss")
gdef = g.as_graph_def()
with tf.Graph().as_default() as g:
input_placeholder = tf.placeholder(tf.float32, shape=[32, 100])
weights_var = tf.Variable(tf.truncated_normal([100, 10]), name="weights")
biases_var = tf.Variable(tf.zeros(10), name="biases")
activations, loss = tf.import_graph_def(
gdef,
input_map={"input:0": input_placeholder,
"weights:0": weights_var,
"biases:0": biases_var},
return_elements=["activations:0", "loss:0"])
self.assertEqual([32, 10], activations.get_shape())
self.assertEqual([], loss.get_shape())
weights_grad, biases_grad = tf.gradients(loss, [weights_var, biases_var])
self.assertEqual([100, 10], weights_grad.get_shape())
self.assertEqual([10], biases_grad.get_shape())
def __init__(self, sess, ob_space, ac_space, nbatch, nsteps, nlstm=256, reuse=False):
nenv = nbatch // nsteps
self.pdtype = make_pdtype(ac_space)
X, processed_x = observation_input(ob_space, nbatch)
M = tf.placeholder(tf.float32, [nbatch]) #mask (done t-1)
S = tf.placeholder(tf.float32, [nenv, nlstm*2]) #states
with tf.variable_scope("model", reuse=reuse):
h = nature_cnn(X)
xs = batch_to_seq(h, nenv, nsteps)
ms = batch_to_seq(M, nenv, nsteps)
h5, snew = lstm(xs, ms, S, 'lstm1', nh=nlstm)
h5 = seq_to_batch(h5)
vf = fc(h5, 'v', 1)
self.pd, self.pi = self.pdtype.pdfromlatent(h5)
v0 = vf[:, 0]
a0 = self.pd.sample()
neglogp0 = self.pd.neglogp(a0)
self.initial_state = np.zeros((nenv, nlstm*2), dtype=np.float32)
def step(ob, state, mask):
return sess.run([a0, v0, snew, neglogp0], {X:ob, S:state, M:mask})
def value(ob, state, mask):
return sess.run(v0, {X:ob, S:state, M:mask})
self.X = X
self.M = M
self.S = S
self.vf = vf
self.step = step
self.value = value
def __init__(self, params, infer=False):
self.params = params
if infer:
self.batch_size = batch_size = 1
self.sequence_length = sequence_length = 1
else:
self.batch_size = batch_size = self.params.batch_size
self.sequence_length = sequence_length = self.params.sequence_length
cell1 = rnn_cell.LSTMCell(self.params.rnn_size,
self.params.input_channels,
use_peepholes=True)
cell2 = rnn_cell.LSTMCell(self.params.rnn_size,
cell1.output_size,
use_peepholes=True,
num_proj=params.output_channels)
self.cell = cell = rnn_cell.MultiRNNCell([cell1, cell2])
self.data_placeholder = tf.placeholder(tf.float32,
shape=(batch_size, params.input_channels, sequence_length),
name='data_placeholder')
self.labels_placeholder = tf.placeholder(tf.float32,
shape=(batch_size, params.input_channels, sequence_length),
name='labels_placeholder')
# Initial state of the LSTM memory.
# To train or to leave as all zeros...that is the question. get_variable means train, zeros means zeros.
# To make this a trainable variable we'd want it to be *the same* initial state for every sequence
# in a batch
self.initial_state = cell.zero_state(batch_size, dtype=tf.float32)
def add_placeholders(self):
"""Generate placeholder variables to represent the input tensors
These placeholders are used as inputs by the rest of the model building
code and will be fed data during training. Note that when "None" is in a
placeholder's shape, it's flexible
Adds following nodes to the computational graph
input_placeholder: Input placeholder tensor of shape
(None, window_size), type tf.int32
labels_placeholder: Labels placeholder tensor of shape
(None, label_size), type tf.float32
dropout_placeholder: Dropout value placeholder (scalar),
type tf.float32
Add these placeholders to self as the instance variables
self.input_placeholder
self.labels_placeholder
self.dropout_placeholder
(Don't change the variable names)
"""
### YOUR CODE HERE
window_size = self.config.window_size
self.input_placeholder = tf.placeholder(tf.int32, shape=(None, window_size))
self.labels_placeholder = tf.placeholder(tf.float32, shape=(None, self.config.label_size))
self.dropout_placeholder = tf.placeholder(tf.float32)
def build_graph(num_actions):
# Create shared deep q network
s, q_network = build_network(num_actions=num_actions, agent_history_length=FLAGS.agent_history_length,
resized_width=FLAGS.resized_width, resized_height=FLAGS.resized_height)
network_params = q_network.trainable_weights
q_values = q_network(s)
# Create shared target network
st, target_q_network = build_network(num_actions=num_actions, agent_history_length=FLAGS.agent_history_length,
resized_width=FLAGS.resized_width, resized_height=FLAGS.resized_height)
target_network_params = target_q_network.trainable_weights
target_q_values = target_q_network(st)
# Op for periodically updating target network with online network weights
reset_target_network_params = [target_network_params[i].assign(network_params[i]) for i in
range(len(target_network_params))]
# Define cost and gradient update op
a = tf.placeholder("float", [None, num_actions])
y = tf.placeholder("float", [None])
action_q_values = tf.reduce_sum(tf.mul(q_values, a), reduction_indices=1)
cost = tf.reduce_mean(tf.square(y - action_q_values))
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
grad_update = optimizer.minimize(cost, var_list=network_params)
graph_ops = {"s": s,
"q_values": q_values,
"st": st,
"target_q_values": target_q_values,
"reset_target_network_params": reset_target_network_params,
"a": a,
"y": y,
"grad_update": grad_update}
return graph_ops
def inputs(self):
"""
Define all the inputs (with type, shape, name) that
the graph will need.
"""
return [tf.placeholder(tf.float32, (None, IMAGE_SIZE, IMAGE_SIZE), 'input'),
tf.placeholder(tf.int32, (None,), 'label')]
def __init__(self, nA,
learning_rate,decay,grad_clip,entropy_beta,
state_shape=[84,84,4],
master=None, device_name='/gpu:0', scope_name='master'):
with tf.device(device_name) :
self.state = tf.placeholder(tf.float32,[None]+state_shape)
block, self.scope = ActorCritic._build_shared_block(self.state,scope_name)
self.policy, self.log_softmax_policy = ActorCritic._build_policy(block,nA,scope_name)
self.value = ActorCritic._build_value(block,scope_name)
self.train_vars = sorted(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, self.scope.name), key=lambda v:v.name)
if( master is not None ) :
self.sync_op= self._sync_op(master)
self.action = tf.placeholder(tf.int32,[None,])
self.target_value = tf.placeholder(tf.float32,[None,])
advantage = self.target_value - self.value
entropy = tf.reduce_sum(-1. * self.policy * self.log_softmax_policy,axis=1)
log_p_s_a = tf.reduce_sum(self.log_softmax_policy * tf.one_hot(self.action,nA),axis=1)
self.policy_loss = tf.reduce_mean(tf.stop_gradient(advantage)*log_p_s_a)
self.entropy_loss = tf.reduce_mean(entropy)
self.value_loss = tf.reduce_mean(advantage**2)
loss = -self.policy_loss - entropy_beta* self.entropy_loss + self.value_loss
self.gradients = tf.gradients(loss,self.train_vars)
clipped_gs = [tf.clip_by_average_norm(g,grad_clip) for g in self.gradients]
self.train_op = master.optimizer.apply_gradients(zip(clipped_gs,master.train_vars))
else :
#self.optimizer = tf.train.AdamOptimizer(learning_rate,beta1=BETA)
self.optimizer = tf.train.RMSPropOptimizer(learning_rate,decay=decay,use_locking=True)
def __init__(self):
# Import data
error = None
for _ in range(10):
try:
self.mnist = input_data.read_data_sets(
"/tmp/tensorflow/mnist/input_data", one_hot=True)
error = None
break
except Exception as e:
error = e
time.sleep(5)
if error:
raise ValueError("Failed to import data", error)
# Set seed and build layers
tf.set_random_seed(0)
self.x = tf.placeholder(tf.float32, [None, 784], name="x")
self.y_ = tf.placeholder(tf.float32, [None, 10], name="y_")
y_conv, self.keep_prob = deepnn(self.x)
# Need to define loss and optimizer attributes
self.loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(
labels=self.y_, logits=y_conv))
self.optimizer = tf.train.AdamOptimizer(1e-4)
self.variables = ray_tf_utils.TensorFlowVariables(
self.loss, tf.get_default_session())
# For evaluating test accuracy
correct_prediction = tf.equal(
tf.argmax(y_conv, 1), tf.argmax(self.y_, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
def create_graph(self):
with self.__graph.as_default():
self.__featurePlaceHolder = tf.placeholder(dtype=tf.int32, shape=[None, self.__window_size * 2])
self.__labelPlaceHolder = tf.placeholder(dtype=tf.int32, shape=[None, 1])
onehot_lookup_tables = tf.Variable(
initial_value=tf.truncated_normal(shape=[self.__vocabulary_size, self.__embedding_size])
)
embedding = tf.nn.embedding_lookup(params=onehot_lookup_tables, ids = self.__featurePlaceHolder)
projection_out = tf.reduce_mean(embedding, axis=1)
softmax_weight = tf.Variable(initial_value=tf.truncated_normal(
shape=[self.__vocabulary_size, self.__embedding_size]
))
softmax_biases = tf.Variable(initial_value=tf.zeros([self.__vocabulary_size]))
sampled_loss_per_batch = tf.nn.sampled_softmax_loss(
weights=softmax_weight,
biases=softmax_biases,
inputs=projection_out,
labels=self.__labelPlaceHolder,
num_sampled=self.__num_sampled,
num_classes=self.__vocabulary_size
)
self.__loss = tf.reduce_mean(sampled_loss_per_batch)
self.__optimizer = tf.train.AdagradOptimizer(1.0).minimize(self.__loss)
norm = tf.sqrt(tf.reduce_sum(tf.square(onehot_lookup_tables), 1, keep_dims=True))
self.__normalized_embedding = onehot_lookup_tables / norm
def ce(model, config, scope, connect, threshold = 1e-5):
with tf.variable_scope(scope), tf.name_scope(scope):
with tf.variable_scope('inputs'), tf.name_scope('inputs'):
model['%s_in0length' %scope] = model['%s_out0length' %connect]
model['%s_in1length' %scope] = model['%s_out1length' %connect]
model['%s_in2length' %scope] = model['%s_out2length' %connect]
model['%s_maxin2length' %scope] = model['%s_maxout2length' %connect]
model['%s_inputs' %scope] = tf.clip_by_value(tf.nn.softmax(model['%s_outputs' %connect]), threshold, 1. - threshold, name = '%s_inputs' %scope)
model['%s_out0length' %scope] = model['%s_in0length' %scope]
model['%s_out1length' %scope] = model['%s_in1length' %scope]
model['%s_out2length' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_out2length' %scope)
model['%s_maxout2length' %scope] = model['%s_maxin2length' %scope]
with tf.variable_scope('labels'), tf.name_scope('labels'):
model['%s_labels_len' %scope] = tf.placeholder(tf.int32, [model['%s_in0length' %scope]], '%s_labels_len' %scope)
model['%s_labels_ind' %scope] = tf.placeholder(tf.int64, [None, 2], '%s_labels_ind' %scope)
model['%s_labels_val' %scope] = tf.placeholder(tf.int32, [None], '%s_labels_val' %scope)
model['%s_labels_collapsed' %scope] = tf.sparse_to_dense(model['%s_labels_ind' %scope], [model['%s_maxin2length' %scope], model['%s_in0length' %scope]], model['%s_labels_val' %scope], -1, name = '%s_labels_collapsed' %scope)
model['%s_labels' %scope] = tf.one_hot(model['%s_labels_collapsed' %scope], model['%s_out1length' %scope], name = '%s_labels' %scope)
with tf.variable_scope('loss'), tf.name_scope('loss'):
model['%s_loss' %scope] = tf.reduce_sum(-tf.multiply(model['%s_labels' %scope], tf.log(model['%s_inputs' %scope])), name = '%s_loss' %scope)
with tf.variable_scope('outputs'), tf.name_scope('outputs'):
model['%s_output' %scope] = model['%s_inputs' %scope]
return model
def __init__(self, encoders, vocabulary, data_id,
layers=[], activation=tf.tanh, dropout_keep_p=0.5, name='seq_classifier'):
self.encoders = encoders
self.vocabulary = vocabulary
self.data_id = data_id
self.layers = layers
self.activation = activation
self.dropout_keep_p = dropout_keep_p
self.name = name
self.max_output_len = 1
with tf.variable_scope(name):
self.learning_step = tf.Variable(0, name="learning_step", trainable=False)
self.dropout_placeholder = tf.placeholder(tf.float32, name="dropout_plc")
self.gt_inputs = [tf.placeholder(tf.int32, shape=[None], name="targets")]
mlp_input = tf.concat(1, [enc.encoded for enc in encoders])
mlp = MultilayerPerceptron(mlp_input, layers, self.dropout_placeholder, len(vocabulary))
self.loss_with_gt_ins = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(mlp.logits, self.gt_inputs[0]))
self.loss_with_decoded_ins = self.loss_with_gt_ins
self.cost = self.loss_with_gt_ins
self.decoded_seq = [mlp.classification]
self.decoded_logits = [mlp.logits]
tf.scalar_summary('val_optimization_cost', self.cost, collections=["summary_val"])
tf.scalar_summary('train_optimization_cost', self.cost, collections=["summary_train"])
def testModelWithBucketsScopeAndLoss(self):
"""Test that variable scope reuse is not reset after model_with_buckets."""
classes = 10
buckets = [(4, 4), (8, 8)]
with self.test_session():
# Here comes a sample Seq2Seq model using GRU cells.
def SampleGRUSeq2Seq(enc_inp, dec_inp, weights, per_example_loss):
"""Example sequence-to-sequence model that uses GRU cells."""
def GRUSeq2Seq(enc_inp, dec_inp):
cell = tf.nn.rnn_cell.MultiRNNCell([tf.nn.rnn_cell.GRUCell(24)] * 2)
return tf.nn.seq2seq.embedding_attention_seq2seq(
enc_inp, dec_inp, cell, num_encoder_symbols=classes,
num_decoder_symbols=classes, embedding_size=24)
targets = [dec_inp[i+1] for i in range(len(dec_inp) - 1)] + [0]
return tf.nn.seq2seq.model_with_buckets(
enc_inp, dec_inp, targets, weights, buckets, GRUSeq2Seq,
per_example_loss=per_example_loss)
# Now we construct the copy model.
inp = [tf.placeholder(tf.int32, shape=[None]) for _ in range(8)]
out = [tf.placeholder(tf.int32, shape=[None]) for _ in range(8)]
weights = [tf.ones_like(inp[0], dtype=tf.float32) for _ in range(8)]
with tf.variable_scope("root"):
_, losses1 = SampleGRUSeq2Seq(inp, out, weights, per_example_loss=False)
# Now check that we did not accidentally set reuse.
self.assertEqual(False, tf.get_variable_scope().reuse)
# Construct one more model with per-example loss.
tf.get_variable_scope().reuse_variables()
_, losses2 = SampleGRUSeq2Seq(inp, out, weights, per_example_loss=True)
# First loss is scalar, the second one is a 1-dimensinal tensor.
self.assertEqual([], losses1[0].get_shape().as_list())
self.assertEqual([None], losses2[0].get_shape().as_list())
def init():
d_model = 512
d_k = 64
d_v = 64
sequence_length =6 #5
decoder_sent_length=6
h = 8
batch_size = 4*32
num_layer=6
# 2.set Q,K,V
vocab_size = 1000
embed_size = d_model
initializer = tf.random_normal_initializer(stddev=0.1)
Embedding = tf.get_variable("Embedding_d", shape=[vocab_size, embed_size], initializer=initializer)
decoder_input_x = tf.placeholder(tf.int32, [batch_size, decoder_sent_length], name="input_x") # [4,10]
print("1.decoder_input_x:", decoder_input_x)
decoder_input_embedding = tf.nn.embedding_lookup(Embedding, decoder_input_x) # [batch_size*sequence_length,embed_size]
#Q = embedded_words # [batch_size*sequence_length,embed_size]
#K_s = embedded_words # [batch_size*sequence_length,embed_size]
#K_v_encoder = tf.placeholder(tf.float32, [batch_size,decoder_sent_length, d_model], name="input_x") #sequence_length
Q = tf.placeholder(tf.float32, [batch_size,sequence_length, d_model], name="input_x")
K_s=decoder_input_embedding
K_v_encoder= tf.get_variable("v_variable",shape=[batch_size,decoder_sent_length, d_model],initializer=initializer) #tf.float32,
print("2.output from encoder:",K_v_encoder)
mask = get_mask(decoder_sent_length) #sequence_length
decoder = Decoder(d_model, d_k, d_v, sequence_length, h, batch_size, Q, K_s, K_v_encoder,decoder_sent_length,mask=mask,num_layer=num_layer)
return decoder,Q, K_s
def _run_rpn_proposal(self, all_anchors, rpn_cls_prob, config,
gt_boxes=None, rpn_bbox_pred=None):
"""
Define one of gt_boxes or rpn_bbox_pred.
If using gt_boxes, the correct rpn_bbox_pred for those gt_boxes will
be used.
"""
feed_dict = {}
rpn_cls_prob_tf = tf.placeholder(
tf.float32, shape=(all_anchors.shape[0], 2))
feed_dict[rpn_cls_prob_tf] = rpn_cls_prob
im_size_tf = tf.placeholder(tf.float32, shape=(2,))
feed_dict[im_size_tf] = self.im_size
all_anchors_tf = tf.placeholder(tf.float32, shape=all_anchors.shape)
feed_dict[all_anchors_tf] = all_anchors
if rpn_bbox_pred is None and gt_boxes is not None:
# Here we encode 'all_anchors' and 'gt_boxes' to get corrects
# predictions that RPNProposal can decodes.
rpn_bbox_pred_tf = encode(all_anchors, gt_boxes)
else:
rpn_bbox_pred_tf = tf.placeholder(
tf.float32, shape=rpn_bbox_pred.shape
)
feed_dict[rpn_bbox_pred_tf] = rpn_bbox_pred
model = RPNProposal(all_anchors.shape[0], config, debug=True)
results = model(
rpn_cls_prob_tf, rpn_bbox_pred_tf, all_anchors_tf, im_size_tf)
with self.test_session() as sess:
results = sess.run(results, feed_dict=feed_dict)
return results
def create_model(self):
print "Setting up model",
sys.stdout.flush()
# placeholders for data + targets
self._input_data = tf.placeholder(tf.int32, shape=(self.batch_size, self.num_steps))
self._targets = tf.placeholder(tf.int32, [self.batch_size, self.num_steps])
# set up lookup function
self.embedding = tf.constant(self.saved_embedding,name="embedding")
self.inputs = tf.nn.embedding_lookup(self.embedding, self._input_data)
# lstm model
self.lstm_cell = rnn_cell.BasicLSTMCell(self.lstm_size)
self.cell = rnn_cell.MultiRNNCell([self.lstm_cell] * self.num_layers)
self._initial_state = self.cell.zero_state(self.batch_size, tf.float32)
from tensorflow.models.rnn import rnn
self.inputs = [tf.squeeze(input_, [1]) for input_ in tf.split(1, self.num_steps, self.inputs)]
self.outputs, self.states = rnn.rnn(self.cell, self.inputs, initial_state=self._initial_state)
self.output = tf.reshape(tf.concat(1, self.outputs), [-1, self.lstm_size])
self.softmax_w = tf.get_variable("softmax_w", [self.lstm_size, self.vocab_size])
self.softmax_b = tf.get_variable("softmax_b", [self.vocab_size])
self.logits = tf.matmul(self.output, self.softmax_w) + self.softmax_b
#print "self.states.get_shape():",self.states.get_shape()
#print "tf.shape(self.states)",tf.shape(self.states)
self._final_state = self.states
self.saver = tf.train.Saver()
#delete data to save memory if network is used for sampling only
if self.only_for_sampling:
del self.data
print "done"
def modelOpt(self):
bn_params = {'decay': 0.999, 'center': True, 'scale': True, 'epsilon': 0.001, 'updates_collections': None,
'is_training': self.is_training}
self.global_step = tf.Variable(0, trainable=False)
self.learning_rate = layers.decayLearningRate(LEARNING_RATE, self.global_step, DECAY_STEPS, DECAY_RATE)
self.towerLogits = tf.placeholder(dtype=tf.float32, shape=[None, self.dataset.frames, 64])
self.Y = tf.placeholder(dtype=tf.int32, shape=[None])
self.Yoh = layers.toOneHot(self.Y, self.dataset.num_classes)
net_shape = self.towerLogits.get_shape()
net = tf.reshape(self.towerLogits, [BATCH_SIZE, int(net_shape[1]) * int(net_shape[2])])
layer_num = 1
for fully_connected_num in [64]:
net = layers.fc(net, fully_connected_num, name='temporal_FC{}'.format(layer_num),
weights_regularizer=layers.l2_regularizer(REGULARIZER_SCALE),
normalizer_fn=layers.batchNormalization, normalizer_params=bn_params)
layer_num += 1
self.logits = layers.fc(net, self.dataset.num_classes, activation_fn=None, name='logits')
self.preds = layers.softmax(self.logits)
cross_entropy_loss = layers.reduce_mean(layers.softmax_cross_entropy(logits=self.logits, labels=self.Yoh))
regularization_loss = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
self.loss = cross_entropy_loss + REGULARIZER_SCALE * tf.reduce_sum(regularization_loss)
self.opt = layers.adam(self.learning_rate)
self.train_op = self.opt.minimize(self.loss, global_step=self.global_step)
self.accuracy, self.precision, self.recall = self.createSummaries(self.Yoh, self.preds, self.loss,
self.learning_rate)
def runNN (train_x, train_y, test_x, test_y, numHidden):
print "NN({})".format(numHidden)
session = tf.InteractiveSession()
x = tf.placeholder("float", shape=[None, train_x.shape[1]])
y_ = tf.placeholder("float", shape=[None, 2])
W1 = tf.Variable(tf.truncated_normal([train_x.shape[1],numHidden], stddev=0.01))
b1 = tf.Variable(tf.truncated_normal([numHidden], stddev=0.01))
W2 = tf.Variable(tf.truncated_normal([numHidden,2], stddev=0.01))
b2 = tf.Variable(tf.truncated_normal([2], stddev=0.01))
z = tf.nn.relu(tf.matmul(x,W1) + b1)
y = tf.nn.softmax(tf.matmul(z,W2) + b2)
cross_entropy = -tf.reduce_sum(y_*tf.log(tf.clip_by_value(y,1e-10,1.0)))
#cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.MomentumOptimizer(learning_rate=.001, momentum=0.1).minimize(cross_entropy)
#train_step = tf.train.AdamOptimizer(learning_rate=.01).minimize(cross_entropy)
session.run(tf.initialize_all_variables())
for i in range(NUM_EPOCHS):
offset = i*BATCH_SIZE % (train_x.shape[0] - BATCH_SIZE)
train_step.run({x: train_x[offset:offset+BATCH_SIZE, :], y_: makeLabels(train_y[offset:offset+BATCH_SIZE])})
if i % 100 == 0:
util.showProgress(cross_entropy, x, y, y_, test_x, test_y)
session.close()
def __init__(self, model_path = "models", threshold = [0.6, 0.7, 0.7], factor = 0.709, scale_factor = 1):
'''
:param face_rec_sess: FaceRecSession
:param threshold: detection threshold
:param factor: default 0.709 image pyramid -- magic number
:param model_path:
'''
self.threshold = threshold
self.factor = factor
self.scale_factor = scale_factor;
with tf.Graph().as_default(), tf.device('/cpu:0'):
print("Loading Face detection model")
self.sess = tf.Session()
if not model_path:
model_path, _ = os.path.split(os.path.realpath(__file__))
with tf.variable_scope('pnet'):
data = tf.placeholder(tf.float32, (None, None, None, 3), 'input')
pnet = PNet({'data': data})
pnet.load(os.path.join(model_path, 'det1.npy'), self.sess)
with tf.variable_scope('rnet'):
data = tf.placeholder(tf.float32, (None, 24, 24, 3), 'input')
rnet = RNet({'data': data})
rnet.load(os.path.join(model_path, 'det2.npy'), self.sess)
with tf.variable_scope('onet'):
data = tf.placeholder(tf.float32, (None, 48, 48, 3), 'input')
onet = ONet({'data': data})
onet.load(os.path.join(model_path, 'det3.npy'), self.sess)
self.pnet = lambda img: self.sess.run(('pnet/conv4-2/BiasAdd:0', 'pnet/prob1:0'), feed_dict={'pnet/input:0': img})
self.rnet = lambda img: self.sess.run(('rnet/conv5-2/conv5-2:0', 'rnet/prob1:0'), feed_dict={'rnet/input:0': img})
self.onet = lambda img: self.sess.run(('onet/conv6-2/conv6-2:0', 'onet/conv6-3/conv6-3:0', 'onet/prob1:0'),
feed_dict={'onet/input:0': img})
print("Face detection model loaded")
def testPartialShapes(self):
np.random.seed(1618)
# Input shape is unknown.
reduction_axes = [1, 2]
c_unknown = tf.placeholder(tf.float32)
s_unknown = tf.reduce_sum(c_unknown, reduction_axes)
self.assertEqual(tensor_shape.unknown_shape(), s_unknown.get_shape())
np_input = np.random.randn(3, 3, 3)
self._compareAll(np_input, reduction_axes, {c_unknown: np_input})
# Input shape only has known rank.
c_known_rank = tf.placeholder(tf.float32)
c_known_rank.set_shape(tensor_shape.unknown_shape(ndims=3))
s_known_rank = tf.reduce_sum(c_known_rank, reduction_axes, keep_dims=True)
self.assertEqual(3, s_known_rank.get_shape().ndims)
np_input = np.random.randn(3, 3, 3)
self._compareAll(np_input, reduction_axes, {c_known_rank: np_input})
# Reduction indices are unknown.
unknown_indices = tf.placeholder(tf.int32)
c_unknown_indices = tf.constant([[10.0], [20.0]])
s_unknown_indices = tf.reduce_sum(c_unknown_indices, unknown_indices,
keep_dims=False)
self.assertEqual(tensor_shape.unknown_shape(),
s_unknown_indices.get_shape())
s_unknown_indices_keep = tf.reduce_sum(c_unknown_indices, unknown_indices,
keep_dims=True)
self.assertEqual(2, s_unknown_indices_keep.get_shape().ndims)
def recover_feeling(checkpoint):
#设置变量
in_sentence = tf.placeholder(tf.float32, [None, 140])
weight = tf.Variable(tf.zeros([140, 6]))
biases = tf.Variable(tf.zeros([6]))
global_step = tf.Variable(0, name='global_step', trainable=False)
#y = softmax(Wx + b)
y = tf.nn.softmax(tf.matmul(in_sentence, weight) + biases)
y_ = tf.placeholder(tf.float32, [None, 6])
sess = tf.InteractiveSession()
#恢复模型.
saver = tf.train.Saver()
saver.restore(sess, checkpoint)
#读取模型完毕，加载词表
vocab, tmp_vocab = read_vocabulary('data/emotion/vocabulary.txt')
#把一些句子转化为vec的array
vec_list1 = convert_sentence_to_vec_list('你今天感觉怎么样', vocab, 140)
vec_list2 = convert_sentence_to_vec_list('高兴啊', vocab, 140)
vec_list_final = [np.array(vec_list1), np.array(vec_list2)]
print (vec_list_final)
#交给sess进行检查
data = np.array(vec_list_final)
result = sess.run(y, feed_dict={in_sentence: data})
print (result)
def buildSpImageConverter(channelOrder, img_dtype):
"""
Convert a imageIO byte encoded image into a image tensor suitable as input to ConvNets
The name of the input must be a subset of those specified in `image.imageIO.imageSchema`.
:param img_dtype: the type of data the underlying image bytes represent
"""
with IsolatedSession() as issn:
# Flat image data -> image dimensions
# This has to conform to `imageIO.imageSchema`
height = tf.placeholder(tf.int32, [], name="height")
width = tf.placeholder(tf.int32, [], name="width")
num_channels = tf.placeholder(tf.int32, [], name="nChannels")
image_buffer = tf.placeholder(tf.string, [], name="data")
# The image is packed into bytes with height as leading dimension
# This is the default behavior of Python Image Library
shape = tf.reshape(tf.stack([height, width, num_channels], axis=0),
shape=(3,), name='shape')
if img_dtype == 'uint8':
image_uint8 = tf.decode_raw(image_buffer, tf.uint8, name="decode_raw")
image_float = tf.to_float(image_uint8)
elif img_dtype == 'float32':
image_float = tf.decode_raw(image_buffer, tf.float32, name="decode_raw")
else:
raise ValueError('''unsupported image data type "%s", currently only know how to
handle uint8 and float32''' % img_dtype)
image_reshaped = tf.reshape(image_float, shape, name="reshaped")
image_reshaped = imageIO.fixColorChannelOrdering(channelOrder, image_reshaped)
image_input = tf.expand_dims(image_reshaped, 0, name="image_input")
gfn = issn.asGraphFunction([height, width, image_buffer, num_channels], [image_input])
return gfn
def main():
sess = tf.Session()
# 2進数3ビットから10進数
x = tf.placeholder(tf.float32, [None, 3])
w = tf.Variable(tf.zeros([3, 8]))
b = tf.Variable(tf.zeros([8]))
y = tf.nn.softmax(tf.matmul(x, w) + b)
y_ = tf.placeholder(tf.float32, [None, 8])
cross_entropy = -tf.reduce_sum(y_ * tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.05).minimize(cross_entropy)
sess.run(tf.initialize_all_variables())
for i in range(1000):
train_step.run({x: [[0, 0, 0]], y_: [[1, 0, 0, 0, 0, 0, 0, 0]]}, session=sess)
train_step.run({x: [[1, 0, 0]], y_: [[0, 1, 0, 0, 0, 0, 0, 0]]}, session=sess)
train_step.run({x: [[0, 1, 0]], y_: [[0, 0, 1, 0, 0, 0, 0, 0]]}, session=sess)
train_step.run({x: [[1, 1, 0]], y_: [[0, 0, 0, 1, 0, 0, 0, 0]]}, session=sess)
train_step.run({x: [[0, 0, 1]], y_: [[0, 0, 0, 0, 1, 0, 0, 0]]}, session=sess)
train_step.run({x: [[1, 0, 1]], y_: [[0, 0, 0, 0, 0, 1, 0, 0]]}, session=sess)
train_step.run({x: [[0, 1, 1]], y_: [[0, 0, 0, 0, 0, 0, 1, 0]]}, session=sess)
train_step.run({x: [[1, 1, 1]], y_: [[0, 0, 0, 0, 0, 0, 0, 1]]}, session=sess)
## 1に近い予測があってるか 平均
#correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
#accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print(sess.run(y, feed_dict={x: [[0, 0, 0]]}))
print(sess.run(y, feed_dict={x: [[1, 0, 0]]}))
print(sess.run(y, feed_dict={x: [[0, 1, 0]]}))
print(sess.run(y, feed_dict={x: [[1, 1, 0]]}))
print(sess.run(y, feed_dict={x: [[0, 0, 1]]}))
return 0
def __init__(self, data_dictionary):
# init some parameters
self.replay_buffer = []
self.time_step = 0
self.epsilon = INITIAL_EPSILON
self.state_dim = data_dictionary["input"]
self.action_dim = data_dictionary["action"]
self.n_input = self.state_dim
self.state_input = tf.placeholder("float", [None, self.n_input])
self.y_input = tf.placeholder("float",[None, self.action_dim])
self.create_pg_network(data_dictionary)
self.create_training_method()
# Init session
self.session = tf.InteractiveSession()
self.session.run(tf.initialize_all_variables())
# loading networks
self.saver = tf.train.Saver()
checkpoint = tf.train.get_checkpoint_state("pg_saved_networks")
if checkpoint and checkpoint.model_checkpoint_path:
self.saver.restore(self.session, checkpoint.model_checkpoint_path)
print "Successfully loaded:", checkpoint.model_checkpoint_path
else:
print "Could not find old network weights"
global summary_writer
summary_writer = tf.train.SummaryWriter('logs',graph=self.session.graph)
def create_network(self, name):
with tf.variable_scope(name) as scope:
inputs = tf.placeholder(fl32, [None, self.state_dim], 'inputs')
actions = tf.placeholder(fl32, [None, self.action_dim], 'actions')
with slim.arg_scope(
[slim.fully_connected],
activation_fn=relu,
weights_initializer=uniform,
weights_regularizer=None
):
net = tf.concat(1, [inputs, actions])
net = slim.fully_connected(net, 1024)
res = net = slim.fully_connected(net, 128)
net = slim.fully_connected(net, 256)
net = slim.fully_connected(net, 128, activation_fn=None)
net = relu(net+res)
res = net = slim.fully_connected(net, 128)
net = slim.fully_connected(net, 256)
net = slim.fully_connected(net, 128, activation_fn=None)
net = relu(net+res)
res = net = slim.fully_connected(net, 128)
net = slim.fully_connected(net, 256)
net = slim.fully_connected(net, 128, activation_fn=None)
net = relu(net+res)
out = slim.fully_connected(net, 1, activation_fn=None)
return (inputs, actions, out, scope.name)
def testMixtureOfEnqueueAndEnqueueMany(self):
with self.test_session() as sess:
q = tf.PaddingFIFOQueue(10, tf.int32, shapes=((),))
enqueue_placeholder = tf.placeholder(tf.int32, shape=())
enqueue_op = q.enqueue((enqueue_placeholder,))
enqueuemany_placeholder = tf.placeholder(
tf.int32, shape=(None,))
enqueuemany_op = q.enqueue_many((enqueuemany_placeholder,))
dequeued_t = q.dequeue()
close_op = q.close()
def dequeue():
for i in xrange(250):
self.assertEqual(i, sess.run(dequeued_t))
dequeue_thread = self.checkedThread(target=dequeue)
dequeue_thread.start()
elements_enqueued = 0
while elements_enqueued < 250:
# With equal probability, run Enqueue or enqueue_many.
if random.random() > 0.5:
enqueue_op.run({enqueue_placeholder: elements_enqueued})
elements_enqueued += 1
else:
count = random.randint(0, min(20, 250 - elements_enqueued))
range_to_enqueue = np.arange(elements_enqueued,
elements_enqueued + count,
dtype=np.int32)
enqueuemany_op.run({enqueuemany_placeholder: range_to_enqueue})
elements_enqueued += count
close_op.run()
dequeue_thread.join()
self.assertEqual(0, q.size().eval())
def build_graph(self, nn_im_w, nn_im_h, num_colour_channels=3, weights=None, biases=None):
num_outputs = 1 #ofc
self.nn_im_w = nn_im_w
self.nn_im_h = nn_im_h
if weights is None:
weights = [None, None, None, None, None]
if biases is None:
biases = [None, None, None, None, None]
with tf.device('/cpu:0'):
# Placeholder variables for the input image and output images
self.x = tf.placeholder(tf.float32, shape=[None, nn_im_w*nn_im_h*3])
self.y_ = tf.placeholder(tf.float32, shape=[None, num_outputs])
self.threshold = tf.placeholder(tf.float32)
# Build the convolutional and pooling layers
conv1_output_channels = 32
conv2_output_channels = 16
conv3_output_channels = 8
conv_layer_1_input = tf.reshape(self.x, [-1, nn_im_h, nn_im_w, num_colour_channels]) #The resized input image
self.build_conv_layer(conv_layer_1_input, num_colour_channels, conv1_output_channels, initial_weights=weights[0], initial_biases=biases[0]) # layer 1
self.build_conv_layer(self.layers[0][0], conv1_output_channels, conv2_output_channels, initial_weights=weights[1], initial_biases=biases[1])# layer 2
self.build_conv_layer(self.layers[1][0], conv2_output_channels, conv3_output_channels, initial_weights=weights[2], initial_biases=biases[2])# layer 3
# Build the fully connected layer
convnet_output_w = nn_im_w//8
convnet_output_h = nn_im_h//8
fully_connected_layer_input = tf.reshape(self.layers[2][0], [-1, convnet_output_w * convnet_output_h * conv3_output_channels])
self.build_fully_connected_layer(fully_connected_layer_input, convnet_output_w, convnet_output_h, conv3_output_channels, initial_weights=weights[3], initial_biases=biases[3])
# The dropout stage and readout layer
self.keep_prob, self.h_drop = self.dropout(self.layers[3][0])
self.y_conv,_,_ = self.build_readout_layer(self.h_drop, num_outputs, initial_weights=weights[4], initial_biases=biases[4])
self.mean_error = tf.sqrt(tf.reduce_mean(tf.square(self.y_ - self.y_conv)))
self.train_step = tf.train.AdamOptimizer(1e-4).minimize(self.mean_error)
self.accuracy = (1.0 - tf.reduce_mean(tf.abs(self.y_ - tf.round(self.y_conv))))
positive_examples = tf.greater_equal(self.y_, 0.5)
negative_examples = tf.logical_not(positive_examples)
positive_classifications = tf.greater_equal(self.y_conv, self.threshold)
negative_classifications = tf.logical_not(positive_classifications)
self.true_positive = tf.reduce_sum(tf.cast(tf.logical_and(positive_examples, positive_classifications),tf.int32)) # count the examples that are positive and classified as positive
self.false_positive = tf.reduce_sum(tf.cast(tf.logical_and(negative_examples, positive_classifications),tf.int32)) # count the examples that are negative but classified as positive
self.true_negative = tf.reduce_sum(tf.cast(tf.logical_and(negative_examples, negative_classifications),tf.int32)) # count the examples that are negative and classified as negative
self.false_negative = tf.reduce_sum(tf.cast(tf.logical_and(positive_examples, negative_classifications),tf.int32)) # count the examples that are positive but classified as negative
self.positive_count = tf.reduce_sum(tf.cast(positive_examples, tf.int32)) # count the examples that are positive
self.negative_count = tf.reduce_sum(tf.cast(negative_examples, tf.int32)) # count the examples that are negative
self.confusion_matrix = tf.reshape(tf.pack([self.true_positive, self.false_positive, self.false_negative, self.true_negative]), [2,2])
self.sess.run(tf.initialize_all_variables())
def main(output_dir, summaries_every, num_steps):
graph = tf.Graph()
with graph.as_default():
features = tf.placeholder(tf.float32, shape=[4, 2])
labels = tf.placeholder(tf.int32, shape=[4])
train_op, loss, gs, update_acc = make_graph(features, labels)
init = tf.global_variables_initializer()
init_local = tf.local_variables_initializer()
summary_op = tf.summary.merge_all()
writer = tf.summary.FileWriter(output_dir, graph=graph, flush_secs=1)
with tf.Session(graph=graph) as sess:
init.run()
init_local.run()
step = 0
xy = np.array([
[True, False],
[True, True],
[False, False],
[False, True]
], dtype=np.float)
y_ = np.array([True, False, False, True], dtype=np.int32)
while step < num_steps:
_, _, step, loss_value, summaries = sess.run(
[train_op, update_acc, gs, loss, summary_op],
feed_dict={features: xy, labels: y_}
)
if step % summaries_every == 0:
writer.add_summary(summaries, global_step=step)
y_test = data_test[:, 0] # Number of stocks in training data n_stocks = X_train.shape[1] # Neurons n_neurons_1 = 1024 n_neurons_2 = 512 n_neurons_3 = 256 n_neurons_4 = 128 # Session net = tf.InteractiveSession() # Placeholder X = tf.placeholder(dtype=tf.float32, shape=[None, n_stocks]) Y = tf.placeholder(dtype=tf.float32, shape=[None]) # Initializers sigma = 1 weight_initializer = tf.variance_scaling_initializer(mode="fan_avg", distribution="uniform", scale=sigma) bias_initializer = tf.zeros_initializer() # Hidden weights W_hidden_1 = tf.Variable(weight_initializer([n_stocks, n_neurons_1])) bias_hidden_1 = tf.Variable(bias_initializer([n_neurons_1])) W_hidden_2 = tf.Variable(weight_initializer([n_neurons_1, n_neurons_2])) bias_hidden_2 = tf.Variable(bias_initializer([n_neurons_2])) W_hidden_3 = tf.Variable(weight_initializer([n_neurons_2, n_neurons_3])) bias_hidden_3 = tf.Variable(bias_initializer([n_neurons_3])) W_hidden_4 = tf.Variable(weight_initializer([n_neurons_3, n_neurons_4]))
print("Train data size -> input: {}, output: {}".format(X_train.shape, y_train.shape))
print("Test data size: -> input: {}, output: {}".format(X_test.shape, y_test.shape))
# Normalize features
# Test data is *not* used when calculating the mean and std
mean = X_train.mean(axis=0)
std = X_train.std(axis=0)
X_train = (X_train - mean) / std
X_test = (X_test - mean) / std
# Create validation data
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2)
# Build the model
# Placeholders for inputs (x) and outputs(y)
x = tf.placeholder(tf.float32, shape=[None, num_features], name='X')
y = tf.placeholder(tf.float32, shape=[None], name='Y')
def DenseLayer(inputs, num_units, layer_name, activation=None):
input_dim = inputs.get_shape().as_list()[-1]
with tf.variable_scope(layer_name):
W = tf.get_variable('W',
dtype=tf.float32,
shape=[input_dim, num_units],
initializer=tf.truncated_normal_initializer(stddev=0.01))
b = tf.get_variable('b',
dtype=tf.float32,
initializer=tf.constant(0., shape=[num_units], dtype=tf.float32))
logits = tf.matmul(inputs, W) + b
if activation:
def __init__(self,
input_size,
summary_dir,
log_dir,
model_name="model.ckpt"):
# 和保存模型相关的参数
self.log_dir = Tools.new_dir(log_dir)
self.model_name = model_name
self.checkpoint_path = os.path.join(self.log_dir, self.model_name)
# 和数据相关的参数
self.input_size = input_size
self.num_classes = 21
# 网络
self.image_placeholder = tf.placeholder(tf.float32,
shape=(None,
self.input_size[0],
self.input_size[1], 3))
# 网络
self.net = BAISNet(self.image_placeholder,
False,
num_classes=self.num_classes)
self.segments, self.features = self.net.build()
self.pred_segment = tf.cast(tf.argmax(self.segments[0], axis=-1),
dtype=tf.uint8)
with tf.name_scope("image"):
tf.summary.image("input", self.image_placeholder)
# segment
for segment_index, segment in enumerate(self.segments):
segment = tf.cast(tf.argmax(segment, axis=-1), dtype=tf.uint8)
tf.summary.image(
"predict-{}".format(segment_index),
tf.expand_dims(segment * 255 // self.num_classes, axis=-1))
pass
pass
for key in list(self.features.keys()):
with tf.name_scope(key):
for feature_index, feature in enumerate(self.features[key]):
feature_split = tf.split(feature,
num_or_size_splits=int(
feature.shape[-1]),
axis=-1)
for feature_one_index, feature_one in enumerate(
feature_split):
tf.summary.image(
"{}-{}".format(feature_index, feature_one_index),
feature_one)
pass
pass
self.summary_op = tf.summary.merge_all()
# sess 和 saver
self.sess = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions(
allow_growth=True)))
self.sess.run(tf.global_variables_initializer())
self.saver = tf.train.Saver(var_list=tf.global_variables(),
max_to_keep=10)
self.summary_writer = tf.summary.FileWriter(summary_dir,
self.sess.graph)
pass
def run(conf, data):
sess = tf.Session()
print 'Model Defining...'
truths = tf.placeholder(
tf.float32, [None, conf.img_height, conf.img_width, conf.channel])
compres = tf.placeholder(
tf.float32, [None, conf.img_height, conf.img_width, conf.channel])
model = ARCNN(conf, truths, compres)
trainer = tf.train.RMSPropOptimizer(1e-3)
gradients = trainer.compute_gradients(model.loss)
clipped_gradients = [(tf.clip_by_value(_[0], -conf.grad_clip,
conf.grad_clip), _[1])
for _ in gradients]
optimizer = trainer.apply_gradients(clipped_gradients)
saver = tf.train.Saver()
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(conf.summary_path, sess.graph)
if os.path.exists(conf.ckpt_path):
ckpt = tf.train.get_checkpoint_state(conf.ckpt_path)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print 'Variables Restored.'
else:
sess.run(tf.global_variables_initializer())
print 'Variables Initialized.'
else:
sess.run(tf.global_variables_initializer())
print 'Variables Initialized.'
print 'Training Start.'
for i in range(conf.epochs):
for j in range(conf.num_batches):
batch_truth, batch_compres = data.train.next_batch()
data_dict = {compres: batch_compres, truths: batch_truth}
_, cost, ori_cost, summary = sess.run(
[optimizer, model.loss, model.original_loss, merged],
feed_dict=data_dict)
writer.add_summary(summary, i)
PSNR = 10.0 * math.log(1.0 / cost) / math.log(10.0)
print 'Epoch: %d, Loss: %f, Original Loss: %f, PSNR: %f' % (
i, cost, ori_cost, PSNR)
if (i + 100) % 1 == 0:
saver.save(sess, conf.ckpt_path + '/model.ckpt')
model.save(sess)
print 'Training completed.'
print 'Validating Start.'
# num_val_epochs = conf.num_val / conf.test_size + 1
num_val_epochs = 50
for i in range(num_val_epochs):
batch_truth, batch_compres = data.test.next_batch()
data_dict = {compres: batch_compres, truths: batch_truth}
cost, ori_cost = sess.run([model.loss, model.original_loss],
feed_dict=data_dict)
PSNR = 10.0 * math.log(1.0 / cost) / math.log(10.0)
print 'Epoch: %d, Loss: %f, Original Loss: %f, PSNR: %f' % (
i, cost, ori_cost, PSNR)
print 'Validating completed.'
height, width = image.shape
return_img = np.zeros([height, width, 3], np.uint8)
for i in range(height):
for j in range(width):
return_img[i, j, :] = cmap[image[i, j]]
return return_img
image_batch_0, image_batch, anno_batch, filename = input_data.read_batch(BATCH_SIZE, type=prediction_on)
with tf.name_scope("input"):
x = tf.placeholder(tf.float32, [BATCH_SIZE, HEIGHT, WIDTH, 3], name='x_input')
y = tf.placeholder(tf.int32, [BATCH_SIZE, HEIGHT, WIDTH], name='ground_truth')
_, logits = PSPNet.PSPNet(x, is_training=False, output_stride=8, pre_trained_model=PRETRAINED_MODEL_PATH)
with tf.name_scope('prediction_and_miou'):
prediction = tf.argmax(logits, axis=-1, name='predictions')
with tf.Session() as sess:
sess.run(tf.local_variables_initializer())
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
model_func = GNN
elif FLAGS.model == 'gcn_cheby': # not used
# support = chebyshev_polynomials(adj, FLAGS.max_degree)
num_supports = 1 + FLAGS.max_degree
model_func = GNN
elif FLAGS.model == 'dense': # not used
# support = [preprocess_adj(adj)]
num_supports = 1
model_func = MLP
else:
raise ValueError('Invalid argument for model: ' + str(FLAGS.model))
# Define placeholders
placeholders = {
'support': tf.placeholder(tf.float32, shape=(None, None, None)),
'features': tf.placeholder(tf.float32, shape=(None, None, FLAGS.input_dim)),
'mask': tf.placeholder(tf.float32, shape=(None, None, 1)),
'labels': tf.placeholder(tf.float32, shape=(None, train_y.shape[1])),
'dropout': tf.placeholder_with_default(0., shape=()),
'num_features_nonzero': tf.placeholder(tf.int32) # helper variable for sparse dropout
}
# label smoothing
# label_smoothing = 0.1
# num_classes = y_train.shape[1]
# y_train = (1.0 - label_smoothing) * y_train + label_smoothing / num_classes
# Create model
def train(steps,
drop_rate,
mc_samples,
lw_scaling,
data_path,
data_provider,
log_dir,
kernel_prior,
learning_rate,
beta1,
beta2,
epsilon,
snapshot_save_path,
snapshot_id):
with tf.Graph().as_default():
raw = tf.placeholder(tf.float32, shape=(1, 1, 268, 268))
# Make a reshape in format tf expects with batch as one dim:
raw_batched = tf.reshape(raw, (1,1,1,268,268))
f_out_batched = bunet(fmaps_in=raw_batched,
num_fmaps=12,
fmap_inc_factor=2,
downsample_factors=[[1,3,3],[1,3,3],[1,3,3]],
drop_rate=drop_rate,
kernel_prior=kernel_prior,
activation='relu')
logits = pre_sigmoid_split_conv_layer(f_out_batched,
drop_rate,
kernel_prior)
logits_with_noise = gaussian_noise_layer(logits, mc_samples)
f_out_shape_batched = f_out_batched.get_shape().as_list()
f_out_shape = f_out_shape_batched[1:] # strip batch dim
gt_affs_shape = f_out_shape
gt_affs_shape[0] = 2
gt_affs = tf.placeholder(tf.float32, shape=gt_affs_shape)
loss_weights = tf.placeholder(tf.float32, shape=gt_affs_shape)
aff_out_batched = tf.sigmoid(logits[:,0:2,:,:,:])
aff_out_with_noise = tf.sigmoid(logits_with_noise[0,:,:,:,:])
aff_out_batched_with_noise = tf.reshape(aff_out_with_noise, tf.shape(aff_out_batched))
sigma = logits[:,2:4,:,:,:]
aff_out_shape = aff_out_batched.get_shape().as_list()[1:]
loss = stochastic_loss_layer(logits_with_noise,
gt_affs,
loss_weights,
mc_samples)
global_step = tf.Variable(0, name='global_step', trainable=False)
train_step = optimize(loss,
global_step,
learning_rate,
beta1,
beta2,
epsilon)
summary = tf.summary.merge_all()
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.Session()
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
#sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
sess.run(init)
for step in range(steps):
start_time = time.time()
feed_dict = fill_feed_dict(data_provider_object=data_provider,
data_path=data_path,
raw_placeholder=raw,
gt_aff_placeholder=gt_affs,
loss_weight_placeholder=loss_weights,
loss_weight_scaling=lw_scaling,
step=step,
f_out_shape=f_out_shape,
group="train")
_, loss_value = sess.run([train_step, loss],
feed_dict=feed_dict)
duration = time.time() - start_time
if step%100 == 0:
print("Step %d: loss %.2f (%3f sec)" % (step, loss_value, duration))
summary_str = sess.run(summary, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
summary_writer.flush()
if (step + 1) % 1000 == 0 or (step + 1) == steps:
checkpoint_file = os.path.join(log_dir, "unet.ckpt")
saver.save(sess, checkpoint_file, global_step=step)
eval_correct = evaluation(affs=aff_out_batched,
affs_noise=aff_out_batched_with_noise,
sigma=sigma,
gt_affs=gt_affs,
threshold=0.1)
print("Training Data Eval:")
do_eval(sess,
eval_correct,
data_provider,
data_path,
raw,
gt_affs,
loss_weights,
lw_scaling,
aff_out_shape,
group="train",
snapshot_save_path=snapshot_save_path,
snapshot_id=snapshot_id)
"""
def build_model(self):
# Placeholdersn
if self.config.trainer.init_type == "normal":
self.init_kernel = tf.random_normal_initializer(mean=0.0,
stddev=0.02)
elif self.config.trainer.init_type == "xavier":
self.init_kernel = tf.contrib.layers.xavier_initializer(
uniform=False, seed=None, dtype=tf.float32)
self.is_training = tf.placeholder(tf.bool)
self.image_input = tf.placeholder(tf.float32,
shape=[None] +
self.config.trainer.image_dims,
name="x")
self.noise_tensor = tf.placeholder(
tf.float32,
shape=[None, self.config.trainer.noise_dim],
name="noise")
# Placeholders for the true and fake labels
self.true_labels = tf.placeholder(dtype=tf.float32, shape=[None, 1])
self.generated_labels = tf.placeholder(dtype=tf.float32,
shape=[None, 1])
self.real_noise = tf.placeholder(dtype=tf.float32,
shape=[None] +
self.config.trainer.image_dims,
name="real_noise")
self.fake_noise = tf.placeholder(dtype=tf.float32,
shape=[None] +
self.config.trainer.image_dims,
name="fake_noise")
# Building the Graph
self.logger.info("Building Graph")
with tf.variable_scope("ANOGAN"):
with tf.variable_scope("Generator_Model"):
self.img_gen = self.generator(
self.noise_tensor) + self.fake_noise
# Discriminator
with tf.variable_scope("Discriminator_Model"):
disc_real, inter_layer_real = self.discriminator(
self.image_input + self.real_noise)
disc_fake, inter_layer_fake = self.discriminator(self.img_gen)
# Losses of the training of Generator and Discriminator
with tf.variable_scope("Loss_Functions"):
with tf.name_scope("Discriminator_Loss"):
self.disc_loss_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.true_labels, logits=disc_real))
self.disc_loss_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=self.generated_labels, logits=disc_fake))
self.total_disc_loss = self.disc_loss_real + self.disc_loss_fake
with tf.name_scope("Generator_Loss"):
if self.config.trainer.flip_labels:
labels = tf.zeros_like(disc_fake)
else:
labels = tf.ones_like(disc_fake)
self.gen_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=disc_fake))
# Build the Optimizers
with tf.variable_scope("Optimizers"):
# Collect all the variables
all_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
# Generator Network Variables
self.generator_vars = [
v for v in all_variables
if v.name.startswith("ANOGAN/Generator_Model")
]
# Discriminator Network Variables
self.discriminator_vars = [
v for v in all_variables
if v.name.startswith("ANOGAN/Discriminator_Model")
]
# Create Training Operations
# Generator Network Operations
self.gen_update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS, scope="ANOGAN/Generator_Model")
# Discriminator Network Operations
self.disc_update_ops = tf.get_collection(
tf.GraphKeys.UPDATE_OPS, scope="ANOGAN/Discriminator_Model")
# Initialization of Optimizers
self.generator_optimizer = tf.train.AdamOptimizer(
self.config.trainer.generator_l_rate,
beta1=self.config.trainer.optimizer_adam_beta1,
beta2=self.config.trainer.optimizer_adam_beta2,
)
self.discriminator_optimizer = tf.train.AdamOptimizer(
self.config.trainer.discriminator_l_rate,
beta1=self.config.trainer.optimizer_adam_beta1,
beta2=self.config.trainer.optimizer_adam_beta2,
)
with tf.control_dependencies(self.gen_update_ops):
self.train_gen = self.generator_optimizer.minimize(
self.gen_loss,
global_step=self.global_step_tensor,
var_list=self.generator_vars)
with tf.control_dependencies(self.disc_update_ops):
self.train_disc = self.discriminator_optimizer.minimize(
self.total_disc_loss, var_list=self.discriminator_vars)
def train_op_with_ema_dependency(vars, op):
ema = tf.train.ExponentialMovingAverage(
decay=self.config.trainer.ema_decay)
maintain_averages_op = ema.apply(vars)
with tf.control_dependencies([op]):
train_op = tf.group(maintain_averages_op)
return train_op, ema
self.train_gen_op, self.gen_ema = train_op_with_ema_dependency(
self.generator_vars, self.train_gen)
self.train_dis_op, self.dis_ema = train_op_with_ema_dependency(
self.discriminator_vars, self.train_disc)
with tf.variable_scope("Latent_variable"):
self.z_optim = tf.get_variable(
name="z_optim",
shape=[
self.config.data_loader.test_batch,
self.config.trainer.noise_dim
],
initializer=tf.truncated_normal_initializer(),
)
reinit_z = self.z_optim.initializer
with tf.variable_scope("ANOGAN"):
with tf.variable_scope("Generator_Model"):
self.x_gen_ema = self.generator(self.noise_tensor,
getter=sn.get_getter(
self.gen_ema))
self.rec_gen_ema = self.generator(self.z_optim,
getter=sn.get_getter(
self.gen_ema))
# Pass real and fake images into discriminator separately
with tf.variable_scope("Discriminator_Model"):
real_d_ema, inter_layer_real_ema = self.discriminator(
self.image_input, getter=sn.get_getter(self.dis_ema))
fake_d_ema, inter_layer_fake_ema = self.discriminator(
self.rec_gen_ema, getter=sn.get_getter(self.dis_ema))
with tf.variable_scope("Testing"):
with tf.variable_scope("Reconstruction_Loss"):
delta = self.image_input - self.rec_gen_ema
delta_flat = tf.layers.Flatten()(delta)
self.reconstruction_score_1 = tf.norm(
delta_flat,
ord=1,
axis=1,
keepdims=False,
name="epsilon",
)
self.reconstruction_score_2 = tf.norm(
delta_flat,
ord=2,
axis=1,
keepdims=False,
name="epsilon",
)
with tf.variable_scope("Discriminator_Scores"):
if self.config.trainer.loss_method == "c_entropy":
dis_score = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(fake_d_ema), logits=fake_d_ema)
elif self.config.trainer.loss_method == "fm":
fm = inter_layer_real_ema - inter_layer_fake_ema
fm = tf.layers.Flatten()(fm)
dis_score = tf.norm(fm,
ord=self.config.trainer.degree,
axis=1,
keepdims=False,
name="d_loss")
self.dis_score = tf.squeeze(dis_score)
with tf.variable_scope("Score"):
self.loss_invert_1 = (
self.config.trainer.weight * self.reconstruction_score_1 +
(1 - self.config.trainer.weight) * self.dis_score)
self.loss_invert_2 = (
self.config.trainer.weight * self.reconstruction_score_2 +
(1 - self.config.trainer.weight) * self.dis_score)
self.rec_error_valid = tf.reduce_mean(self.gen_loss)
with tf.variable_scope("Test_Learning_Rate"):
step_lr = tf.Variable(0, trainable=False)
learning_rate_invert = 0.001
reinit_lr = tf.variables_initializer(
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="Test_Learning_Rate"))
with tf.name_scope("Test_Optimizer"):
self.invert_op = tf.train.AdamOptimizer(
learning_rate_invert).minimize(self.loss_invert_1,
global_step=step_lr,
var_list=[self.z_optim],
name="optimizer")
reinit_optim = tf.variables_initializer(
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="Test_Optimizer"))
self.reinit_test_graph_op = [reinit_z, reinit_lr, reinit_optim]
with tf.name_scope("Scores"):
self.list_scores = self.loss_invert_1
if self.config.log.enable_summary:
with tf.name_scope("Training_Summary"):
with tf.name_scope("Dis_Summary"):
tf.summary.scalar("Real_Discriminator_Loss",
self.disc_loss_real, ["dis"])
tf.summary.scalar("Fake_Discriminator_Loss",
self.disc_loss_fake, ["dis"])
tf.summary.scalar("Discriminator_Loss",
self.total_disc_loss, ["dis"])
with tf.name_scope("Gen_Summary"):
tf.summary.scalar("Loss_Generator", self.gen_loss, ["gen"])
with tf.name_scope("Img_Summary"):
heatmap_pl_latent = tf.placeholder(tf.float32,
shape=(1, 480, 640, 3),
name="heatmap_pl_latent")
sum_op_latent = tf.summary.image("heatmap_latent",
heatmap_pl_latent)
with tf.name_scope("Validation_Summary"):
tf.summary.scalar("valid", self.rec_error_valid, ["v"])
with tf.name_scope("image_summary"):
tf.summary.image("reconstruct", self.img_gen, 3, ["image"])
tf.summary.image("input_images", self.image_input, 3,
["image"])
tf.summary.image("reconstruct", self.rec_gen_ema, 3,
["image_2"])
tf.summary.image("input_images", self.image_input, 3,
["image_2"])
self.sum_op_dis = tf.summary.merge_all("dis")
self.sum_op_gen = tf.summary.merge_all("gen")
self.sum_op = tf.summary.merge([self.sum_op_dis, self.sum_op_gen])
self.sum_op_im = tf.summary.merge_all("image")
self.sum_op_im_test = tf.summary.merge_all("image_2")
self.sum_op_valid = tf.summary.merge_all("v")
import pandas as pd
import tensorflow as tf
# Iris Dataset
#dataset = pd.read_csv('iris.csv',header= None)
x_cordinates = pd.Series(data=[11, 1, 2, 1, 8, 9, 10])
y_cordinates = pd.Series(data=[11, 1, 1, 3, 8, 10, 9])
group = pd.Series(data=['b', 'a', 'a', 'a', 'b', 'b', 'b'])
dataset = pd.concat([x_cordinates, y_cordinates, group], axis=1)
test_point = (0, 1)
# K value
k = 3
test_features = tf.placeholder(tf.float32)
features = tf.placeholder(tf.float32)
euclidain_distance = tf.sqrt(
tf.reduce_sum(tf.square(tf.subtract(features, test_features))))
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run(init)
nearest_neighbors = []
for index, row in dataset.iterrows():
feed_dict = {
features: np.array([row[0], row[1]]),
test_features: np.array(test_point)
}
dist = sess.run(euclidain_distance, feed_dict=feed_dict)
nearest_neighbors.append([dist, row[2]])