def restore(self, checkpoint_dir, recurrent_part_only=False):
session = tf.Session()
if recurrent_part_only:
var_list = tf.trainable_variables(scope='lstm')
else:
var_list = None
saver = Saver(var_list=var_list)
checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
saver.restore(session, checkpoint)
return session
def captcha2text(image_list, height=CAPTCHA_HEIGHT, width=CAPTCHA_WIDTH):
if not isdir('./model'):
print('Model directory does not exists.')
return
x = placeholder(float32, [None, height * width])
keep_prob = placeholder(float32)
y_conv = cnn_graph(x, keep_prob, (height, width))
saver = Saver()
with Session() as sess:
saver.restore(sess, latest_checkpoint('./model/'))
predict = argmax(reshape(
y_conv, [-1, CAPTCHA_LEN, len(CAPTCHA_LIST)]), 2)
vector_list = sess.run(predict,
feed_dict={
x: image_list,
keep_prob: 1
})
vector_list = vector_list.tolist()
text_list = [vec2text(vector) for vector in vector_list]
return text_list
with Graph().as_default() as g:
X = placeholder("float32", shape=[4, 2], name='X')
Y = placeholder("float32", shape=[4, 1], name='Y')
W = get_variable(shape=[2, 2], name='W')
w = get_variable(shape=[2, 1], name='w')
c = get_variable(shape=[4, 2], name='c')
b = get_variable(shape=[4, 1], name='b')
with name_scope("hidden_layer") as scope:
h = nn.relu(matmul(X, W) + c)
with name_scope("output") as scope:
y_estimated = sigmoid(matmul(h, w) + b)
with name_scope("loss") as scope:
loss = reduce_mean(square(y_estimated - Y))
with Session() as sess:
saver = Saver()
saver.restore(sess, "/tmp/model.ckpt")
print("Model restored.")
print(
'loss:',
sess.run(
loss, {
X: array([[0, 0], [0, 1], [1, 0], [1, 1]], "float32"),
Y: array([[0], [1], [1], [0]], "float32")
}))
class PpoGraph:
"""
Proximal Policy Implementation in tensorflow. https://arxiv.org/abs/1707.06347 ("Proximal Policy Optimization Algorithms", J. Schulman et al, 2017)
This class encapsulates all tensorflow interactions
"""
def __init__(self, observation_size, net_arch, initializer, activation,
clip_range, value_coef, entropy_coef, learning_rate,
pre_training_learning_rate, action_bounds, policy):
"""
:param observation_size:
:param net_arch:
:param initializer:
:param activation:
:param clip_range:
:param value_coef:
:param entropy_coef:
:param learning_rate:
:param pre_training_learning_rate:
:param action_bounds:
:param policy:
"""
"""Set class constants"""
self.observation_size = observation_size
self.net_arch = net_arch
self.initializer = initializer
self.activation = activation
self.clip_range = clip_range
self.value_coef = value_coef
self.entropy_coef = entropy_coef
if action_bounds is None:
action_bounds = [0.0, 1.5]
self.action_bounds = action_bounds
self.learning_rate = learning_rate
self.pre_training_learning_rate = pre_training_learning_rate
if policy is None:
policy = GaussFull()
self.policy = policy
"""Set up the tensorflow graph"""
self.graph = Graph()
with self.graph.as_default():
self.sess = Session(graph=self.graph)
""" core """
# place holders
self.observation_string_ph = placeholder(
shape=(None, 1), dtype=string, name="observation_string_ph")
self.action_ph = placeholder(dtype=float32,
shape=(None, 1),
name="action_ph")
self.old_neg_logits = placeholder(dtype=float32,
shape=(None, 1),
name="old_neg_logits")
self.advantage_ph = placeholder(dtype=float32,
shape=(None, 1),
name="advantage_ph")
self.value_target_ph = placeholder(dtype=float32,
shape=(None, 1),
name="value_target_ph")
# learning rate tensors
self.learning_rate_ph = placeholder_with_default(
input=self.learning_rate, shape=())
self.pre_training_learning_rate_ph = placeholder_with_default(
input=self.pre_training_learning_rate, shape=())
# observation tensor
replaced1 = regex_replace(self.observation_string_ph, "/", "_")
replaced2 = regex_replace(replaced1, r"\+", "-")
byte_tensor = decode_base64(replaced2)
decoded = decode_raw(byte_tensor, out_type=float32)
squeezed = squeeze(decoded, axis=1)
self.observation_input = ensure_shape(
squeezed,
shape=(None, self.observation_size),
name="observation_input")
# policy net
latent_policy = net_core(self.observation_input, self.net_arch,
self.initializer, self.activation)
self.policy.construct(latent_policy=latent_policy)
self.clipped_action = clip_by_value(
cast(self.policy.action, float32), self.action_bounds[0],
self.action_bounds[1], "clipped_action")
# value net
latent_value = net_core(self.observation_input, self.net_arch,
self.initializer, self.activation)
self.value = identity(
input=Dense(units=1,
activation=None,
kernel_initializer=self.initializer)(latent_value),
name="value")
"""loss calculation"""
# policy loss
self.neg_logits = self.policy.neg_logits_from_actions(
self.action_ph)
ratio = exp(self.old_neg_logits - self.neg_logits)
standardized_adv = (self.advantage_ph - reduce_mean(
self.advantage_ph)) / (reduce_std(self.advantage_ph) + 1e-8)
raw_policy_loss = -standardized_adv * ratio
clipped_policy_loss = -standardized_adv * clip_by_value(
ratio, 1 - self.clip_range, 1 + self.clip_range)
self.policy_loss = reduce_mean(
maximum(raw_policy_loss, clipped_policy_loss))
self.value_loss = mean_squared_error(self.value_target_ph,
self.value)
# entropy loss
self.entropy_loss = -reduce_mean(self.policy.entropy)
# total loss
self.total_loss = self.policy_loss + self.value_coef * self.value_loss + self.entropy_coef * self.entropy_loss
# optimizer
optimizer = AdamOptimizer(learning_rate=self.learning_rate_ph)
# training ops
self.training_op = optimizer.minimize(self.total_loss)
# pre training
self.dist_param_target_ph = placeholder(
dtype=float32,
shape=(None, self.policy.dist_params.shape[1]),
name="dist_param_label_ph")
self.pre_training_loss = mean_squared_error(
self.dist_param_target_ph, self.policy.dist_params)
pre_training_optimizer = GradientDescentOptimizer(
learning_rate=self.pre_training_learning_rate_ph)
self.pre_training_op = pre_training_optimizer.minimize(
self.pre_training_loss)
"""utility nodes"""
# inspect model weights
self.trainable_variables = trainable_variables()
# saviour
self.saver = Saver()
# tensorboard summaries
self.summary = merge([
histogram("values", self.value),
histogram("advantages", standardized_adv),
histogram("actions", self.clipped_action),
histogram("det_actions",
replace_nan(self.policy.det_action, 0.0)),
histogram("value_targets", self.value_target_ph),
scalar("policy_loss", self.policy_loss),
scalar("value_loss", self.value_loss),
scalar("entropy_loss", self.entropy_loss)
])
self.pre_summary = merge([
histogram("pretraining_actions", self.clipped_action),
scalar("pretraining_loss", self.pre_training_loss)
])
# initialization
init = global_variables_initializer()
self.sess.run(init)
def predict(self, observation):
"""
:param observation: input environment state
:return: action, deterministic action (mode), negative log dist value, value prediction
"""
fetches = [
self.clipped_action, self.policy.dist_params,
self.policy.neg_logits, self.value
]
action, dist_params, neg_logit, value = self.sess.run(
fetches, {self.observation_input: observation})
return action, dist_params, neg_logit, value
def train_step(self,
observations,
actions,
old_neg_logits,
value_targets,
advantages,
obs_as_string=False,
learning_rate=None,
additional_fetches=None):
fetches = [self.training_op, self.summary] + (
[] if additional_fetches is None else additional_fetches)
obs_tensor = self.observation_string_ph if obs_as_string else self.observation_input
feed_dict = {
obs_tensor: observations,
self.action_ph: actions,
self.old_neg_logits: old_neg_logits,
self.value_target_ph: value_targets,
self.advantage_ph: advantages
}
if learning_rate is not None:
feed_dict.update({self.learning_rate_ph: learning_rate})
return self.sess.run(fetches, feed_dict)
def pre_train_step(self,
observations,
dist_param_targets,
obs_as_string=False,
learning_rate=None,
additional_fetches=None):
fetches = [self.pre_training_op, self.pre_summary] + (
[] if additional_fetches is None else additional_fetches)
obs_tensor = self.observation_string_ph if obs_as_string else self.observation_input
feed_dict = {
obs_tensor: observations,
self.dist_param_target_ph: dist_param_targets
}
if learning_rate is not None:
feed_dict.update(
{self.pre_training_learning_rate_ph: learning_rate})
return self.sess.run(fetches, feed_dict)
def simple_save(self, path):
with self.graph.as_default():
simple_save(self.sess,
path,
inputs={"obs": self.observation_input},
outputs={"action": self.clipped_action})
def save(self, path):
with self.graph.as_default():
self.saver.save(sess=self.sess, save_path=path)
def restore(self, path):
with self.graph.as_default():
self.saver.restore(sess=self.sess, save_path=path)
def close_session(self):
self.sess.close()
def get_trainable_variables(self):
return self.sess.run(self.trainable_variables)
class Network(object):
def __init__(self, dimensions, batch_size, initialize_loss=True, lr=0.0001, lr_stair_width=10, lr_decay=0.95):
self.batch_size = batch_size
self.dimensions = dimensions
self.scale_factor = 2
self.layer_params = []
self.inputs = placeholder(
tf.float32, [batch_size, dimensions[1], dimensions[0], 3], name='input_images'
)
scaled_inputs = self.inputs / 256.0
print("inputs shape: " + str(self.inputs.get_shape()))
resized = resize_bicubic(
scaled_inputs,
[dimensions[1] * self.scale_factor, dimensions[0] * self.scale_factor],
name="scale_bicubic")
self.layer_params.append({
'filter_count': 64 * 3,
'filter_shape': [9, 9]
})
patch_extraction_layer = self.conv_layer("patch_extraction", self.layer_params[-1], resized)
self.layer_params.append({
'filter_count': 32 * 3,
'filter_shape': [1, 1]
})
non_linear_mapping_layer = self.conv_layer("non_linear_mapping_layer", self.layer_params[-1],
patch_extraction_layer)
self.layer_params.append({
'filter_count': 3,
'filter_shape': [5, 5]
})
self.output = self.conv_layer("reconstruction_layer", self.layer_params[-1],
non_linear_mapping_layer, linear=True)
if initialize_loss:
self.real_images = placeholder(tf.float32,
[self.batch_size,
dimensions[1] * self.scale_factor,
dimensions[0] * self.scale_factor,
3],
name='real_images')
self.loss = self.get_loss()
self.summary = tf.summary.scalar("loss", self.loss)
self.epoch = placeholder(tf.int32, name='epoch')
self.learning_rate = exponential_decay(
lr, self.epoch, lr_stair_width, lr_decay, staircase=True)
self.optimized = AdamOptimizer(
self.learning_rate,
beta1=0.9, beta2=0.999, epsilon=1e-08).minimize(self.loss)
self.sess = Session()
self.saver = Saver()
def initialize(self):
init = tf.compat.v1.global_variables_initializer()
self.sess.run(init)
@staticmethod
def weight_variable(shape, stddev=0.1):
initial = truncated_normal(shape, stddev=stddev)
return tf.Variable(initial)
@staticmethod
def bias_variable(shape, initial=0.1):
initial = tf.constant(initial, shape=shape)
return tf.Variable(initial)
def conv_layer(self, name, params, data, weight_stddev=0.1, bias_init=0.1, linear=False):
with tf.compat.v1.variable_scope(name):
weights = self.weight_variable(params['filter_shape'] + [data.get_shape().as_list()[3],
params['filter_count']], stddev=weight_stddev)
biases = self.bias_variable([params['filter_count']], initial=bias_init)
padded_data = tf.pad(data, [[0, 0],
[(params['filter_shape'][0] - 1) // 2,
(params['filter_shape'][0] - 1) // 2],
[(params['filter_shape'][1] - 1) // 2,
(params['filter_shape'][1] - 1) // 2],
[0, 0]], "SYMMETRIC")
conv = tf.nn.conv2d(padded_data, weights, strides=[1, 1, 1, 1], padding='VALID')
if not linear:
params['output'] = tf.nn.relu(conv + biases)
else:
params['output'] = conv + biases
params['biases'] = biases
params['weights'] = weights
return params['output']
def get_loss(self):
return tf.reduce_sum(tf.nn.l2_loss(self.output - tf.raw_ops.Div(x=self.real_images, y=256.0)))
def get_batch_size(self):
return self.batch_size
def get_scale_factor(self):
return self.scale_factor
def get_dimensions(self):
return self.dimensions
def train_step(self, images, target_images, epoch=0):
_, loss, lr = self.sess.run([self.optimized, self.loss, self.learning_rate], feed_dict={
self.inputs: np.array(images),
self.real_images: np.array(target_images),
self.epoch: epoch
})
return loss, lr
def validation_step(self, images, target_images):
loss = self.sess.run([self.loss], feed_dict={
self.inputs: np.array(images),
self.real_images: np.array(target_images)
})
return loss
def inference(self, images):
return self.sess.run(self.output * 256.0, feed_dict={
self.inputs: np.array(images)
})
def save(self):
save_path = self.saver.save(self.sess, os.path.abspath("network_params"))
return save_path
def load(self, path="network_params"):
self.saver.restore(self.sess, os.path.abspath(path))