def discriminator_net(x, training, opts, name='Discriminator'):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
sz = opts.img_size // 16
# input is (sz*16) x (sz*16) x nc
# state size. (sz*8) x (sz*8) x ndf
y = conv(x, opts.ndf, 4, 2, 'same', 'conv1')
y = leaky_relu(y, 0.2)
# state size. (sz*4) x (sz*4) x (ndf*2)
y = conv(y, opts.ndf * 2, 4, 2, 'same', 'conv2')
y = leaky_relu(y, 0.2)
# state size. (sz*2) x (sz*2) x (ndf*4)
y = conv(y, opts.ndf * 4, 4, 2, 'same', 'conv3')
y = leaky_relu(y, 0.2)
# state size. sz x sz x (ndf*8)
y = conv(y, opts.ndf * 8, 4, 2, 'same', 'conv4')
y = leaky_relu(y, 0.2)
flatten = tf.reshape(y, (opts.batch_size, sz * sz * opts.ndf * 8))
# discriminator output
logits = conv(y, 1, sz, 1, 'valid', 'conv5')
logits = tf.reshape(logits, (-1, 1))
with tf.variable_scope('QNet', reuse=tf.AUTO_REUSE):
# Q output
dim = opts.num_categorical + opts.num_continuous
q = tf.layers.dense(flatten, dim)
return logits, q
def self_attention(x, size, scope='self_attention_content', reuse=False):
tf.logging.debug(x)
with tf.variable_scope(scope, reuse=reuse):
C = x.shape[3]
f = utils.conv(x,
C // 2,
kernel=1,
stride=1,
pad=0,
scope='f_conv') # [bs, h, w, c']
g = utils.conv(x,
C // 2,
kernel=1,
stride=1,
pad=0,
scope='g_conv') # [bs, h, w, c']
h = utils.conv(x, C, kernel=1, stride=1, pad=0,
scope='h_conv') # [bs, h, w, c]
s = tf.matmul(utils.hw_flatten(g),
utils.hw_flatten(f),
transpose_b=True) ## [bs, N, N]
beta = tf.nn.softmax(s) # self_attention map
o = tf.matmul(beta, utils.hw_flatten(h)) # [bs, N, C]
o = tf.reshape(o, shape=size) # [bs, h, w, C]
return o
def discriminator_net(x, training, opts, name='Discriminator'):
with tf.variable_scope(name, reuse=tf.AUTO_REUSE):
sz = opts.img_size // 16
# input is (sz*16) x (sz*16) x nc
# state size. (sz*8) x (sz*8) x ndf
y = conv(x, opts.ndf, 4, 2, 'same', 'conv1')
y = leaky_relu(y, 0.2)
# state size. (sz*4) x (sz*4) x (ndf*2)
y = conv(y, opts.ndf * 2, 4, 2, 'same', 'conv2')
y = leaky_relu(y, 0.2)
# state size. (sz*2) x (sz*2) x (ndf*4)
y = conv(y, opts.ndf * 4, 4, 2, 'same', 'conv3')
y = leaky_relu(y, 0.2)
# state size. sz x sz x (ndf*8)
y = conv(y, opts.ndf * 8, 4, 2, 'same', 'conv4')
y = leaky_relu(y, 0.2)
# output
y = conv(y, 1, sz, 1, 'valid', 'conv5')
logits = tf.reshape(y, (-1, 1))
return logits
def _build_target_net(self, target_x, scope, reuse, convfeat, rep_size,
enlargement):
with tf.variable_scope(scope, reuse=reuse):
xr = tf.nn.leaky_relu(
conv(target_x,
'c1r',
nf=convfeat * 1,
rf=8,
stride=4,
init_scale=np.sqrt(2)))
xr = tf.nn.leaky_relu(
conv(xr,
'c2r',
nf=convfeat * 2 * 1,
rf=4,
stride=2,
init_scale=np.sqrt(2)))
xr = tf.nn.leaky_relu(
conv(xr,
'c3r',
nf=convfeat * 2 * 1,
rf=3,
stride=1,
init_scale=np.sqrt(2)))
rgbr = [to2d(xr)]
X_r = fc(rgbr[0], 'fc1r', nh=rep_size, init_scale=np.sqrt(2))
return X_r
def apply_policy(ph_ob, reuse, scope, hidsize, memsize, extrahid, sy_nenvs, sy_nsteps, pdparamsize):
data_format = 'NHWC'
ph = ph_ob
assert len(ph.shape.as_list()) == 5 # B,T,H,W,C
logger.info("CnnPolicy: using '%s' shape %s as image input" % (ph.name, str(ph.shape)))
X = tf.cast(ph, tf.float32) / 255.
X = tf.reshape(X, (-1, *ph.shape.as_list()[-3:]))
activ = tf.nn.relu
yes_gpu = any(get_available_gpus())
with tf.variable_scope(scope, reuse=reuse), tf.device('/gpu:0' if yes_gpu else '/cpu:0'):
X = activ(conv(X, 'c1', nf=32, rf=8, stride=4, init_scale=np.sqrt(2), data_format=data_format))
X = activ(conv(X, 'c2', nf=64, rf=4, stride=2, init_scale=np.sqrt(2), data_format=data_format))
X = activ(conv(X, 'c3', nf=64, rf=4, stride=1, init_scale=np.sqrt(2), data_format=data_format))
X = to2d(X)
mix_other_observations = [X]
X = tf.concat(mix_other_observations, axis=1)
X = activ(fc(X, 'fc1', nh=hidsize, init_scale=np.sqrt(2)))
additional_size = 448
X = activ(fc(X, 'fc_additional', nh=additional_size, init_scale=np.sqrt(2)))
snext = tf.zeros((sy_nenvs, memsize))
mix_timeout = [X]
Xtout = tf.concat(mix_timeout, axis=1)
if extrahid:
Xtout = X + activ(fc(Xtout, 'fc2val', nh=additional_size, init_scale=0.1))
X = X + activ(fc(X, 'fc2act', nh=additional_size, init_scale=0.1))
pdparam = fc(X, 'pd', nh=pdparamsize, init_scale=0.01)
vpred_int = fc(Xtout, 'vf_int', nh=1, init_scale=0.01)
vpred_ext = fc(Xtout, 'vf_ext', nh=1, init_scale=0.01)
pdparam = tf.reshape(pdparam, (sy_nenvs, sy_nsteps, pdparamsize))
vpred_int = tf.reshape(vpred_int, (sy_nenvs, sy_nsteps))
vpred_ext = tf.reshape(vpred_ext, (sy_nenvs, sy_nsteps))
return pdparam, vpred_int, vpred_ext, snext
def residual(x, in_channel, out_channel, is_training, norm):
"""residual unit with 2 layers
convolution:
width filter: 1
height filter: 3
"""
orig_x = x
with tf.variable_scope('conv1'):
conv1 = utils.conv(x, [1, 3, in_channel, out_channel],
[out_channel],
padding='SAME')
if norm:
conv1 = utils.batch_norm(conv1, is_training)
relu1 = utils.activation(conv1)
with tf.variable_scope('conv2'):
conv2 = utils.conv(relu1, [1, 3, out_channel, out_channel],
[out_channel],
padding='SAME')
if norm:
conv2 = utils.batch_norm(conv2, is_training)
with tf.variable_scope('add'):
if in_channel != out_channel:
orig_x = utils.conv(x, [1, 1, in_channel, out_channel],
[out_channel],
padding='SAME')
return utils.activation(conv2 + orig_x)
def __init__(self, sess, ob_space, ac_space, nenv, nsteps, nstack, reuse=False):
nbatch = nenv*nsteps
nh, nw, nc = ob_space.shape
ob_shape = (nbatch, nh, nw, nc*nstack)
nact = ac_space.n
X = tf.placeholder(tf.uint8, ob_shape) #obs
with tf.variable_scope("model", reuse=reuse):
h = conv(tf.cast(X, tf.float32)/255., 'c1', nf=32, rf=8, stride=4, init_scale=np.sqrt(2))
h2 = conv(h, 'c2', nf=64, rf=4, stride=2, init_scale=np.sqrt(2))
h3 = conv(h2, 'c3', nf=64, rf=3, stride=1, init_scale=np.sqrt(2))
h3 = conv_to_fc(h3)
h4 = fc(h3, 'fc1', nh=512, init_scale=np.sqrt(2))
pi = fc(h4, 'pi', nact, act=lambda x:x)
vf = fc(h4, 'v', 1, act=lambda x:x)
v0 = vf[:, 0]
a0 = sample(pi)
self.initial_state = [] #not stateful
def step(ob, *_args, **_kwargs):
a, v = sess.run([a0, v0], {X:ob})
return a, v, [] #dummy state
def value(ob, *_args, **_kwargs):
return sess.run(v0, {X:ob})
self.X = X
self.pi = pi
self.vf = vf
self.step = step
self.value = value
def apply_policy(ph_ob, ph_new, ph_istate, reuse, scope, hidsize, memsize, extrahid, sy_nenvs, sy_nsteps, pdparamsize, rec_gate_init):
data_format = 'NHWC'
ph = ph_ob
assert len(ph.shape.as_list()) == 5 # B,T,H,W,C
logger.info("CnnGruPolicy: using '%s' shape %s as image input" % (ph.name, str(ph.shape)))
X = tf.cast(ph, tf.float32) / 255.
X = tf.reshape(X, (-1, *ph.shape.as_list()[-3:]))
activ = tf.nn.relu
yes_gpu = any(get_available_gpus())
with tf.variable_scope(scope, reuse=reuse), tf.device('/gpu:0' if yes_gpu else '/cpu:0'):
X = activ(conv(X, 'c1', nf=32, rf=8, stride=4, init_scale=np.sqrt(2), data_format=data_format))
X = activ(conv(X, 'c2', nf=64, rf=4, stride=2, init_scale=np.sqrt(2), data_format=data_format))
X = activ(conv(X, 'c3', nf=64, rf=4, stride=1, init_scale=np.sqrt(2), data_format=data_format))
X = to2d(X)
X = activ(fc(X, 'fc1', nh=hidsize, init_scale=np.sqrt(2)))
X = tf.reshape(X, [sy_nenvs, sy_nsteps, hidsize])
X, snext = tf.nn.dynamic_rnn(
GRUCell(memsize, rec_gate_init=rec_gate_init), (X, ph_new[:,:,None]),
dtype=tf.float32, time_major=False, initial_state=ph_istate)
X = tf.reshape(X, (-1, memsize))
Xtout = X
if extrahid:
Xtout = X + activ(fc(Xtout, 'fc2val', nh=memsize, init_scale=0.1))
X = X + activ(fc(X, 'fc2act', nh=memsize, init_scale=0.1))
pdparam = fc(X, 'pd', nh=pdparamsize, init_scale=0.01)
vpred_int = fc(Xtout, 'vf_int', nh=1, init_scale=0.01)
vpred_ext = fc(Xtout, 'vf_ext', nh=1, init_scale=0.01)
pdparam = tf.reshape(pdparam, (sy_nenvs, sy_nsteps, pdparamsize))
vpred_int = tf.reshape(vpred_int, (sy_nenvs, sy_nsteps))
vpred_ext = tf.reshape(vpred_ext, (sy_nenvs, sy_nsteps))
return pdparam, vpred_int, vpred_ext, snext
def build_layers(self,
state,
action,
c_names,
units_1,
units_2,
w_i,
b_i,
reg=None):
with tf.variable_scope('conv1'):
conv1 = conv(state, [5, 5, 3, 6], [6], [1, 2, 2, 1], w_i, b_i)
with tf.variable_scope('conv2'):
conv2 = conv(conv1, [3, 3, 6, 12], [12], [1, 2, 2, 1], w_i, b_i)
with tf.variable_scope('flatten'):
flatten = tf.contrib.layers.flatten(conv2)
with tf.variable_scope('dense1'):
dense1 = dense(flatten, units_1, [units_1], w_i, b_i)
with tf.variable_scope('dense2'):
dense2 = dense(dense1, units_2, [units_2], w_i, b_i)
with tf.variable_scope('concat'):
concatenated = tf.concat([dense2, tf.cast(action, tf.float32)], 1)
with tf.variable_scope('dense3'):
dense3 = dense(concatenated, self.atoms, [self.atoms], w_i,
b_i) # 返回
return tf.nn.softmax(dense3)
def __init__(self, sess, ob_space, ac_space, nbatch, nsteps, reuse=False): # pylint: disable=W0613
nh, nw, nc = ob_space.shape
ob_shape = (nbatch, nh, nw, nc)
nact = ac_space.n
X = tf.placeholder(tf.uint8, ob_shape) # obs
with tf.variable_scope("model", reuse=reuse):
h = conv(tf.cast(X, tf.float32) / 255., 'c1', nf=32, rf=8, stride=4, init_scale=np.sqrt(2))
h2 = conv(h, 'c2', nf=64, rf=4, stride=2, init_scale=np.sqrt(2))
h3 = conv(h2, 'c3', nf=64, rf=3, stride=1, init_scale=np.sqrt(2))
h3 = conv_to_fc(h3)
h4 = fc(h3, 'fc1', nh=512, init_scale=np.sqrt(2))
pi = fc(h4, 'pi', nact, act=lambda x: x, init_scale=0.01)
vf = fc(h4, 'v', 1, act=lambda x: x)[:, 0]
self.pdtype = make_pdtype(ac_space)
self.pd = self.pdtype.pdfromflat(pi)
a0 = self.pd.sample()
neglogp0 = self.pd.neglogp(a0)
self.initial_state = None
def step(ob, *_args, **_kwargs):
a, v, neglogp = sess.run([a0, vf, neglogp0], {X: ob})
return a, v, self.initial_state, neglogp
def value(ob, *_args, **_kwargs):
return sess.run(vf, {X: ob})
self.X = X
self.pi = pi
self.vf = vf
self.step = step
self.value = value
def __init__(self, sess, ob_space, ac_space, nenv,
nsteps, nstack, nplayers, map_size=[15,15,32], reuse=False):
nbatch = nenv * nsteps
nh, nw, nc = ob_space.shape
ob_shape = (nbatch, nplayers, nh, nw, nc*nstack)
nact = ac_space.n
X = tf.placeholder(tf.uint8, ob_shape) #obs
C = tf.placeholder(tf.int32, [nbatch, nplayers, 2])
pis = []
vfs = []
with tf.variable_scope("model", reuse=reuse):
x = tf.reshape(tf.cast(X, tf.float32)/255., [nbatch*nplayers, nh, nw, nc*nstack])
crds = tf.cast(C, tf.float32) / map_size[0]
h = conv(x, 'c1', nf=32, rf=8, stride=4, init_scale=np.sqrt(2))
h2 = conv(h, 'c2', nf=64, rf=4, stride=2, init_scale=np.sqrt(2))
h3 = conv(h2, 'c3', nf=64, rf=3, stride=1, init_scale=np.sqrt(2))
h3 = conv_to_fc(h3)
h4 = fc(h3, 'fc1', nh=512, init_scale=np.sqrt(2))
h5 = fc(h4, 'fc2', nh=512, init_scale=np.sqrt(2))
h6 = fc(h5, 'fc3', nh=256, init_scale=np.sqrt(2)) # to give compatible network size
h6 = tf.reshape(h6, [nbatch, nplayers, -1])
h6 = tf.concat([h6, crds], axis=2)
_reuse = False
for i in range(nplayers):
pi = fc(h6[:,i], 'pi', nact, act=tf.identity, reuse=_reuse)
vf = fc(h6[:,i], 'v', 1, act=tf.identity, reuse=_reuse)
pis.append(pi)
vfs.append(vf)
_reuse = True
pi = tf.reshape(tf.concat(pis, axis=1), [nbatch*nplayers, -1])
vf = tf.reshape(tf.concat(vfs, axis=1), [nbatch*nplayers, -1])
v0 = vf
a0 = sample(pi)
self.init_map = []
self.init_state = []
def step(ob, coords, *_args, **_kwargs):
a, v = sess.run([a0, v0], {X:ob, C:coords})
a = [a[i:i+nplayers] for i in range(0, len(a), nplayers)]
v = [v[i:i+nplayers] for i in range(0, len(v), nplayers)]
return a, v, [], [] #dummy state and recon
def value(ob, coords, *_args, **_kwargs):
v = sess.run(v0, {X:ob, C:coords})
v = [v[i:i+nplayers] for i in range(0, len(v), nplayers)]
return v
self.X = X
self.C = C
self.pi = pi
self.vf = vf
self.step = step
self.value = value
def decode(x, encode_features, reuse=False):
with tf.variable_scope("decoder", reuse=reuse):
x = utils.conv(x, 256, 3, 1, scope='inv_conv4_1')
x = tf.nn.relu(x)
x = utils.upsampling(x, 2, scope='upsample_1')
x = utils.conv(x, 256, 3, 1, scope='inv_conv3_4')
x = tf.nn.relu(x)
x = utils.conv(x, 256, 3, 1, scope='inv_conv3_3')
x = tf.nn.relu(x)
x = utils.conv(x, 256, 3, 1, scope='inv_conv3_2')
x = tf.nn.relu(x)
x = utils.adaptive_instance_normalization(
x, encode_features['vgg_19/conv3/conv3_1'])
x = utils.conv(x, 128, 3, 1, scope='inv_conv3_1')
x = tf.nn.relu(x)
x = utils.upsampling(x, 2, scope='upsample_2')
x = utils.conv(x, 128, 3, 1, scope='inv_conv2_2')
x = tf.nn.relu(x)
x = utils.adaptive_instance_normalization(
x, encode_features['vgg_19/conv2/conv2_1'])
x = utils.conv(x, 64, 3, 1, scope='inv_conv2_1')
x = tf.nn.relu(x)
x = utils.upsampling(x, 2, scope='upsample_3')
x = utils.conv(x, 64, 3, 1, scope='inv_conv1_2')
x = tf.nn.relu(x)
x = utils.adaptive_instance_normalization(
x, encode_features['vgg_19/conv1/conv1_1'])
x = utils.conv(x, 3, 3, 1, scope='inv_conv1_1')
return x + 127.5
def nature_cnn(unscaled_images, **conv_kwargs):
"""
CNN from Nature paper.
"""
scaled_images = tf.cast(unscaled_images, tf.float32) / 255.
activ = tf.nn.relu
h = activ(conv(scaled_images, 'c1', nf=32, rf=8, stride=4, init_scale=np.sqrt(2),
**conv_kwargs))
h2 = activ(conv(h, 'c2', nf=64, rf=4, stride=2, init_scale=np.sqrt(2), **conv_kwargs))
h3 = activ(conv(h2, 'c3', nf=64, rf=3, stride=1, init_scale=np.sqrt(2), **conv_kwargs))
h3 = conv_to_fc(h3)
return activ(fc(h3, 'fc1', nh=512, init_scale=np.sqrt(2)))
def connvolution_process(img):
'''----------------------Reading the image-------------------------------'''
# # print(img.shape) # 3D image
# Converting the image into gray.
img = color.rgb2gray(img)
# # print(img.shape) # 2D image
# io.imshow(img)
# plt.show()
'''----------------------Preparing Filter-------------------------------'''
l1_filter = numpy.zeros((2, 3, 3))
# Vertical ditector Filter
l1_filter[0, :, :] = numpy.array([[[-1, 0, 1], [-1, 0, 1], [-1, 0, 1]]])
# Horizontal ditector Filter
l1_filter[1, :, :] = numpy.array([[[1, 1, 1], [0, 0, 0], [-1, -1, -1]]])
# # print(l1_filter)
'''---------------------- Convolutional Layer 1 ---------------------------'''
l1_feature_map = conv(img, l1_filter)
# print("l1_feature_map", l1_feature_map.shape)
l1_feature_map_relu = relu(l1_feature_map)
# print("l1_feature_map_relu", l1_feature_map_relu.shape)
l1_feature_map_relu_pool = pooling(l1_feature_map_relu, 2, 2)
# print("l1_feature_map_relu_pool", l1_feature_map_relu_pool.shape)
# print("**End of conv layer 1**\n\n")
'''---------------------- Convolutional Layer 2 ---------------------------'''
l2_filter = numpy.random.rand(3, 5, 5, l1_feature_map_relu_pool.shape[-1])
l2_feature_map = conv(l1_feature_map_relu_pool, l2_filter)
# print("l2_feature_map", l2_feature_map.shape)
l2_feature_map_relu = relu(l2_feature_map)
# print("l2_feature_map_relu", l2_feature_map_relu.shape)
l2_feature_map_relu_pool = pooling(l2_feature_map_relu, 2, 2)
# print("l2_feature_map_relu_pool", l2_feature_map_relu_pool.shape)
# print("**End of conv layer 2**\n\n")
'''---------------------- Convolutional Layer 3 ---------------------------'''
l3_filter = numpy.random.rand(1, 7, 7, l2_feature_map_relu_pool.shape[-1])
l3_feature_map = conv(l2_feature_map_relu_pool, l3_filter)
# print("l3_feature_map", l3_feature_map.shape)
l3_feature_map_relu = relu(l3_feature_map)
# print("l3_feature_map_relu", l3_feature_map_relu.shape)
l3_feature_map_relu_pool = pooling(l3_feature_map_relu, 2, 2)
# print("l3_feature_map_relu_pool", l3_feature_map_relu_pool.shape)
# print("**End of conv layer 3**\n\n")
'''---------------------- Graphing results of convolution ---------------------------'''
draw_layer(img, l1_feature_map, l1_feature_map_relu,
l1_feature_map_relu_pool, l2_feature_map, l2_feature_map_relu,
l2_feature_map_relu_pool, l3_feature_map, l3_feature_map_relu,
l3_feature_map_relu_pool)
'''---------------------- Fully Connected layer ---------------------------'''
# print("**Fully connected layer(Convolutional layer to Fully connected layer)**")
fc = l3_feature_map_relu_pool.reshape(-1)
## print(fc.shape)
return fc
def predict_next_char(s):
'''
Given string s of previous chars, returns predicted next char based on highest probability
'''
global model
while len(s) < num_previous:
s = ' ' + s
s = s[::-1]
s = s[:num_previous]
arr = list(s)
x_input = np.array([list(map(lambda c: conv(c), arr))])
return conv(model.predict(x_input))
def _build_policy_net(self, X, ph_new, ph_istate, reuse, scope, hidsize,
memsize, extrahid, sy_nenvs, sy_nsteps, pdparamsize,
rec_gate_init):
activ = tf.nn.relu
data_format = 'NHWC'
with tf.variable_scope(scope, reuse=reuse):
X = activ(
conv(X,
'c1',
nf=32,
rf=8,
stride=4,
init_scale=np.sqrt(2),
data_format=data_format))
X = activ(
conv(X,
'c2',
nf=64,
rf=4,
stride=2,
init_scale=np.sqrt(2),
data_format=data_format))
X = activ(
conv(X,
'c3',
nf=64,
rf=4,
stride=1,
init_scale=np.sqrt(2),
data_format=data_format))
X = to2d(X)
X = activ(fc(X, 'fc1', nh=hidsize, init_scale=np.sqrt(2)))
X = tf.reshape(X, [sy_nenvs, sy_nsteps, hidsize])
X, snext = tf.nn.dynamic_rnn(GRUCell(memsize,
rec_gate_init=rec_gate_init),
(X, ph_new[:, :, None]),
dtype=tf.float32,
time_major=False,
initial_state=ph_istate)
X = tf.reshape(X, (-1, memsize))
Xtout = X
if extrahid:
Xtout = X + activ(
fc(Xtout, 'fc2val', nh=memsize, init_scale=0.1))
X = X + activ(fc(X, 'fc2act', nh=memsize, init_scale=0.1))
pdparam = fc(X, 'pd', nh=pdparamsize, init_scale=0.01)
vpred_int = fc(Xtout, 'vf_int', nh=1, init_scale=0.01)
vpred_ext = fc(Xtout, 'vf_ext', nh=1, init_scale=0.01)
return pdparam, vpred_int, vpred_ext, snext
def define_bottleneck_rew(self, convfeat, rep_size, enlargement, beta=1e-2, rew_counter=None):
# convfeat=32, rep_size=64, enlargement=2, beta=0.001, rew_counter=None
logger.info("Using Curiosity Bottleneck ****************************************************")
v_target = tf.reshape(self.ph_ret_ext, (-1, 1))
if rew_counter is None:
sched_coef = 1.
else:
sched_coef = tf.minimum(rew_counter/1000, 1.)
# Random target network.
for ph in self.ph_ob.values():
if len(ph.shape.as_list()) == 5: # B,T,H,W,C. ph.shape=(None,None,84,84,4)
logger.info("CnnTarget: using '%s' shape %s as image input" % (ph.name, str(ph.shape)))
xr = ph[:,1:]
xr = tf.cast(xr, tf.float32)
xr = tf.reshape(xr, (-1, *ph.shape.as_list()[-3:]))[:, :, :, -1:] # (None, 84, 84, 1)
xr = tf.clip_by_value((xr - self.ph_mean) / self.ph_std, -5.0, 5.0) # (None, 84, 84, 1)
xr = tf.nn.leaky_relu(conv(xr, 'c1r', nf=convfeat * 1, rf=8, stride=4, init_scale=np.sqrt(2))) # (None, 20, 20, 32)
xr = tf.nn.leaky_relu(conv(xr, 'c2r', nf=convfeat * 2 * 1, rf=4, stride=2, init_scale=np.sqrt(2))) # (None, 9, 9, 64)
xr = tf.nn.leaky_relu(conv(xr, 'c3r', nf=convfeat * 2 * 1, rf=3, stride=1, init_scale=np.sqrt(2))) # (None, 7, 7, 64)
rgbr = [to2d(xr)] # (None, 3136)
mu = fc(rgbr[0], 'fc_mu', nh=rep_size, init_scale=np.sqrt(2)) # (None, 64)
sigma = tf.nn.softplus(fc(rgbr[0], 'fc_sigma', nh=rep_size, init_scale=np.sqrt(2))) # (None, 64)
z = mu + sigma * tf.random_normal(tf.shape(mu), 0, 1, dtype=tf.float32) # (None, 64)
v = fc(z, 'value', nh=1, init_scale=np.sqrt(2)) # (None, 64)
self.feat_var = tf.reduce_mean(sigma)
self.max_feat = tf.reduce_max(tf.abs(z))
self.kl = 0.5 * tf.reduce_sum(
tf.square(mu) + tf.square(sigma) - tf.log(1e-8 + tf.square(sigma)) - 1,
axis=-1, keep_dims=True)
self.int_rew = tf.stop_gradient(self.kl)
self.int_rew = tf.reshape(self.int_rew, (self.sy_nenvs, self.sy_nsteps - 1))
self.aux_loss_raw = sched_coef * tf.square(v_target - v) + beta * self.kl
# self.aux_loss_raw = beta * self.kl
self.aux_loss = sched_coef * tf.square(v_target - v) + beta * self.kl # (None, 1)
# mask 是 0-1 之间的随机数
mask = tf.random_uniform(shape=tf.shape(self.aux_loss), minval=0., maxval=1., dtype=tf.float32) # (None, 1)
# 全为 true 的矩阵. shape=(None,1)
mask = tf.cast(mask < self.proportion_of_exp_used_for_predictor_update, tf.float32) # (None, 1)
# 对 aux_loss.shape=(None,1) 的每个位置取平均
self.aux_loss = tf.reduce_sum(mask * self.aux_loss) / tf.maximum(tf.reduce_sum(mask), 1.) # (None, )
self.v_int = v # (None,1)
def crn(conv5, weights_file):
# Contextual Reweighting Network
conv5 = downsample(conv5)
# g Multiscale Context Filters, dimension is Bx13x13x84
convg3x3 = conv(conv5,
3,
3,
32,
1,
1,
name='convg3x3',
trainable=True,
initializer=tf.contrib.layers.xavier_initializer())
convg5x5 = conv(conv5,
5,
5,
32,
1,
1,
name='convg5x5',
trainable=True,
initializer=tf.contrib.layers.xavier_initializer())
convg7x7 = conv(conv5,
7,
7,
20,
1,
1,
name='convg7x7',
trainable=True,
initializer=tf.contrib.layers.xavier_initializer())
# convg = tf.concat([convg3x3, convg5x5, convg7x7], 3)
convg = tf.concat(3, [convg3x3, convg5x5, convg7x7])
# w Accumulation Weight, 13x13x84 to 13x13x1
convw = conv(convg,
1,
1,
1,
1,
1,
name='convw',
trainable=True,
initializer=tf.contrib.layers.xavier_initializer())
# Bx13x13x1 to BxWxHx1
m = upsample(convw)
return m
def roberts(image):
gx = np.array([[1, 0],
[0, -1]])
gy = np.array([[0, 1],
[-1, 0]])
hx = conv(image, gx)
hy = conv(image, gy)
h = np.abs(hx) + np.abs(hy)
h = np_value_range(h)
h = normalize(h)
show(hx, "Roberts-hx")
show(hy, "Roberts-hy")
show(h, "Roberts-h")
save("roberts-fig")
return h, "roberts.jpg"
def predict_next_chars(s):
'''
Returns array of all predicted chars sorted in decreasing probability
'''
global model
while len(s) < num_previous:
s = ' ' + s
s = s[::-1]
s = s[:num_previous]
arr = list(s)
x_input = np.array([list(map(lambda c: conv(c), arr))])
probs = model.predict_proba(x_input).flatten().tolist()
chars = list(zip([conv(i) for i in range(len(probs))], probs))
chars = sorted(chars, key=lambda c: c[1], reverse=True)
return list(map(lambda c: c[0], chars))
def c(i, l, l_):
o = conv(filters=layer_dict[l_] * self.filter_num,
kernel_size=3,
strides=1)(i)
o = norm_layer(o)
o = act_layer(o)
return o
def _decode(self):
with tf.variable_scope("output_layer"):
start_logits = tf.squeeze(
conv(tf.concat([self.enc[1], self.enc[2]], axis=-1),
1,
bias=False,
name="start_pointer"), -1)
end_logits = tf.squeeze(
conv(tf.concat([self.enc[1], self.enc[3]], axis=-1),
1,
bias=False,
name="end_pointer"), -1)
# self.logits = [mask_logits(start_logits, mask=self.p_mask),
# mask_logits(end_logits, mask=self.p_mask)]
self.start_probs, self.end_probs = start_logits, end_logits
def model2(X, nact, scope, reuse = False, layer_norm = False):
# X should be nbatch * ncol * nrow * 2 (boolean)
with tf.variable_scope(scope, reuse = reuse):
h = conv(tf.cast(X, tf.float32), 'c1', nf = 32, rf = 8, stride = 1, init_scale = np.sqrt(2))
# x = layers.layer_norm(x, scale = True, center = True)
h2 = conv(h, 'c2', nf = 64, rf = 4, stride = 1, init_scale = np.sqrt(2))
h3 = conv(h2, 'c3', nf = 64, rf = 3, stride = 1, init_scale = np.sqrt(2))
h3 = conv_to_fc(h3)
h4 = fc(h3, 'fc1', nh = 512, init_scale = np.sqrt(2))
pi = fc(h4, 'pi', nact, act = lambda x : x)
vf = fc(h4, 'v', 1, act = lambda x : tf.tanh(x))
# filter out non-valid actions from pi
valid = tf.reduce_max(tf.cast(X, tf.float32), axis = 1)
valid_flat = tf.reshape(valid, [-1, nact]) # this is the equavalent of "ind_flat_filter"
pi_fil = pi + (valid_flat - tf.ones(tf.shape(valid_flat))) * 1e32
return pi_fil, vf[:, 0]
def prewitt(image):
gx = np.array([[-1, -1, -1],
[0, 0, 0],
[1, 1, 1]])
gy = np.array([[-1, 0, 1],
[-1, 0, 1],
[-1, 0, 1]])
hx = conv(image, gx)
hy = conv(image, gy)
h = np.sqrt(np.power(hx, 2.0) + np.power(hy, 2.0))
h = np_value_range(h)
h = normalize(h)
show(hx, "Prewitt-hx")
show(hy, "Prewitt-hy")
show(h, "Prewitt-h")
save("prewitt-fig")
return h, "prewitt.jpg"
def sobel(image):
gx = np.array([[-1, -2, -1],
[0, 0, 0],
[1, 2, 1]])
gy = np.array([[-1, 0, 1],
[-2, 0, 2],
[-1, 0, 1]])
hx = conv(image, gx)
hy = conv(image, gy)
h = np.sqrt(np.power(hx, 2.0) + np.power(hy, 2.0))
h = np_value_range(h)
h = normalize(h)
show(hx, "Sobel-hx")
show(hy, "Sobel-hy")
show(h, "Sobel-h")
save("Sobel-fig")
return h, "sobel.jpg"
def scharr(image):
gx = np.array([[-3, -10, -3],
[0, 0, 0],
[3, 10, 3]])
gy = np.array([[-3, 0, 3],
[-10, 0, 10],
[-3, 0, 3]])
hx = conv(image, gx)
hy = conv(image, gy)
h = np.sqrt(np.power(hx, 2.0) + np.power(hy, 2.0))
h = np_value_range(h)
h = normalize(h)
show(hx, "Scharr-hx")
show(hy, "Scharr-hy")
show(h, "Scharr-h")
save("scharr-fig")
return h, "scharr.jpg"
def _model(self, X, reuse=False):
with tf.variable_scope(self.scope, reuse=reuse):
conv1 = conv(X, [4, 4, self.depth, 32], [1, 4, 4, 1],
activation_fn=tf.nn.relu, scope="conv1")
conv2 = conv(conv1, [4, 4, 32, 64], [1, 2, 2, 1],
activation_fn=tf.nn.relu, scope="conv2")
conv3 = conv(conv2, [3, 3, 64, 64], [1, 1, 1, 1],
activation_fn=tf.nn.relu, scope="conv3")
flt, dim = flatten(conv3)
fc1 = fc(flt, dim, 512, scope="fc1")
return fc(fc1, 512, self.a, scope="fc2")
"""
def frn(featuremap, W, H, D):
D_sqrt = int(math.sqrt(D))
# transpose feature from BxWxHxD to BxDxWxH
CROW = tf.transpose(featuremap, [0, 3, 1, 2])
# reshape feature from BxDxWxH to BxDsxDsx(WxH)
CROW = tf.reshape(CROW, shape=[-1, D_sqrt, D_sqrt, W * H])
# BxDsxDsx(WxH) to BxDsxDsx(WxH)
# conv(input, filter_height, filter_width, num_filters, stride_y, stride_x, name, padding='SAME', groups=1, trainable=False, init_wb=None, initializer=None)
convd3x3 = conv(CROW,
3,
3,
W * H,
1,
1,
name='convd3x3',
padding='SAME',
trainable=True)
# BxDsxDsx(WxH) to BxDsxDsx1
# 왜 (WxH) > 1 로 변환하지?? WxH 크기 그대로이지 않나??
convd = conv(convd3x3,
1,
1,
W * H,
1,
1,
name='convd',
padding='SAME',
trainable=True)
# BxDsxDsx1 to BxD
convd = tf.reshape(
convd, [-1, D]) # spatial weight인줄 알았는데, D 차원인거 보니 channel weight인듯
convd = tf.nn.softmax(convd)
# BxD multiply BxWxHxD
convd_tile = tf.tile(convd, [1, W, H, 1])
featuremap_with_Dweights = tf.multiply(convd_tile, featuremap)
return featuremap_with_Dweights
def model(X, nact, scope, reuse=False, layer_norm=False):
with tf.variable_scope(scope, reuse=reuse):
h = conv(tf.cast(X, tf.float32), 'c1', nf=32, rf=8, stride=1, init_scale=np.sqrt(2)) # TODO: when upgraded to batch run, add layer_norm to conv
# x = layers.layer_norm(x, scale=True, center=True)
h2 = conv(h, 'c2', nf=64, rf=4, stride=1, init_scale=np.sqrt(2))
h3 = conv(h2, 'c3', nf=64, rf=3, stride=1, init_scale=np.sqrt(2))
h3 = conv_to_fc(h3)
h4 = fc(h3, 'fc1', nh=512, init_scale=np.sqrt(2))
pi = fc(h4, 'pi', nact, act=lambda x: x)
vf = fc(h4, 'v', 1, act=lambda x: tf.tanh(x))
pos = tf.reduce_max(X, axis = 1) # Comments by Fei: get 1 if the postive variable exists in any clauses, otherwise 0
neg = tf.reduce_min(X, axis = 1) # Comments by Fei: get -1 if the negative variables exists in any clauses, otherwise 0
ind = tf.concat([pos, neg], axis = 2) # Comments by Fei: get (1, -1) if this var is present, (1, 0) if only as positive, (0, -1) if only as negative
ind_flat = tf.reshape(ind, [-1, nact]) # Comments by Fei: this is nbatch * nact, with 0 values labeling non_valid actions, 1 or -1 for other
ind_flat_filter = tf.abs(tf.cast(ind_flat, tf.float32)) # Comments by Fei: this is nbatch * nact, with 0 values labeling non_valid actions, 1 for other
#pi_fil = pi + (ind_flat_filter - tf.ones(tf.shape(ind_flat_filter))) * 1e32
pi_fil = pi + (ind_flat_filter - tf.ones(tf.shape(ind_flat_filter))) * 1e32
return pi_fil, vf[:, 0]
def build_layers(self,
state,
c_names,
units_1,
units_2,
w_i,
b_i,
reg=None):
with tf.variable_scope('conv1'):
conv1 = conv(state, [5, 5, 3, 6], [6], [1, 2, 2, 1], w_i, b_i)
with tf.variable_scope('conv2'):
conv2 = conv(conv1, [3, 3, 6, 12], [12], [1, 2, 2, 1], w_i, b_i)
with tf.variable_scope('flatten'):
flatten = tf.contrib.layers.flatten(conv2)
with tf.variable_scope('dense1'):
dense1 = noisy_dense(
flatten,
units_1, [units_1],
c_names,
w_i,
b_i,
noisy_distribution=self.config.noisy_distribution)
with tf.variable_scope('dense2'):
dense2 = noisy_dense(
dense1,
units_2, [units_2],
c_names,
w_i,
b_i,
noisy_distribution=self.config.noisy_distribution)
with tf.variable_scope('dense3'):
dense3 = noisy_dense(
dense2,
self.action_dim, [self.action_dim],
c_names,
w_i,
b_i,
noisy_distribution=self.config.noisy_distribution)
return dense3
directList = ["sign1","sign2","sign3","sign4","sign5"] for kin in range(1,6): gms = [] testdata = [] for d in directList: #d = directList[1] model, test = dogmmsig(d,ki = kin) gms.append(model) testdata.append(test) # Test Data on model: resultList = [] original = [] for d in range(len(directList)): result = [] for a in range(len(testdata[d])): resulttemp = np.zeros(len(directList)) for d2 in range(len(directList)): resulttemp[d2] = (gms[d2].loglikelihood(testdata[d][a][:,:])) # print result[a] result.append(resulttemp) original.append(directList[d]) resultList.append(result) # classify most likely model classified = mostlikely(resultList,directList) #print classified #print original conv(original,classified) print "done with k = ", kin
